aws-sdk-core 3.125.5 → 3.130.0

data/lib/aws-sdk-core/plugins/checksum_algorithm.rb

@@ -0,0 +1,340 @@
+# frozen_string_literal: true
+
+module Aws
+  module Plugins
+    # @api private
+    class ChecksumAlgorithm < Seahorse::Client::Plugin
+      CHUNK_SIZE = 1 * 1024 * 1024 # one MB
+
+      # determine the set of supported client side checksum algorithms
+      # CRC32c requires aws-crt (optional sdk dependency) for support
+      CLIENT_ALGORITHMS = begin
+        supported = %w[SHA256 SHA1 CRC32]
+        begin
+          require 'aws-crt'
+          supported << 'CRC32C'
+        rescue LoadError
+        end
+        supported
+      end.freeze
+
+      # priority order of checksum algorithms to validate responses against
+      # Remove any algorithms not supported by client (ie, depending on CRT availability)
+      CHECKSUM_ALGORITHM_PRIORITIES = %w[CRC32C SHA1 CRC32 SHA256] & CLIENT_ALGORITHMS
+
+      # byte size of checksums, used in computing the trailer length
+      CHECKSUM_SIZE = {
+        'CRC32' => 16,
+        'CRC32C' => 16,
+        'SHA1' => 36,
+        'SHA256' => 52
+      }
+
+      # Interface for computing digests on request/response bodies
+      # which may be files, strings or IO like objects
+      # Applies only to digest functions that produce 32 bit integer checksums
+      # (eg CRC32)
+      class Digest32
+
+        attr_reader :value
+
+        # @param [Object] digest_fn
+        def initialize(digest_fn)
+          @digest_fn = digest_fn
+          @value = 0
+        end
+
+        def update(chunk)
+          @value = @digest_fn.call(chunk, @value)
+        end
+
+        def base64digest
+          Base64.encode64([@value].pack('N')).chomp
+        end
+      end
+
+      def add_handlers(handlers, _config)
+        handlers.add(OptionHandler, step: :initialize)
+        # priority set low to ensure checksum is computed AFTER the request is
+        # built but before it is signed
+        handlers.add(ChecksumHandler, priority: 15, step: :build)
+      end
+
+      private
+
+      def self.request_algorithm_selection(context)
+        return unless context.operation.http_checksum
+
+        input_member = context.operation.http_checksum['requestAlgorithmMember']
+        context.params[input_member.to_sym]&.upcase if input_member
+      end
+
+      def self.request_validation_mode(context)
+        return unless context.operation.http_checksum
+
+        input_member = context.operation.http_checksum['requestValidationModeMember']
+        context.params[input_member.to_sym] if input_member
+      end
+
+      def self.operation_response_algorithms(context)
+        return unless context.operation.http_checksum
+
+        context.operation.http_checksum['responseAlgorithms']
+      end
+
+
+      # @api private
+      class OptionHandler < Seahorse::Client::Handler
+        def call(context)
+          context[:http_checksum] ||= {}
+
+          # validate request configuration
+          if (request_input = ChecksumAlgorithm.request_algorithm_selection(context))
+            unless CLIENT_ALGORITHMS.include? request_input
+              if (request_input == 'CRC32C')
+                raise ArgumentError, "CRC32C requires crt support - install the aws-crt gem for support."
+              else
+                raise ArgumentError, "#{request_input} is not a supported checksum algorithm."
+              end
+            end
+          end
+
+        # validate response configuration
+          if (ChecksumAlgorithm.request_validation_mode(context))
+            # Compute an ordered list as the union between priority supported and the
+            # operation's modeled response algorithms.
+            validation_list = CHECKSUM_ALGORITHM_PRIORITIES &
+              ChecksumAlgorithm.operation_response_algorithms(context)
+            context[:http_checksum][:validation_list] = validation_list
+          end
+
+          @handler.call(context)
+        end
+      end
+
+      # @api private
+      class ChecksumHandler < Seahorse::Client::Handler
+
+        def call(context)
+          if should_calculate_request_checksum?(context)
+            request_algorithm_input = ChecksumAlgorithm.request_algorithm_selection(context)
+            context[:checksum_algorithms] = request_algorithm_input
+
+            request_checksum_property = {
+              'algorithm' => request_algorithm_input,
+              'in' => checksum_request_in(context),
+              'name' => "x-amz-checksum-#{request_algorithm_input.downcase}"
+            }
+
+            calculate_request_checksum(context, request_checksum_property)
+          end
+
+          if should_verify_response_checksum?(context)
+            add_verify_response_checksum_handlers(context)
+          end
+
+          @handler.call(context)
+        end
+
+        private
+
+        def should_calculate_request_checksum?(context)
+          context.operation.http_checksum &&
+            ChecksumAlgorithm.request_algorithm_selection(context)
+        end
+
+        def should_verify_response_checksum?(context)
+          context[:http_checksum][:validation_list] && !context[:http_checksum][:validation_list].empty?
+        end
+
+        def calculate_request_checksum(context, checksum_properties)
+          case checksum_properties['in']
+          when 'header'
+            header_name = checksum_properties['name']
+            body = context.http_request.body_contents
+            if body
+              context.http_request.headers[header_name] ||=
+                ChecksumAlgorithm.calculate_checksum(checksum_properties['algorithm'], body)
+            end
+          when 'trailer'
+            apply_request_trailer_checksum(context, checksum_properties)
+          end
+        end
+
+        def apply_request_trailer_checksum(context, checksum_properties)
+          location_name = checksum_properties['name']
+
+          # set required headers
+          headers = context.http_request.headers
+          headers['Content-Encoding'] = 'aws-chunked'
+          headers['X-Amz-Content-Sha256'] = 'STREAMING-UNSIGNED-PAYLOAD-TRAILER'
+          headers['X-Amz-Trailer'] = location_name
+
+          # We currently always compute the size in the modified body wrapper - allowing us
+          # to set the Content-Length header (set by content_length plugin).
+          # This means we cannot use Transfer-Encoding=chunked
+
+          if !context.http_request.body.respond_to?(:size)
+            raise Aws::Errors::ChecksumError, 'Could not determine length of the body'
+          end
+          headers['X-Amz-Decoded-Content-Length'] = context.http_request.body.size
+
+          context.http_request.body = AwsChunkedTrailerDigestIO.new(
+            context.http_request.body,
+            checksum_properties['algorithm'],
+            location_name
+          )
+        end
+
+        # Add events to the http_response to verify the checksum as its read
+        # This prevents the body from being read multiple times
+        # verification is done only once a successful response has completed
+        def add_verify_response_checksum_handlers(context)
+          http_response = context.http_response
+          checksum_context = { }
+          http_response.on_headers do |_status, headers|
+            header_name, algorithm = response_header_to_verify(headers, context[:http_checksum][:validation_list])
+            if header_name
+              expected = headers[header_name]
+
+              unless context[:http_checksum][:skip_on_suffix] && /-[\d]+$/.match(expected)
+                checksum_context[:algorithm] = algorithm
+                checksum_context[:header_name] = header_name
+                checksum_context[:digest] = ChecksumAlgorithm.digest_for_algorithm(algorithm)
+                checksum_context[:expected] = expected
+              end
+            end
+          end
+
+          http_response.on_data do |chunk|
+            checksum_context[:digest].update(chunk) if checksum_context[:digest]
+          end
+
+          http_response.on_success do
+            if checksum_context[:digest] &&
+              (computed = checksum_context[:digest].base64digest)
+
+              if computed != checksum_context[:expected]
+                raise Aws::Errors::ChecksumError,
+                      "Checksum validation failed on #{checksum_context[:header_name]} "\
+                      "computed: #{computed}, expected: #{checksum_context[:expected]}"
+              end
+
+              context[:http_checksum][:validated] = checksum_context[:algorithm]
+            end
+          end
+        end
+
+        # returns nil if no headers to verify
+        def response_header_to_verify(headers, validation_list)
+          validation_list.each do |algorithm|
+            header_name = "x-amz-checksum-#{algorithm}"
+            return [header_name, algorithm] if headers[header_name]
+          end
+          nil
+        end
+
+        # determine where (header vs trailer) a request checksum should be added
+        def checksum_request_in(context)
+          if context.operation['authtype'].eql?('v4-unsigned-body')
+            'trailer'
+          else
+            'header'
+          end
+        end
+
+      end
+
+      def self.calculate_checksum(algorithm, body)
+        digest = ChecksumAlgorithm.digest_for_algorithm(algorithm)
+        if body.respond_to?(:read)
+          ChecksumAlgorithm.update_in_chunks(digest, body)
+        else
+          digest.update(body)
+        end
+        digest.base64digest
+      end
+
+      def self.digest_for_algorithm(algorithm)
+        case algorithm
+        when 'CRC32'
+          Digest32.new(Zlib.method(:crc32))
+        when 'CRC32C'
+          # this will only be used if input algorithm is CRC32C AND client supports it (crt available)
+          Digest32.new(Aws::Crt::Checksums.method(:crc32c))
+        when 'SHA1'
+          Digest::SHA1.new
+        when 'SHA256'
+          Digest::SHA256.new
+        end
+      end
+
+      # The trailer size (in bytes) is the overhead + the trailer name +
+      # the length of the base64 encoded checksum
+      def self.trailer_length(algorithm, location_name)
+        CHECKSUM_SIZE[algorithm] + location_name.size
+      end
+
+      def self.update_in_chunks(digest, io)
+        loop do
+          chunk = io.read(CHUNK_SIZE)
+          break unless chunk
+          digest.update(chunk)
+        end
+        io.rewind
+      end
+
+      # Wrapper for request body that implements application-layer
+      # chunking with Digest computed on chunks + added as a trailer
+      class AwsChunkedTrailerDigestIO
+        CHUNK_SIZE = 16384
+
+        def initialize(io, algorithm, location_name)
+          @io = io
+          @location_name = location_name
+          @algorithm = algorithm
+          @digest = ChecksumAlgorithm.digest_for_algorithm(algorithm)
+          @trailer_io = nil
+        end
+
+        # the size of the application layer aws-chunked + trailer body
+        def size
+          # compute the number of chunks
+          # a full chunk has 4 + 4 bytes overhead, a partial chunk is len.to_s(16).size + 4
+          orig_body_size = @io.size
+          n_full_chunks = orig_body_size / CHUNK_SIZE
+          partial_bytes = orig_body_size % CHUNK_SIZE
+          chunked_body_size = n_full_chunks * (CHUNK_SIZE + 8)
+          chunked_body_size += partial_bytes.to_s(16).size + partial_bytes + 4 unless  partial_bytes.zero?
+          trailer_size = ChecksumAlgorithm.trailer_length(@algorithm, @location_name)
+          chunked_body_size + trailer_size
+        end
+
+        def rewind
+          @io.rewind
+        end
+
+        def read(length, buf)
+          # account for possible leftover bytes at the end, if we have trailer bytes, send them
+          if @trailer_io
+            return @trailer_io.read(length, buf)
+          end
+
+          chunk = @io.read(length)
+          if chunk
+            @digest.update(chunk)
+            application_chunked = "#{chunk.bytesize.to_s(16)}\r\n#{chunk}\r\n"
+            return StringIO.new(application_chunked).read(application_chunked.size, buf)
+          else
+            trailers = {}
+            trailers[@location_name] = @digest.base64digest
+            trailers = trailers.map { |k,v| "#{k}:#{v}"}.join("\r\n")
+            @trailer_io = StringIO.new("0\r\n#{trailers}\r\n\r\n")
+            chunk = @trailer_io.read(length, buf)
+          end
+          chunk
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-core/plugins/http_checksum.rb

@@ -11,7 +11,8 @@ module Aws
         CHUNK_SIZE = 1 * 1024 * 1024 # one MB
 
         def call(context)
-          if context.operation.http_checksum_required
+          if checksum_required?(context) &&
+             !context[:checksum_algorithms] # skip in favor of flexible checksum
             body = context.http_request.body
             context.http_request.headers['Content-Md5'] ||= md5(body)
           end
@@ -20,6 +21,12 @@ module Aws
 
         private
 
+        def checksum_required?(context)
+          context.operation.http_checksum_required ||
+            (context.operation.http_checksum &&
+              context.operation.http_checksum['requestChecksumRequired'])
+        end
+
         # @param [File, Tempfile, IO#read, String] value
         # @return [String<MD5>]
         def md5(value)

data/lib/aws-sdk-core/plugins/recursion_detection.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Aws
+  module Plugins
+    # @api private
+    class RecursionDetection < Seahorse::Client::Plugin
+
+      # @api private
+      class Handler < Seahorse::Client::Handler
+        def call(context)
+
+          unless context.http_request.headers.key?('x-amz-trace-id')
+            if ENV['AWS_LAMBDA_FUNCTION_NAME'] &&
+              (trace_id = ENV['_X_AMZ_TRACE_ID'])
+              context.http_request.headers['x-amz-trace-id'] = trace_id
+            end
+          end
+          @handler.call(context)
+        end
+      end
+
+      # should be at the end of build so that
+      # modeled traits / service customizations apply first
+      handler(Handler, step: :build, order: 99)
+    end
+  end
+end

data/lib/aws-sdk-core/plugins/response_paging.rb

@@ -10,7 +10,7 @@ module Aws
         def call(context)
           context[:original_params] = context.params
           resp = @handler.call(context)
-          resp.extend(PageableResponse)
+          PageableResponse.apply(resp)
           resp.pager = context.operation[:pager] || Aws::Pager::NullPager.new
           resp
         end

data/lib/aws-sdk-core/plugins/retries/error_inspector.rb

@@ -82,7 +82,7 @@ module Aws
         end
 
         def checksum?
-          CHECKSUM_ERRORS.include?(@name) || @error.is_a?(Errors::ChecksumError)
+          CHECKSUM_ERRORS.include?(@name)
         end
 
         def networking?

data/lib/aws-sdk-core/plugins/signature_v4.rb

@@ -98,6 +98,7 @@ module Aws
           req.headers.delete('Authorization')
           req.headers.delete('X-Amz-Security-Token')
           req.headers.delete('X-Amz-Date')
+          req.headers.delete('x-Amz-Region-Set')
 
           if context.config.respond_to?(:clock_skew) &&
              context.config.clock_skew &&
@@ -134,7 +135,7 @@ module Aws
         def apply_authtype(context)
           if context.operation['authtype'].eql?('v4-unsigned-body') &&
              context.http_request.endpoint.scheme.eql?('https')
-            context.http_request.headers['X-Amz-Content-Sha256'] = 'UNSIGNED-PAYLOAD'
+            context.http_request.headers['X-Amz-Content-Sha256'] ||= 'UNSIGNED-PAYLOAD'
           end
           context
         end

data/lib/aws-sdk-core/plugins/stub_responses.rb

@@ -51,7 +51,11 @@ requests are made, and retries are disabled.
           stub = context.client.next_stub(context)
           resp = Seahorse::Client::Response.new(context: context)
           async_mode = context.client.is_a? Seahorse::Client::AsyncBase
-          apply_stub(stub, resp, async_mode)
+          if Hash === stub && stub[:mutex]
+            stub[:mutex].synchronize { apply_stub(stub, resp, async_mode) }
+          else
+            apply_stub(stub, resp, async_mode)
+          end
 
           async_mode ? Seahorse::Client::AsyncResponse.new(
             context: context, stream: context[:input_event_stream_handler].event_emitter.stream, sync_queue: Queue.new) : resp

data/lib/aws-sdk-core/process_credentials.rb

@@ -27,6 +27,7 @@ module Aws
     def initialize(process)
       @process = process
       @credentials = credentials_from_process(@process)
+      @async_refresh = false
 
       super
     end
@@ -73,9 +74,9 @@ module Aws
       @credentials = credentials_from_process(@process)
     end
 
-    def near_expiration?
+    def near_expiration?(expiration_length)
       # are we within 5 minutes of expiration?
-      @expiration && (Time.now.to_i + 5 * 60) > @expiration.to_i
+      @expiration && (Time.now.to_i + expiration_length) > @expiration.to_i
     end
   end
 end

data/lib/aws-sdk-core/refreshing_credentials.rb

@@ -17,45 +17,74 @@ module Aws
   # @api private
   module RefreshingCredentials
 
+    SYNC_EXPIRATION_LENGTH = 300 # 5 minutes
+    ASYNC_EXPIRATION_LENGTH = 600 # 10 minutes
+
     def initialize(options = {})
       @mutex = Mutex.new
+      @before_refresh = options.delete(:before_refresh) if Hash === options
+
+      @before_refresh.call(self) if @before_refresh
       refresh
     end
 
     # @return [Credentials]
     def credentials
-      refresh_if_near_expiration
+      refresh_if_near_expiration!
       @credentials
     end
 
     # @return [Time,nil]
     def expiration
-      refresh_if_near_expiration
+      refresh_if_near_expiration!
       @expiration
     end
 
     # Refresh credentials.
     # @return [void]
     def refresh!
-      @mutex.synchronize { refresh }
+      @mutex.synchronize do
+        @before_refresh.call(self) if @before_refresh
+
+        refresh
+      end
     end
 
     private
 
-    # Refreshes instance metadata credentials if they are within
-    # 5 minutes of expiration.
-    def refresh_if_near_expiration
-      if near_expiration?
+    # Refreshes credentials asynchronously and synchronously.
+    # If we are near to expiration, block while getting new credentials.
+    # Otherwise, if we're approaching expiration, use the existing credentials
+    # but attempt a refresh in the background.
+    def refresh_if_near_expiration!
+      # Note: This check is an optimization. Rather than acquire the mutex on every #refresh_if_near_expiration
+      # call, we check before doing so, and then we check within the mutex to avoid a race condition.
+      # See issue: https://github.com/aws/aws-sdk-ruby/issues/2641 for more info.
+      if near_expiration?(SYNC_EXPIRATION_LENGTH)
         @mutex.synchronize do
-          refresh if near_expiration?
+          if near_expiration?(SYNC_EXPIRATION_LENGTH)
+            @before_refresh.call(self) if @before_refresh
+            refresh
+          end
+        end
+      elsif @async_refresh && near_expiration?(ASYNC_EXPIRATION_LENGTH)
+        unless @mutex.locked?
+          Thread.new do
+            @mutex.synchronize do
+              if near_expiration?(ASYNC_EXPIRATION_LENGTH)
+                @before_refresh.call(self) if @before_refresh
+                refresh
+              end
+            end
+          end
         end
       end
     end
 
-    def near_expiration?
+    def near_expiration?(expiration_length)
       if @expiration
-        # are we within 5 minutes of expiration?
-        (Time.now.to_i + 5 * 60) > @expiration.to_i
+        # Are we within expiration?
+        (Time.now.to_i + expiration_length) > @expiration.to_i
       else
         true
       end

data/lib/aws-sdk-core/shared_config.rb

@@ -100,7 +100,7 @@ module Aws
     #   or `nil` if no valid credentials were found.
     def credentials(opts = {})
       p = opts[:profile] || @profile_name
-      validate_profile_exists(p) if credentials_present?
+      validate_profile_exists(p)
       if (credentials = credentials_from_shared(p, opts))
         credentials
       elsif (credentials = credentials_from_config(p, opts))
@@ -195,11 +195,6 @@ module Aws
       value
     end
 
-    def credentials_present?
-      (@parsed_credentials && !@parsed_credentials.empty?) ||
-        (@parsed_config && !@parsed_config.empty?)
-    end
-
     def assume_role_from_profile(cfg, profile, opts, chain_config)
       if cfg && prof_cfg = cfg[profile]
         opts[:source_profile] ||= prof_cfg['source_profile']

data/lib/aws-sdk-core/sso_credentials.rb

@@ -63,6 +63,11 @@ module Aws
     #
     # @option options [SSO::Client] :client Optional `SSO::Client`.  If not
     #   provided, a client will be constructed.
+    #
+    # @option options [Callable] before_refresh Proc called before
+    #   credentials are refreshed. `before_refresh` is called
+    #   with an instance of this object when
+    #   AWS credentials are required and need to be refreshed.
     def initialize(options = {})
 
       missing_keys = SSO_REQUIRED_OPTS.select { |k| options[k].nil? }
@@ -81,6 +86,7 @@ module Aws
       options[:region] = @sso_region
       options[:credentials] = nil
       @client = options[:client] || Aws::SSO::Client.new(options)
+      @async_refresh = true
       super
     end
 

data/lib/aws-sdk-sso/client.rb

@@ -27,7 +27,9 @@ require 'aws-sdk-core/plugins/client_metrics_plugin.rb'
 require 'aws-sdk-core/plugins/client_metrics_send_plugin.rb'
 require 'aws-sdk-core/plugins/transfer_encoding.rb'
 require 'aws-sdk-core/plugins/http_checksum.rb'
+require 'aws-sdk-core/plugins/checksum_algorithm.rb'
 require 'aws-sdk-core/plugins/defaults_mode.rb'
+require 'aws-sdk-core/plugins/recursion_detection.rb'
 require 'aws-sdk-core/plugins/signature_v4.rb'
 require 'aws-sdk-core/plugins/protocols/rest_json.rb'
 
@@ -74,7 +76,9 @@ module Aws::SSO
     add_plugin(Aws::Plugins::ClientMetricsSendPlugin)
     add_plugin(Aws::Plugins::TransferEncoding)
     add_plugin(Aws::Plugins::HttpChecksum)
+    add_plugin(Aws::Plugins::ChecksumAlgorithm)
     add_plugin(Aws::Plugins::DefaultsMode)
+    add_plugin(Aws::Plugins::RecursionDetection)
     add_plugin(Aws::Plugins::SignatureV4)
     add_plugin(Aws::Plugins::Protocols::RestJson)
 
@@ -541,7 +545,7 @@ module Aws::SSO
         params: params,
         config: config)
       context[:gem_name] = 'aws-sdk-core'
-      context[:gem_version] = '3.125.5'
+      context[:gem_version] = '3.130.0'
       Seahorse::Client::Request.new(handlers, context)
     end
 

data/lib/aws-sdk-sso.rb

@@ -50,6 +50,6 @@ require_relative 'aws-sdk-sso/customizations'
 # @!group service
 module Aws::SSO
 
-  GEM_VERSION = '3.125.5'
+  GEM_VERSION = '3.130.0'
 
 end