aws-sdk-core 3.125.6 → 3.188.0

data/lib/aws-sdk-core/plugins/user_agent.rb

@@ -4,7 +4,31 @@ module Aws
   module Plugins
     # @api private
     class UserAgent < Seahorse::Client::Plugin
+      # @api private
       option(:user_agent_suffix)
+      # @api private
+      option(:user_agent_frameworks, default: [])
+
+      option(
+        :sdk_ua_app_id,
+        doc_type: 'String',
+        docstring: <<-DOCS) do |cfg|
+A unique and opaque application ID that is appended to the
+User-Agent header as app/<sdk_ua_app_id>. It should have a
+maximum length of 50.
+        DOCS
+        app_id = ENV['AWS_SDK_UA_APP_ID']
+        app_id ||= Aws.shared_config.sdk_ua_app_id(profile: cfg.profile)
+        app_id
+      end
+
+      def self.feature(feature, &block)
+        Thread.current[:aws_sdk_core_user_agent_feature] ||= []
+        Thread.current[:aws_sdk_core_user_agent_feature] << "ft/#{feature}"
+        block.call
+      ensure
+        Thread.current[:aws_sdk_core_user_agent_feature].pop
+      end
 
       # @api private
       class Handler < Seahorse::Client::Handler
@@ -14,33 +38,112 @@ module Aws
         end
 
         def set_user_agent(context)
-          ua = "aws-sdk-ruby3/#{CORE_GEM_VERSION}"
+          context.http_request.headers['User-Agent'] = UserAgent.new(context).to_s
+        end
+
+        class UserAgent
+          def initialize(context)
+            @context = context
+          end
+
+          def to_s
+            ua = "aws-sdk-ruby3/#{CORE_GEM_VERSION}"
+            ua += ' ua/2.0'
+            ua += " #{api_metadata}" if api_metadata
+            ua += " #{os_metadata}"
+            ua += " #{language_metadata}"
+            ua += " #{env_metadata}" if env_metadata
+            ua += " #{config_metadata}" if config_metadata
+            ua += " #{app_id}" if app_id
+            ua += " #{feature_metadata}" if feature_metadata
+            ua += " #{framework_metadata}" if framework_metadata
+            if @context.config.user_agent_suffix
+              ua += " #{@context.config.user_agent_suffix}"
+            end
+            ua.strip
+          end
+
+          private
 
-          begin
-            ua += " #{RUBY_ENGINE}/#{RUBY_VERSION}"
-          rescue
-            ua += " RUBY_ENGINE_NA/#{RUBY_VERSION}"
+          # Used to be gem_name/gem_version
+          def api_metadata
+            service_id = @context.config.api.metadata['serviceId']
+            return unless service_id
+
+            service_id = service_id.gsub(' ', '_').downcase
+            gem_version = @context[:gem_version]
+            "api/#{service_id}##{gem_version}"
+          end
+
+          # Used to be RUBY_PLATFORM
+          def os_metadata
+            os =
+              case RbConfig::CONFIG['host_os']
+              when /mac|darwin/
+                'macos'
+              when /linux|cygwin/
+                'linux'
+              when /mingw|mswin/
+                'windows'
+              else
+                'other'
+              end
+            metadata = "os/#{os}"
+            local_version = Gem::Platform.local.version
+            metadata += "##{local_version}" if local_version
+            metadata += " md/#{RbConfig::CONFIG['host_cpu']}"
+            metadata
           end
 
-          ua += " #{RUBY_PLATFORM}"
+          # Used to be RUBY_ENGINE/RUBY_VERSION
+          def language_metadata
+            "lang/#{RUBY_ENGINE}##{RUBY_ENGINE_VERSION} md/#{RUBY_VERSION}"
+          end
+
+          def env_metadata
+            return unless (execution_env = ENV['AWS_EXECUTION_ENV'])
+
+            "exec-env/#{execution_env}"
+          end
 
-          if context[:gem_name] && context[:gem_version]
-            ua += " #{context[:gem_name]}/#{context[:gem_version]}"
+          def config_metadata
+            "cfg/retry-mode##{@context.config.retry_mode}"
           end
 
-          if (execution_env = ENV['AWS_EXECUTION_ENV'])
-            ua += " exec-env/#{execution_env}"
+          def app_id
+            return unless (app_id = @context.config.sdk_ua_app_id)
+
+            # Sanitize and only allow these characters
+            app_id = app_id.gsub(/[^!#$%&'*+\-.^_`|~0-9A-Za-z]/, '-')
+            "app/#{app_id}"
           end
 
-          if context.config.user_agent_suffix
-            ua += " #{context.config.user_agent_suffix}"
+          def feature_metadata
+            return unless Thread.current[:aws_sdk_core_user_agent_feature]
+
+            Thread.current[:aws_sdk_core_user_agent_feature].join(' ')
           end
 
-          context.http_request.headers['User-Agent'] = ua.strip
+          def framework_metadata
+            if (frameworks_cfg = @context.config.user_agent_frameworks).empty?
+              return
+            end
+
+            # Frameworks may be aws-record, aws-sdk-rails, etc.
+            regex = /gems\/(?<name>#{frameworks_cfg.join('|')})-(?<version>\d+\.\d+\.\d+)/.freeze
+            frameworks = {}
+            Kernel.caller.each do |line|
+              match = line.match(regex)
+              next unless match
+
+              frameworks[match[:name]] = match[:version]
+            end
+            frameworks.map { |n, v| "lib/#{n}##{v}" }.join(' ')
+          end
         end
       end
 
-      handler(Handler)
+      handler(Handler, priority: 1)
     end
   end
 end

data/lib/aws-sdk-core/process_credentials.rb

@@ -1,19 +1,16 @@
 # frozen_string_literal: true
 
 module Aws
-
   # A credential provider that executes a given process and attempts
-  # to read its stdout to recieve a JSON payload containing the credentials
-  #
-  # Automatically handles refreshing credentials if an Expiration time is
-  # provided in the credentials payload
-  #
-  #     credentials = Aws::ProcessCredentials.new('/usr/bin/credential_proc').credentials
+  # to read its stdout to recieve a JSON payload containing the credentials.
   #
+  #     credentials = Aws::ProcessCredentials.new('/usr/bin/credential_proc')
   #     ec2 = Aws::EC2::Client.new(credentials: credentials)
   #
-  # More documentation on process based credentials can be found here:
-  # https://docs.aws.amazon.com/cli/latest/topic/config-vars.html#sourcing-credentials-from-external-processes
+  # Automatically handles refreshing credentials if an Expiration time is
+  # provided in the credentials payload.
+  #
+  # @see https://docs.aws.amazon.com/cli/latest/topic/config-vars.html#sourcing-credentials-from-external-processes
   class ProcessCredentials
 
     include CredentialProvider
@@ -27,6 +24,7 @@ module Aws
     def initialize(process)
       @process = process
       @credentials = credentials_from_process(@process)
+      @async_refresh = false
 
       super
     end
@@ -73,9 +71,9 @@ module Aws
       @credentials = credentials_from_process(@process)
     end
 
-    def near_expiration?
+    def near_expiration?(expiration_length)
       # are we within 5 minutes of expiration?
-      @expiration && (Time.now.to_i + 5 * 60) > @expiration.to_i
+      @expiration && (Time.now.to_i + expiration_length) > @expiration.to_i
     end
   end
 end

data/lib/aws-sdk-core/refreshing_credentials.rb

@@ -17,45 +17,70 @@ module Aws
   # @api private
   module RefreshingCredentials
 
+    SYNC_EXPIRATION_LENGTH = 300 # 5 minutes
+    ASYNC_EXPIRATION_LENGTH = 600 # 10 minutes
+
+    CLIENT_EXCLUDE_OPTIONS = Set.new([:before_refresh]).freeze
+
     def initialize(options = {})
       @mutex = Mutex.new
+      @before_refresh = options.delete(:before_refresh) if Hash === options
+
+      @before_refresh.call(self) if @before_refresh
       refresh
     end
 
     # @return [Credentials]
     def credentials
-      refresh_if_near_expiration
+      refresh_if_near_expiration!
       @credentials
     end
 
-    # @return [Time,nil]
-    def expiration
-      refresh_if_near_expiration
-      @expiration
-    end
-
     # Refresh credentials.
     # @return [void]
     def refresh!
-      @mutex.synchronize { refresh }
+      @mutex.synchronize do
+        @before_refresh.call(self) if @before_refresh
+
+        refresh
+      end
     end
 
     private
 
-    # Refreshes instance metadata credentials if they are within
-    # 5 minutes of expiration.
-    def refresh_if_near_expiration
-      if near_expiration?
+    # Refreshes credentials asynchronously and synchronously.
+    # If we are near to expiration, block while getting new credentials.
+    # Otherwise, if we're approaching expiration, use the existing credentials
+    # but attempt a refresh in the background.
+    def refresh_if_near_expiration!
+      # Note: This check is an optimization. Rather than acquire the mutex on every #refresh_if_near_expiration
+      # call, we check before doing so, and then we check within the mutex to avoid a race condition.
+      # See issue: https://github.com/aws/aws-sdk-ruby/issues/2641 for more info.
+      if near_expiration?(SYNC_EXPIRATION_LENGTH)
         @mutex.synchronize do
-          refresh if near_expiration?
+          if near_expiration?(SYNC_EXPIRATION_LENGTH)
+            @before_refresh.call(self) if @before_refresh
+            refresh
+          end
+        end
+      elsif @async_refresh && near_expiration?(ASYNC_EXPIRATION_LENGTH)
+        unless @mutex.locked?
+          Thread.new do
+            @mutex.synchronize do
+              if near_expiration?(ASYNC_EXPIRATION_LENGTH)
+                @before_refresh.call(self) if @before_refresh
+                refresh
+              end
+            end
+          end
         end
       end
     end
 
-    def near_expiration?
+    def near_expiration?(expiration_length)
       if @expiration
-        # are we within 5 minutes of expiration?
-        (Time.now.to_i + 5 * 60) > @expiration.to_i
+        # Are we within expiration?
+        (Time.now.to_i + expiration_length) > @expiration.to_i
       else
         true
       end

data/lib/aws-sdk-core/refreshing_token.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'thread'
+
+module Aws
+
+  # Module/mixin used by token provider classes that can be refreshed. This
+  # provides basic refresh logic in a thread-safe manner. Classes mixing in
+  # this module are expected to implement a #refresh method that populates
+  # the following instance variable:
+  #
+  # * `@token` [Token] - {Aws::Token} object with the `expiration` and `token`
+  #       fields set.
+  #
+  # @api private
+  module RefreshingToken
+
+    def initialize(options = {})
+      @mutex = Mutex.new
+      @before_refresh = options.delete(:before_refresh) if Hash === options
+
+      @before_refresh.call(self) if @before_refresh
+      refresh
+    end
+
+    # @return [Token]
+    def token
+      refresh_if_near_expiration
+      @token
+    end
+
+    # @return [Time,nil]
+    def expiration
+      refresh_if_near_expiration
+      @expiration
+    end
+
+    # Refresh token.
+    # @return [void]
+    def refresh!
+      @mutex.synchronize do
+        @before_refresh.call(self) if @before_refresh
+        refresh
+      end
+    end
+
+    private
+
+    # Refreshes token if it is within
+    # 5 minutes of expiration.
+    def refresh_if_near_expiration
+      if near_expiration?
+        @mutex.synchronize do
+          if near_expiration?
+            @before_refresh.call(self) if @before_refresh
+            refresh
+          end
+        end
+      end
+    end
+
+    def near_expiration?
+      if @token && @token.expiration
+        # are we within 5 minutes of expiration?
+        (Time.now.to_i + 5 * 60) > @token.expiration.to_i
+      else
+        true
+      end
+    end
+  end
+end

data/lib/aws-sdk-core/rest/handler.rb

@@ -17,7 +17,7 @@ module Aws
 
       def apply_request_id(context)
         h = context.http_response.headers
-        context[:request_id] = h['x-amz-request-id'] || h['x-amzn-requestid']
+        context[:request_id] ||= h['x-amz-request-id'] || h['x-amzn-requestid']
       end
 
     end

data/lib/aws-sdk-core/rest/request/headers.rb

@@ -53,12 +53,8 @@ module Aws
           return if !value || value.empty?
           headers[ref.location_name] = value
             .compact
-            .map { |s| escape_header_list_string(s.to_s) }
-            .join(",")
-        end
-
-        def escape_header_list_string(s)
-          (s.include?('"') || s.include?(",")) ? "\"#{s.gsub('"', '\"')}\"" : s
+            .map { |s| Seahorse::Util.escape_header_list_string(s.to_s) }
+            .join(',')
         end
 
         def apply_header_map(headers, ref, values)

data/lib/aws-sdk-core/rest/request/querystring_builder.rb

@@ -4,9 +4,16 @@ module Aws
   module Rest
     module Request
       class QuerystringBuilder
-
         include Seahorse::Model::Shapes
 
+        SUPPORTED_TYPES = [
+          BooleanShape,
+          FloatShape,
+          IntegerShape,
+          StringShape,
+          TimestampShape
+        ].freeze
+
         # Provide shape references and param values:
         #
         #     [
@@ -33,29 +40,12 @@ module Aws
         def build_part(shape_ref, param_value)
           case shape_ref.shape
           # supported scalar types
-          when StringShape, BooleanShape, FloatShape, IntegerShape, StringShape
-            param_name = shape_ref.location_name
-            "#{param_name}=#{escape(param_value.to_s)}"
-          when TimestampShape
-            param_name = shape_ref.location_name
-            "#{param_name}=#{escape(timestamp(shape_ref, param_value))}"
+          when *SUPPORTED_TYPES
+            "#{shape_ref.location_name}=#{query_value(shape_ref, param_value)}"
           when MapShape
-            if StringShape === shape_ref.shape.value.shape
-              query_map_of_string(param_value)
-            elsif ListShape === shape_ref.shape.value.shape
-              query_map_of_string_list(param_value)
-            else
-              msg = "only map of string and string list supported"
-              raise NotImplementedError, msg
-            end
+            generate_query_map(shape_ref, param_value)
           when ListShape
-            if StringShape === shape_ref.shape.member.shape
-              list_of_strings(shape_ref.location_name, param_value)
-            else
-              msg = "Only list of strings supported, got "\
-                    "#{shape_ref.shape.member.shape.class.name}"
-              raise NotImplementedError, msg
-            end
+            generate_query_list(shape_ref, param_value)
           else
             raise NotImplementedError
           end
@@ -71,6 +61,37 @@ module Aws
           end
         end
 
+        def query_value(ref, value)
+          case ref.shape
+          when TimestampShape
+            escape(timestamp(ref, value))
+          when *SUPPORTED_TYPES
+            escape(value.to_s)
+          else
+            raise NotImplementedError
+          end
+        end
+
+        def generate_query_list(ref, values)
+          member_ref = ref.shape.member
+          values.map do |value|
+            value = query_value(member_ref, value)
+            "#{ref.location_name}=#{value}"
+          end
+        end
+
+        def generate_query_map(ref, value)
+          case ref.shape.value.shape
+          when StringShape
+            query_map_of_string(value)
+          when ListShape
+            query_map_of_string_list(value)
+          else
+            msg = 'Only map of string and string list supported'
+            raise NotImplementedError, msg
+          end
+        end
+
         def query_map_of_string(hash)
           list = []
           hash.each_pair do |key, value|
@@ -89,16 +110,9 @@ module Aws
           list
         end
 
-        def list_of_strings(name, values)
-          values.map do |value|
-            "#{name}=#{escape(value)}"
-          end
-        end
-
         def escape(string)
           Seahorse::Util.uri_escape(string)
         end
-
       end
     end
   end

data/lib/aws-sdk-core/shared_config.rb

@@ -3,7 +3,11 @@
 module Aws
   # @api private
   class SharedConfig
-    SSO_PROFILE_KEYS = %w[sso_start_url sso_region sso_account_id sso_role_name].freeze
+    SSO_CREDENTIAL_PROFILE_KEYS = %w[sso_account_id sso_role_name].freeze
+    SSO_PROFILE_KEYS = %w[sso_session sso_start_url sso_region sso_account_id sso_role_name].freeze
+    SSO_TOKEN_PROFILE_KEYS = %w[sso_session].freeze
+    SSO_SESSION_KEYS = %w[sso_region sso_start_url].freeze
+
 
     # @return [String]
     attr_reader :credentials_path
@@ -51,10 +55,12 @@ module Aws
       @config_enabled = options[:config_enabled]
       @credentials_path = options[:credentials_path] ||
                           determine_credentials_path
+      @credentials_path = File.expand_path(@credentials_path) if @credentials_path
       @parsed_credentials = {}
       load_credentials_file if loadable?(@credentials_path)
       if @config_enabled
         @config_path = options[:config_path] || determine_config_path
+        @config_path = File.expand_path(@config_path) if @config_path
         load_config_file if loadable?(@config_path)
       end
     end
@@ -100,7 +106,7 @@ module Aws
     #   or `nil` if no valid credentials were found.
     def credentials(opts = {})
       p = opts[:profile] || @profile_name
-      validate_profile_exists(p) if credentials_present?
+      validate_profile_exists(p)
       if (credentials = credentials_from_shared(p, opts))
         credentials
       elsif (credentials = credentials_from_config(p, opts))
@@ -149,6 +155,38 @@ module Aws
       credentials
     end
 
+    # Attempts to load from shared config or shared credentials file.
+    # Will always attempt first to load from the shared credentials
+    # file, if present.
+    def sso_token_from_config(opts = {})
+      p = opts[:profile] || @profile_name
+      token = sso_token_from_profile(@parsed_credentials, p)
+      if @parsed_config
+        token ||= sso_token_from_profile(@parsed_config, p)
+      end
+      token
+    end
+
+    # Source a custom configured endpoint from the shared configuration file
+    #
+    # @param [Hash] opts
+    # @option opts [String] :profile
+    # @option opts [String] :service_id
+    def configured_endpoint(opts = {})
+      # services section is only allowed in the shared config file (not credentials)
+      profile = opts[:profile] || @profile_name
+      service_id = opts[:service_id]&.gsub(" ", "_")&.downcase
+      if @parsed_config && (prof_config = @parsed_config[profile])
+        services_section_name = prof_config['services']
+        if (services_config = @parsed_config["services #{services_section_name}"]) &&
+          (service_config = services_config[service_id])
+          return service_config['endpoint_url'] if service_config['endpoint_url']
+        end
+        return prof_config['endpoint_url']
+      end
+      nil
+    end
+
     # Add an accessor method (similar to attr_reader) to return a configuration value
     # Uses the get_config_value below to control where
     # values are loaded from
@@ -167,6 +205,7 @@ module Aws
       :use_fips_endpoint,
       :ec2_metadata_service_endpoint,
       :ec2_metadata_service_endpoint_mode,
+      :ec2_metadata_v1_disabled,
       :max_attempts,
       :retry_mode,
       :adaptive_retry_wait_to_fill,
@@ -179,7 +218,11 @@ module Aws
       :s3_use_arn_region,
       :s3_us_east_1_regional_endpoint,
       :s3_disable_multiregion_access_points,
-      :defaults_mode
+      :defaults_mode,
+      :sdk_ua_app_id,
+      :disable_request_compression,
+      :request_min_compression_size_bytes,
+      :ignore_configured_endpoint_urls
     )
 
     private
@@ -195,11 +238,6 @@ module Aws
       value
     end
 
-    def credentials_present?
-      (@parsed_credentials && !@parsed_credentials.empty?) ||
-        (@parsed_config && !@parsed_config.empty?)
-    end
-
     def assume_role_from_profile(cfg, profile, opts, chain_config)
       if cfg && prof_cfg = cfg[profile]
         opts[:source_profile] ||= prof_cfg['source_profile']
@@ -319,13 +357,53 @@ module Aws
     def sso_credentials_from_profile(cfg, profile)
       if @parsed_config &&
          (prof_config = cfg[profile]) &&
-         !(prof_config.keys & SSO_PROFILE_KEYS).empty?
+         !(prof_config.keys & SSO_CREDENTIAL_PROFILE_KEYS).empty?
+
+        if sso_session_name = prof_config['sso_session']
+          sso_session = sso_session(cfg, profile, sso_session_name)
+
+          sso_region = sso_session['sso_region']
+          sso_start_url = sso_session['sso_start_url']
+
+          # validate sso_region and sso_start_url don't conflict if set on profile and session
+          if prof_config['sso_region'] &&  prof_config['sso_region'] != sso_region
+            raise ArgumentError,
+                  "sso-session #{sso_session_name}'s sso_region (#{sso_region}) " \
+                    "does not match the profile #{profile}'s sso_region (#{prof_config['sso_region']}'"
+          end
+          if prof_config['sso_start_url'] &&  prof_config['sso_start_url'] != sso_start_url
+            raise ArgumentError,
+                  "sso-session #{sso_session_name}'s sso_start_url (#{sso_start_url}) " \
+                    "does not match the profile #{profile}'s sso_start_url (#{prof_config['sso_start_url']}'"
+          end
+        else
+          sso_region = prof_config['sso_region']
+          sso_start_url = prof_config['sso_start_url']
+        end
 
         SSOCredentials.new(
-          sso_start_url: prof_config['sso_start_url'],
-          sso_region: prof_config['sso_region'],
           sso_account_id: prof_config['sso_account_id'],
-          sso_role_name: prof_config['sso_role_name']
+          sso_role_name: prof_config['sso_role_name'],
+          sso_session: prof_config['sso_session'],
+          sso_region: sso_region,
+          sso_start_url: sso_start_url
+          )
+      end
+    end
+
+    # If the required sso_ profile values are present, attempt to construct
+    # SSOTokenProvider
+    def sso_token_from_profile(cfg, profile)
+      if @parsed_config &&
+        (prof_config = cfg[profile]) &&
+        !(prof_config.keys & SSO_TOKEN_PROFILE_KEYS).empty?
+
+        sso_session_name = prof_config['sso_session']
+        sso_session = sso_session(cfg, profile, sso_session_name)
+
+        SSOTokenProvider.new(
+          sso_session: sso_session_name,
+          sso_region: sso_session['sso_region']
         )
       end
     end
@@ -379,5 +457,22 @@ module Aws
       ret ||= 'default'
       ret
     end
+
+    def sso_session(cfg, profile, sso_session_name)
+      # aws sso-configure may add quotes around sso session names with whitespace
+      sso_session = cfg["sso-session #{sso_session_name}"] || cfg["sso-session '#{sso_session_name}'"]
+
+      unless sso_session
+        raise ArgumentError,
+          "sso-session #{sso_session_name} must be defined in the config file. " \
+                    "Referenced by profile #{profile}"
+      end
+
+      unless sso_session['sso_region']
+        raise ArgumentError, "sso-session #{sso_session_name} missing required parameter: sso_region"
+      end
+
+      sso_session
+    end
   end
 end