aws-sdk-core 3.31.0 → 3.168.4

data/lib/aws-sdk-core/ec2_metadata.rb

@@ -0,0 +1,238 @@
+# frozen_string_literal: true
+
+require 'time'
+require 'net/http'
+
+module Aws
+  # A client that can query version 2 of the EC2 Instance Metadata
+  class EC2Metadata
+    # Path for PUT request for token
+    # @api private
+    METADATA_TOKEN_PATH = '/latest/api/token'.freeze
+
+    # Raised when the PUT request is not valid. This would be thrown if
+    # `token_ttl` is not an Integer.
+    # @api private
+    class TokenRetrievalError < RuntimeError; end
+
+    # Token has expired, and the request can be retried with a new token.
+    # @api private
+    class TokenExpiredError < RuntimeError; end
+
+    # The requested metadata path does not exist.
+    # @api private
+    class MetadataNotFoundError < RuntimeError; end
+
+    # The request is not allowed or IMDS is turned off.
+    # @api private
+    class RequestForbiddenError < RuntimeError; end
+
+    # Creates a client that can query version 2 of the EC2 Instance Metadata
+    #   service (IMDS).
+    #
+    # @note Customers using containers may need to increase their hop limit
+    #   to access IMDSv2.
+    # @see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html#instance-metadata-transition-to-version-2
+    #
+    # @param [Hash] options
+    # @option options [Integer] :token_ttl (21600) The session token's TTL,
+    #   defaulting to 6 hours.
+    # @option options [Integer] :retries (3) The number of retries for failed
+    #   requests.
+    # @option options [String] :endpoint ('http://169.254.169.254') The IMDS
+    #   endpoint. This option has precedence over the :endpoint_mode.
+    # @option options [String] :endpoint_mode ('IPv4') The endpoint mode for
+    #   the instance metadata service. This is either 'IPv4'
+    #   ('http://169.254.169.254') or 'IPv6' ('http://[fd00:ec2::254]').
+    # @option options [Integer] :port (80) The IMDS endpoint port.
+    # @option options [Integer] :http_open_timeout (1) The number of seconds to
+    #   wait for the connection to open.
+    # @option options [Integer] :http_read_timeout (1) The number of seconds for
+    #   one chunk of data to be read.
+    # @option options [IO] :http_debug_output An output stream for debugging. Do
+    #   not use this in production.
+    # @option options [Integer,Proc] :backoff A backoff used for retryable
+    #   requests. When given an Integer, it sleeps that amount. When given a
+    #   Proc, it is called with the current number of failed retries.
+    def initialize(options = {})
+      @token_ttl = options[:token_ttl] || 21_600
+      @retries = options[:retries] || 3
+      @backoff = backoff(options[:backoff])
+
+      endpoint_mode = options[:endpoint_mode] || 'IPv4'
+      @endpoint = resolve_endpoint(options[:endpoint], endpoint_mode)
+      @port = options[:port] || 80
+
+      @http_open_timeout = options[:http_open_timeout] || 1
+      @http_read_timeout = options[:http_read_timeout] || 1
+      @http_debug_output = options[:http_debug_output]
+
+      @token = nil
+      @mutex = Mutex.new
+    end
+
+    # Fetches a given metadata category using a String path, and returns the
+    #   result as a String. A path starts with the API version (usually
+    #   "/latest/"). See the instance data categories for possible paths.
+    #
+    # @example Fetching the instance ID
+    #
+    #   ec2_metadata = Aws::EC2Metadata.new
+    #   ec2_metadata.get('/latest/meta-data/instance-id')
+    #   => "i-023a25f10a73a0f79"
+    #
+    # @note This implementation always returns a String and will not parse any
+    #   responses. Parsable responses may include JSON objects or directory
+    #   listings, which are strings separated by line feeds (ASCII 10).
+    #
+    # @example Fetching and parsing JSON meta-data
+    #
+    #   require 'json'
+    #   data = ec2_metadata.get('/latest/dynamic/instance-identity/document')
+    #   JSON.parse(data)
+    #   => {"accountId"=>"012345678912", ... }
+    #
+    # @example Fetching and parsing directory listings
+    #
+    #   listing = ec2_metadata.get('/latest/meta-data')
+    #   listing.split(10.chr)
+    #   => ["ami-id", "ami-launch-index", ...]
+    #
+    # @note Unlike other services, IMDS does not have a service API model. This
+    #   means that we cannot confidently generate code with methods and
+    #   response structures. This implementation ensures that new IMDS features
+    #   are always supported by being deployed to the instance and does not
+    #   require code changes.
+    #
+    # @see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-categories.html
+    # @see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-identity-documents.html
+    # @param [String] path The full path to the metadata.
+    def get(path)
+      retry_errors(max_retries: @retries) do
+        @mutex.synchronize do
+          fetch_token unless @token && !@token.expired?
+        end
+
+        open_connection do |conn|
+          http_get(conn, path, @token.value)
+        end
+      end
+    end
+
+    private
+
+    def resolve_endpoint(endpoint, endpoint_mode)
+      return endpoint if endpoint
+
+      case endpoint_mode.downcase
+      when 'ipv4' then 'http://169.254.169.254'
+      when 'ipv6' then 'http://[fd00:ec2::254]'
+      else
+        raise ArgumentError,
+              ':endpoint_mode is not valid, expected IPv4 or IPv6, '\
+              "got: #{endpoint_mode}"
+      end
+    end
+
+    def fetch_token
+      open_connection do |conn|
+        created_time = Time.now
+        token_value, token_ttl = http_put(conn, @token_ttl)
+        @token = Token.new(value: token_value, ttl: token_ttl, created_time: created_time)
+      end
+    end
+
+    def http_get(connection, path, token)
+      headers = {
+        'User-Agent' => "aws-sdk-ruby3/#{CORE_GEM_VERSION}",
+        'x-aws-ec2-metadata-token' => token
+      }
+      request = Net::HTTP::Get.new(path, headers)
+      response = connection.request(request)
+
+      case response.code.to_i
+      when 200
+        response.body
+      when 401
+        raise TokenExpiredError
+      when 404
+        raise MetadataNotFoundError
+      end
+    end
+
+    def http_put(connection, ttl)
+      headers = {
+        'User-Agent' => "aws-sdk-ruby3/#{CORE_GEM_VERSION}",
+        'x-aws-ec2-metadata-token-ttl-seconds' => ttl.to_s
+      }
+      request = Net::HTTP::Put.new(METADATA_TOKEN_PATH, headers)
+      response = connection.request(request)
+
+      case response.code.to_i
+      when 200
+        [
+          response.body,
+          response.header['x-aws-ec2-metadata-token-ttl-seconds'].to_i
+        ]
+      when 400
+        raise TokenRetrievalError
+      when 403
+        raise RequestForbiddenError
+      end
+    end
+
+    def open_connection
+      uri = URI.parse(@endpoint)
+      http = Net::HTTP.new(uri.hostname || @endpoint, @port || uri.port)
+      http.open_timeout = @http_open_timeout
+      http.read_timeout = @http_read_timeout
+      http.set_debug_output(@http_debug_output) if @http_debug_output
+      http.start
+      yield(http).tap { http.finish }
+    end
+
+    def retry_errors(options = {}, &_block)
+      max_retries = options[:max_retries]
+      retries = 0
+      begin
+        yield
+      # These errors should not be retried.
+      rescue TokenRetrievalError, MetadataNotFoundError, RequestForbiddenError
+        raise
+      # StandardError is not ideal but it covers Net::HTTP errors.
+      # https://gist.github.com/tenderlove/245188
+      rescue StandardError, TokenExpiredError
+        raise unless retries < max_retries
+
+        @backoff.call(retries)
+        retries += 1
+        retry
+      end
+    end
+
+    def backoff(backoff)
+      case backoff
+      when Proc then backoff
+      when Numeric then ->(_) { Kernel.sleep(backoff) }
+      else ->(num_failures) { Kernel.sleep(1.2**num_failures) }
+      end
+    end
+
+    # @api private
+    class Token
+      def initialize(options = {})
+        @ttl   = options[:ttl]
+        @value = options[:value]
+        @created_time = options[:created_time] || Time.now
+      end
+
+      # [String] Returns the token value.
+      attr_reader :value
+
+      # [Boolean] Returns true if the token expired.
+      def expired?
+        Time.now - @created_time > @ttl
+      end
+    end
+  end
+end

data/lib/aws-sdk-core/ecs_credentials.rb

@@ -1,8 +1,14 @@
-require 'json'
+# frozen_string_literal: true
+
 require 'time'
 require 'net/http'
 
 module Aws
+  # An auto-refreshing credential provider that loads credentials from
+  # instances running in ECS.
+  #
+  #     ecs_credentials = Aws::ECSCredentials.new(retries: 3)
+  #     ec2 = Aws::EC2::Client.new(credentials: ecs_credentials)
   class ECSCredentials
 
     include CredentialProvider
@@ -42,6 +48,10 @@ module Aws
     # @option options [IO] :http_debug_output (nil) HTTP wire
     #   traces are sent to this object.  You can specify something
     #   like $stdout.
+    # @option options [Callable] before_refresh Proc called before
+    #   credentials are refreshed. `before_refresh` is called
+    #   with an instance of this object when
+    #   AWS credentials are required and need to be refreshed.
     def initialize options = {}
       @retries = options[:retries] || 5
       @ip_address = options[:ip_address] || '169.254.170.2'
@@ -57,6 +67,7 @@ module Aws
       @http_read_timeout = options[:http_read_timeout] || 5
       @http_debug_output = options[:http_debug_output]
       @backoff = backoff(options[:backoff])
+      @async_refresh = false
       super
     end
 
@@ -78,14 +89,18 @@ module Aws
       # Retry loading credentials up to 3 times is the instance metadata
       # service is responding but is returning invalid JSON documents
       # in response to the GET profile credentials call.
-      retry_errors([JSON::ParserError, StandardError], max_retries: 3) do
-        c = JSON.parse(get_credentials.to_s)
-        @credentials = Credentials.new(
-          c['AccessKeyId'],
-          c['SecretAccessKey'],
-          c['Token']
-        )
-        @expiration = c['Expiration'] ? Time.iso8601(c['Expiration']) : nil
+      begin
+        retry_errors([Aws::Json::ParseError, StandardError], max_retries: 3) do
+          c = Aws::Json.load(get_credentials.to_s)
+          @credentials = Credentials.new(
+            c['AccessKeyId'],
+            c['SecretAccessKey'],
+            c['Token']
+          )
+          @expiration = c['Expiration'] ? Time.iso8601(c['Expiration']) : nil
+        end
+      rescue Aws::Json::ParseError
+        raise Aws::Errors::MetadataParserError.new
       end
     end
 

data/lib/aws-sdk-core/endpoint_cache.rb

@@ -0,0 +1,193 @@
+# frozen_string_literal: true
+
+module Aws
+  # @api private
+  # a LRU cache caching endpoints data
+  class EndpointCache
+
+    # default cache entries limit
+    MAX_ENTRIES = 1000
+
+    # default max threads pool size
+    MAX_THREADS = 10
+
+    def initialize(options = {})
+      @max_entries = options[:max_entries] || MAX_ENTRIES
+      @entries = {} # store endpoints
+      @max_threads = options[:max_threads] || MAX_THREADS
+      @pool = {} # store polling threads
+      @mutex = Mutex.new
+      @require_identifier = nil # whether endpoint operation support identifier
+    end
+
+    # @return [Integer] Max size limit of cache
+    attr_reader :max_entries
+
+    # @return [Integer] Max count of polling threads
+    attr_reader :max_threads
+
+    # return [Hash] Polling threads pool
+    attr_reader :pool
+
+    # @param [String] key
+    # @return [Endpoint]
+    def [](key)
+      @mutex.synchronize do
+        # fetching an existing endpoint delete it and then append it
+        endpoint = @entries[key]
+        if endpoint
+          @entries.delete(key)
+          @entries[key] = endpoint
+        end
+        endpoint
+      end
+    end
+
+    # @param [String] key
+    # @param [Hash] value
+    def []=(key, value)
+      @mutex.synchronize do
+        # delete the least recent used endpoint when cache is full
+        unless @entries.size < @max_entries
+          old_key, = @entries.shift
+          delete_polling_thread(old_key)
+        end
+        # delete old value if exists
+        @entries.delete(key)
+        @entries[key] = Endpoint.new(value.to_hash)
+      end
+    end
+
+    # checking whether an unexpired endpoint key exists in cache
+    # @param [String] key
+    # @return [Boolean]
+    def key?(key)
+      @mutex.synchronize do
+        if @entries.key?(key) && (@entries[key].nil? || @entries[key].expired?)
+          @entries.delete(key)
+        end
+        @entries.key?(key)
+      end
+    end
+
+    # checking whether an polling thread exist for the key
+    # @param [String] key
+    # @return [Boolean]
+    def threads_key?(key)
+      @pool.key?(key)
+    end
+
+    # remove entry only
+    # @param [String] key
+    def delete(key)
+      @mutex.synchronize do
+        @entries.delete(key)
+      end
+    end
+
+    # kill the old polling thread and remove it from pool
+    # @param [String] key
+    def delete_polling_thread(key)
+      Thread.kill(@pool[key]) if threads_key?(key)
+      @pool.delete(key)
+    end
+
+    # update cache with requests (using service endpoint operation)
+    # to fetch endpoint list (with identifiers when available)
+    # @param [String] key
+    # @param [RequestContext] ctx
+    def update(key, ctx)
+      resp = _request_endpoint(ctx)
+      if resp && resp.endpoints
+        resp.endpoints.each { |e| self[key] = e }
+      end
+    end
+
+    # extract the key to be used in the cache from request context
+    # @param [RequestContext] ctx
+    # @return [String]
+    def extract_key(ctx)
+      parts = []
+      # fetching from cred provider directly gives warnings
+      parts << ctx.config.credentials.credentials.access_key_id
+      if _endpoint_operation_identifier(ctx)
+        parts << ctx.operation_name
+        ctx.operation.input.shape.members.inject(parts) do |p, (name, ref)|
+          p << ctx.params[name] if ref['endpointdiscoveryid']
+          p
+        end
+      end
+      parts.join('_')
+    end
+
+    # update polling threads pool
+    # param [String] key
+    # param [Thread] thread
+    def update_polling_pool(key, thread)
+      unless @pool.size < @max_threads
+        _, thread = @pool.shift
+        Thread.kill(thread)
+      end
+      @pool[key] = thread
+    end
+
+    # kill all polling threads
+    def stop_polling!
+      @pool.each { |_, t| Thread.kill(t) }
+      @pool = {}
+    end
+
+    private
+
+    def _request_endpoint(ctx)
+      params = {}
+      if _endpoint_operation_identifier(ctx)
+        # build identifier params when available
+        params[:operation] = ctx.operation.name
+        ctx.operation.input.shape.members.inject(params) do |p, (name, ref)|
+          if ref['endpointdiscoveryid']
+            p[:identifiers] ||= {}
+            p[:identifiers][ref.location_name] = ctx.params[name]
+          end
+          p
+        end
+      end
+
+      begin
+        endpoint_operation_name = ctx.config.api.endpoint_operation
+        ctx.client.send(endpoint_operation_name, params)
+      rescue Aws::Errors::ServiceError
+        nil
+      end
+    end
+
+    def _endpoint_operation_identifier(ctx)
+      return @require_identifier unless @require_identifier.nil?
+
+      operation_name = ctx.config.api.endpoint_operation
+      operation = ctx.config.api.operation(operation_name)
+      @require_identifier = operation.input.shape.members.any?
+    end
+
+    class Endpoint
+
+      # default endpoint cache time, 1 minute
+      CACHE_PERIOD = 1
+
+      def initialize(options)
+        @address = options.fetch(:address)
+        @cache_period = options[:cache_period_in_minutes] || CACHE_PERIOD
+        @created_time = Time.now
+      end
+
+      # [String] valid URI address (with path)
+      attr_reader :address
+
+      def expired?
+        Time.now - @created_time > @cache_period * 60
+      end
+
+    end
+
+  end
+end

data/lib/aws-sdk-core/endpoints/condition.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Aws
+  module Endpoints
+    # @api private
+    class Condition
+      def initialize(fn:, argv:, assign: nil)
+        @fn = Function.new(fn: fn, argv: argv)
+        @assign = assign
+        @assigned = {}
+      end
+
+      attr_reader :fn
+      attr_reader :argv
+      attr_reader :assign
+
+      attr_reader :assigned
+
+      def match?(parameters, assigns)
+        output = @fn.call(parameters, assigns)
+        @assigned = @assigned.merge({ @assign => output }) if @assign
+        output
+      end
+
+      def self.from_json(conditions_json)
+        conditions_json.each.with_object([]) do |condition, conditions|
+          conditions << new(
+            fn: condition['fn'],
+            argv: condition['argv'],
+            assign: condition['assign']
+          )
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-core/endpoints/endpoint.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Aws
+  module Endpoints
+    class Endpoint
+      def initialize(url:, properties: {}, headers: {})
+        @url = url
+        @properties = properties
+        @headers = headers
+      end
+
+      attr_reader :url
+      attr_reader :properties
+      attr_reader :headers
+    end
+  end
+end

data/lib/aws-sdk-core/endpoints/endpoint_rule.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Aws
+  module Endpoints
+    # @api private
+    class EndpointRule < Rule
+      def initialize(type: 'endpoint', conditions:, endpoint:,
+                     documentation: nil)
+        @type = type
+        @conditions = Condition.from_json(conditions)
+        @endpoint = endpoint
+        @documentation = documentation
+      end
+
+      attr_reader :type
+      attr_reader :conditions
+      attr_reader :endpoint
+      attr_reader :documentation
+
+      def match(parameters, assigned = {})
+        assigns = assigned.dup
+        matched = conditions.all? do |condition|
+          output = condition.match?(parameters, assigns)
+          assigns = assigns.merge(condition.assigned) if condition.assign
+          output
+        end
+        resolved_endpoint(parameters, assigns) if matched
+      end
+
+      def resolved_endpoint(parameters, assigns)
+        Endpoint.new(
+          url: resolve_value(@endpoint['url'], parameters, assigns),
+          properties: resolve_properties(
+            @endpoint['properties'] || {},
+            parameters,
+            assigns
+          ),
+          headers: resolve_headers(parameters, assigns)
+        )
+      end
+
+      private
+
+      def resolve_headers(parameters, assigns)
+        (@endpoint['headers'] || {}).each.with_object({}) do |(key, arr), headers|
+          headers[key] = []
+          arr.each do |value|
+            headers[key] << resolve_value(value, parameters, assigns)
+          end
+        end
+      end
+
+      def resolve_properties(obj, parameters, assigns)
+        case obj
+        when Hash
+          obj.each.with_object({}) do |(key, value), hash|
+            hash[key] = resolve_properties(value, parameters, assigns)
+          end
+        when Array
+          obj.collect { |value| resolve_properties(value, parameters, assigns) }
+        else
+          if obj.is_a?(String)
+            Templater.resolve(obj, parameters, assigns)
+          else
+            obj
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-core/endpoints/error_rule.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Aws
+  module Endpoints
+    # @api private
+    class ErrorRule < Rule
+      def initialize(type: 'error', conditions:, error: nil, documentation: nil)
+        @type = type
+        @conditions = Condition.from_json(conditions)
+        @error = error
+        @documentation = documentation
+      end
+
+      attr_reader :type
+      attr_reader :conditions
+      attr_reader :error
+      attr_reader :documentation
+
+      def match(parameters, assigned = {})
+        assigns = assigned.dup
+        matched = conditions.all? do |condition|
+          output = condition.match?(parameters, assigns)
+          assigns = assigns.merge(condition.assigned) if condition.assign
+          output
+        end
+        resolved_error(parameters, assigns) if matched
+      end
+
+      private
+
+      def resolved_error(parameters, assigns)
+        error = resolve_value(@error, parameters, assigns)
+        ArgumentError.new(error)
+      end
+    end
+  end
+end