aws-sdk-core 3.36.0 → 3.37.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: '083dd2b29c24dd42e393906e157ed16992abd9ce'
-  data.tar.gz: d7aec8220a6df968102325ffc645801d67e7183b
+  metadata.gz: 3a0ca80cd27bbc952ef138198462d128f8586206
+  data.tar.gz: f45b8a21bbefd877679e6f9eb61b5d20e53377a7
 SHA512:
-  metadata.gz: 85b1ce5dc4e283f2407e9718e298b82773505cdb13b53e9734e176c05e130e691d7329d89e8e51c9c81cbb2dd6e3c3a443ac5bd2bb31422424d5952ff2650dac
-  data.tar.gz: 5c9cdcfdf9029fd8c0177859097fde46d0c48f8bcfc03361ff5261be6cf9ca0bc6ce71ad4bf8f6ac69d4e5017d09ebc5a0e5bba9ca97c8d6b7d7c1441962ba01
+  metadata.gz: 417e4adb9bd6a423160f6ce7a2fed489d77f46bf685535588a38145832d07dee582f674c501f622207129496502fcaebf36cac2a76fd0a73244827c0f6e6ad86
+  data.tar.gz: f2299177e51157ec28c44d7a5eed130c3a3bb3b50ca1f717291907b34b05c221442cd786a7101ab83f5f9cb2db174f8e7dee0ba28a07d85dba6cef3711c63464

data/VERSION

@@ -1 +1 @@
-3.36.0
+3.37.0

data/lib/aws-sdk-core.rb

@@ -64,9 +64,14 @@ require_relative 'aws-sdk-core/xml'
 require_relative 'aws-sdk-core/json'
 
 # event stream
+
 require_relative 'aws-sdk-core/binary'
 require_relative 'aws-sdk-core/event_emitter'
 
+# endpoint discovery
+
+require_relative 'aws-sdk-core/endpoint_cache'
+
 # client metrics
 
 require_relative 'aws-sdk-core/client_side_monitoring/request_metrics'

data/lib/aws-sdk-core/endpoint_cache.rb

@@ -0,0 +1,188 @@
+module Aws
+  # @api private
+  # a LRU cache caching endpoints data
+  class EndpointCache
+
+    # default cache entries limit
+    MAX_ENTRIES = 1000
+
+    # default max threads pool size
+    MAX_THREADS = 10
+
+    def initialize(options = {})
+      @max_entries = options[:max_entries] || MAX_ENTRIES
+      @entries = {} # store endpoints
+      @max_threads = options[:max_threads] || MAX_THREADS
+      @pool = {} # store polling threads
+      @mutex = Mutex.new
+      @require_identifier = nil # whether endpoint operation support identifier
+    end
+
+    # @return [Integer] Max size limit of cache
+    attr_reader :max_entries
+
+    # @return [Integer] Max count of polling threads
+    attr_reader :max_threads
+
+    # return [Hash] Polling threads pool
+    attr_reader :pool
+
+    # @param [String] key
+    # @return [Endpoint]
+    def [](key)
+      @mutex.synchronize do
+        # fetching an existing endpoint delete it and then append it
+        endpoint = @entries[key]
+        if endpoint
+          @entries.delete(key)
+          @entries[key] = endpoint
+        end
+        endpoint
+      end
+    end
+
+    # @param [String] key
+    # @param [Hash] value
+    def []=(key, value)
+      @mutex.synchronize do
+        # delete the least recent used endpoint when cache is full
+        unless @entries.size < @max_entries
+          old_key, _ = @entries.shift
+          self.delete_polling_thread(old_key)
+        end
+        # delete old value if exists
+        @entries.delete(key)
+        @entries[key] = Endpoint.new(value.to_hash)
+      end
+    end
+
+    # checking whether an unexpired endpoint key exists in cache
+    # @param [String] key
+    # @return [Boolean]
+    def key?(key)
+      if @entries.key?(key) && (@entries[key].nil? || @entries[key].expired?)
+        self.delete(key)
+      end
+      @entries.key?(key)
+    end
+
+    # checking whether an polling thread exist for the key
+    # @param [String] key
+    # @return [Boolean]
+    def threads_key?(key)
+      @pool.key?(key)
+    end
+
+    # remove entry only
+    # @param [String] key
+    def delete(key)
+      @mutex.synchronize do
+        @entries.delete(key)
+      end
+    end
+
+    # kill the old polling thread and remove it from pool
+    # @param [String] key
+    def delete_polling_thread(key)
+      Thread.kill(@pool[key]) if self.threads_key?(key)
+      @pool.delete(key)
+    end
+
+    # update cache with requests (using service endpoint operation)
+    # to fetch endpoint list (with identifiers when available)
+    # @param [String] key
+    # @param [RequestContext] ctx
+    def update(key, ctx)
+      resp = _request_endpoint(ctx)
+      if resp && resp.endpoints
+        resp.endpoints.each { |e| self[key] = e }
+      end
+    end
+
+    # extract the key to be used in the cache from request context
+    # @param [RequestContext] ctx
+    # @return [String]
+    def extract_key(ctx)
+      parts = []
+      # fetching from cred provider directly gives warnings
+      parts << ctx.config.credentials.credentials.access_key_id
+      if _endpoint_operation_identifier(ctx)
+        parts << ctx.operation_name
+        ctx.operation.input.shape.members.inject(parts) do |p, (name, ref)|
+          p << ctx.params[name] if ref["endpointdiscoveryid"]
+          p
+        end
+      end
+      parts.join('_')
+    end
+
+    # update polling threads pool
+    # param [String] key
+    # param [Thread] thread
+    def update_polling_pool(key, thread)
+      unless @pool.size < @max_threads
+        _, thread = @pool.shift
+        Thread.kill(thread)
+      end
+      @pool[key] = thread
+    end
+
+    # kill all polling threads
+    def stop_polling!
+      @pool.each { |_, t| Thread.kill(t) }
+      @pool = {}
+    end
+
+    private
+
+    def _request_endpoint(ctx)
+      params = {}
+      if _endpoint_operation_identifier(ctx)
+        # build identifier params when available
+        params[:operation] = ctx.operation.name
+        ctx.operation.input.shape.members.inject(params) do |p, (name, ref)|
+          if ref["endpointdiscoveryid"]
+            p[:identifiers] ||= {}
+            p[:identifiers][ref.location_name] = ctx.params[name]
+          end
+          p
+        end
+      end
+
+      begin
+        endpoint_operation_name = ctx.config.api.endpoint_operation
+        ctx.client.send(endpoint_operation_name, params)
+      rescue Aws::Errors::ServiceError
+        nil 
+      end
+    end
+
+    def _endpoint_operation_identifier(ctx)
+      return @require_identifier unless @require_identifier.nil?
+      operation_name = ctx.config.api.endpoint_operation
+      operation = ctx.config.api.operation(operation_name)
+      @require_identifier = operation.input.shape.members.any?
+    end
+
+    class Endpoint
+    
+      # default endpoint cache time, 1 minute
+      CACHE_PERIOD = 1
+
+      def initialize(options)
+        @address = options.fetch(:address)
+        @cache_period = options[:cache_period_in_minutes] || CACHE_PERIOD
+        @created_time = Time.now
+      end
+
+      # [String] valid URI address (with path) 
+      attr_reader :address
+
+      def expired?
+        Time.now - @created_time > @cache_period * 60
+      end
+
+    end
+
+  end
+end

data/lib/aws-sdk-core/errors.rb

@@ -34,6 +34,16 @@ module Aws
       end
     end
 
+    # Rasied when endpoint discovery failed for operations
+    # that requires endpoints from endpoint discovery
+    class EndpointDiscoveryError < RuntimeError
+      def initialize(*args)
+        msg = 'Endpoint discovery failed for the operation or discovered endpoint is not working, '\
+          'request will keep failing until endpoint discovery succeeds or :endpoint option is provided.'
+        super(msg)
+      end
+    end
+
     # Raised when EventStream Parser failed to parse
     # a raw event message
     class EventStreamParserError < RuntimeError; end

data/lib/aws-sdk-core/plugins/endpoint_discovery.rb

@@ -0,0 +1,160 @@
+module Aws
+  module Plugins
+    # @api private
+    class EndpointDiscovery < Seahorse::Client::Plugin
+
+      option(:endpoint_discovery,
+        default: false,
+        doc_type: 'Boolean',
+        docstring: <<-DOCS) do |cfg|
+When set to `true`, endpoint discovery will be enabled for operations when available. Defaults to `false`.
+        DOCS
+        resolve_endpoint_discovery(cfg)
+      end
+
+      option(:endpoint_cache_max_entries,
+        default: 1000,
+        doc_type: Integer,
+        docstring: <<-DOCS
+Used for the maximum size limit of the LRU cache storing endpoints data
+for endpoint discovery enabled operations. Defaults to 1000.
+        DOCS
+      )
+
+      option(:endpoint_cache_max_threads,
+        default: 10,
+        doc_type: Integer,
+        docstring: <<-DOCS
+Used for the maximum threads in use for polling endpoints to be cached, defaults to 10.
+        DOCS
+      )
+
+      option(:endpoint_cache_poll_interval,
+        default: 60,
+        doc_type: Integer,
+        docstring: <<-DOCS
+When :endpoint_discovery and :active_endpoint_cache is enabled,
+Use this option to config the time interval in seconds for making
+requests fetching endpoints information. Defaults to 60 sec.
+        DOCS
+      )
+
+      option(:endpoint_cache) do |cfg|
+        Aws::EndpointCache.new(
+          max_entries: cfg.endpoint_cache_max_entries,
+          max_threads: cfg.endpoint_cache_max_threads
+        )
+      end
+
+      option(:active_endpoint_cache,
+        default: false,
+        doc_type: 'Boolean',
+        docstring: <<-DOCS
+When set to `true`, a thread polling for endpoints will be running in
+the background every 60 secs (default). Defaults to `false`.
+        DOCS
+      )
+
+      def add_handlers(handlers, config)
+        handlers.add(Handler, priority: 90) if config.regional_endpoint
+      end
+
+      class Handler < Seahorse::Client::Handler
+
+        def call(context)
+          if context.operation.endpoint_operation
+            context.http_request.headers['x-amz-api-version'] = context.config.api.version
+            _apply_endpoint_discovery_user_agent(context)
+          elsif discovery_cfg = context.operation.endpoint_discovery
+            endpoint = _discover_endpoint(
+              context,
+              Aws::Util.str_2_bool(discovery_cfg["required"])
+            )
+            context.http_request.endpoint = _valid_uri(endpoint.address) if endpoint
+            if endpoint || context.config.endpoint_discovery
+              _apply_endpoint_discovery_user_agent(context)
+            end
+          end
+          @handler.call(context)
+        end
+
+        private
+
+        def _valid_uri(address)
+          # returned address can be missing scheme
+          if address.start_with?('http')
+            URI.parse(address)
+          else
+            URI.parse("https://" + address)
+          end
+        end
+
+        def _apply_endpoint_discovery_user_agent(ctx)
+          if ctx.config.user_agent_suffix.nil?
+            ctx.config.user_agent_suffix = "endpoint-discovery"
+          elsif !ctx.config.user_agent_suffix.include? "endpoint-discovery"
+            ctx.config.user_agent_suffix += "endpoint-discovery"
+          end
+        end
+
+        def _discover_endpoint(ctx, required)
+          cache = ctx.config.endpoint_cache 
+          key = cache.extract_key(ctx)
+
+          if required
+            # required for the operation
+            unless cache.key?(key)
+              cache.update(key, ctx)
+            end
+            endpoint = cache[key]
+            # hard fail if endpoint is not discovered
+            raise Aws::Errors::EndpointDiscoveryError.new unless endpoint
+            endpoint
+          elsif ctx.config.endpoint_discovery
+            # not required for the operation
+            # but enabled
+            if cache.key?(key)
+              cache[key]
+            elsif ctx.config.active_endpoint_cache
+              # enabled active cache pull
+              interval = ctx.config.endpoint_cache_poll_interval
+              if key.include?('_')
+                # identifier related, kill the previous polling thread by key
+                # because endpoint req params might be changed
+                cache.delete_polling_thread(key)
+              end
+
+              # start a thread for polling endpoints when non-exist
+              unless cache.threads_key?(key)
+                thread = Thread.new do
+                  while !cache.key?(key) do
+                    cache.update(key, ctx)
+                    sleep(interval)
+                  end
+                end
+                cache.update_polling_pool(key, thread)
+              end
+
+              cache[key]
+            else
+              # disabled active cache pull
+              # attempt, buit fail soft
+              cache.update(key, ctx)
+              cache[key]
+            end
+          end
+        end
+
+      end
+
+      private
+
+      def self.resolve_endpoint_discovery(cfg)
+        env = ENV['AWS_ENABLE_ENDPOINT_DISCOVERY']
+        shared_cfg = Aws.shared_config.endpoint_discovery(profile: cfg.profile)
+        Aws::Util.str_2_bool(env) || Aws::Util.str_2_bool(shared_cfg)
+      end
+
+    end
+  end
+end

data/lib/aws-sdk-core/plugins/regional_endpoint.rb

@@ -26,6 +26,8 @@ a default `:region` is search for in the following locations:
         resolve_region(cfg)
       end
 
+      option(:regional_endpoint, false)
+
       option(:endpoint, doc_type: String, docstring: <<-DOCS) do |cfg|
 The client endpoint is normally constructed from the `:region`
 option. You should only configure an `:endpoint` when connecting

data/lib/aws-sdk-core/plugins/retry_errors.rb

@@ -120,12 +120,32 @@ A delay randomiser function used by the default backoff function. Some predefine
           (500..599).include?(@http_status_code)
         end
 
+        def endpoint_discovery?(context)
+          return false unless context.operation.endpoint_discovery
+
+          if @http_status_code == 421 ||
+            extract_name(@error) == 'InvalidEndpointException'
+            @error = Errors::EndpointDiscoveryError.new
+          end
+
+          # When endpoint discovery error occurs
+          # evict the endpoint from cache
+          if @error.is_a?(Errors::EndpointDiscoveryError)
+            key = context.config.endpoint_cache.extract_key(context)
+            context.config.endpoint_cache.delete(key)
+            true
+          else
+            false
+          end
+        end
+          
         def retryable?(context)
           (expired_credentials? and refreshable_credentials?(context)) or
             throttling_error? or
             checksum? or
             networking? or
-            server?
+            server? or
+            endpoint_discovery?(context)
         end
 
         private

data/lib/aws-sdk-core/shared_config.rb

@@ -136,6 +136,13 @@ module Aws
       end
     end
 
+    def endpoint_discovery(opts = {})
+      p = opts[:profile] || @profile_name
+      if @config_enabled && @parsed_config
+        @parsed_config.fetch(p, {})["endpoint_discovery_enabled"]
+      end
+    end
+
     def credentials_process(profile)
       validate_profile_exists(profile)
       @parsed_config[profile]['credential_process']

data/lib/aws-sdk-core/util.rb

@@ -52,6 +52,15 @@ module Aws
         end
       end
 
+      def str_2_bool(str)
+        case str.to_s
+        when "true" then true
+        when "false" then false
+        else
+          nil
+        end
+      end
+
     end
   end
 end

data/lib/aws-sdk-sts.rb

@@ -40,6 +40,6 @@ require_relative 'aws-sdk-sts/customizations'
 # @service
 module Aws::STS
 
-  GEM_VERSION = '3.36.0'
+  GEM_VERSION = '3.37.0'
 
 end

data/lib/aws-sdk-sts/client.rb

@@ -15,6 +15,7 @@ require 'aws-sdk-core/plugins/helpful_socket_errors.rb'
 require 'aws-sdk-core/plugins/retry_errors.rb'
 require 'aws-sdk-core/plugins/global_configuration.rb'
 require 'aws-sdk-core/plugins/regional_endpoint.rb'
+require 'aws-sdk-core/plugins/endpoint_discovery.rb'
 require 'aws-sdk-core/plugins/response_paging.rb'
 require 'aws-sdk-core/plugins/stub_responses.rb'
 require 'aws-sdk-core/plugins/idempotency_token.rb'
@@ -45,6 +46,7 @@ module Aws::STS
     add_plugin(Aws::Plugins::RetryErrors)
     add_plugin(Aws::Plugins::GlobalConfiguration)
     add_plugin(Aws::Plugins::RegionalEndpoint)
+    add_plugin(Aws::Plugins::EndpointDiscovery)
     add_plugin(Aws::Plugins::ResponsePaging)
     add_plugin(Aws::Plugins::StubResponses)
     add_plugin(Aws::Plugins::IdempotencyToken)
@@ -98,6 +100,10 @@ module Aws::STS
     #
     #   @option options [String] :access_key_id
     #
+    #   @option options [Boolean] :active_endpoint_cache (false)
+    #     When set to `true`, a thread polling for endpoints will be running in
+    #     the background every 60 secs (default). Defaults to `false`.
+    #
     #   @option options [Boolean] :client_side_monitoring (false)
     #     When `true`, client-side metrics will be collected for all API requests from
     #     this client.
@@ -123,6 +129,21 @@ module Aws::STS
     #     option. You should only configure an `:endpoint` when connecting
     #     to test endpoints. This should be avalid HTTP(S) URI.
     #
+    #   @option options [Integer] :endpoint_cache_max_entries (1000)
+    #     Used for the maximum size limit of the LRU cache storing endpoints data
+    #     for endpoint discovery enabled operations. Defaults to 1000.
+    #
+    #   @option options [Integer] :endpoint_cache_max_threads (10)
+    #     Used for the maximum threads in use for polling endpoints to be cached, defaults to 10.
+    #
+    #   @option options [Integer] :endpoint_cache_poll_interval (60)
+    #     When :endpoint_discovery and :active_endpoint_cache is enabled,
+    #     Use this option to config the time interval in seconds for making
+    #     requests fetching endpoints information. Defaults to 60 sec.
+    #
+    #   @option options [Boolean] :endpoint_discovery (false)
+    #     When set to `true`, endpoint discovery will be enabled for operations when available. Defaults to `false`.
+    #
     #   @option options [Aws::Log::Formatter] :log_formatter (Aws::Log::Formatter.default)
     #     The log formatter.
     #
@@ -1508,7 +1529,7 @@ module Aws::STS
         params: params,
         config: config)
       context[:gem_name] = 'aws-sdk-core'
-      context[:gem_version] = '3.36.0'
+      context[:gem_version] = '3.37.0'
       Seahorse::Client::Request.new(handlers, context)
     end
 

data/lib/seahorse/client/configuration.rb

@@ -197,7 +197,13 @@ module Seahorse
 
         def value_at(opt_name)
           value = @struct[opt_name]
-          value.is_a?(Defaults) ? resolve_defaults(opt_name, value) : value
+          if value.is_a?(Defaults)
+            # this config value is used by endpoint discovery
+            @struct[:regional_endpoint] = true if opt_name == :endpoint
+            resolve_defaults(opt_name, value)
+          else
+            value
+          end
         end
 
         def resolve_defaults(opt_name, defaults)

data/lib/seahorse/model/api.rb

@@ -6,6 +6,7 @@ module Seahorse
         @metadata = {}
         @operations = {}
         @authorizers = {}
+        @endpoint_operation = nil
       end
 
       # @return [String, nil]
@@ -14,6 +15,9 @@ module Seahorse
       # @return [Hash]
       attr_accessor :metadata
 
+      # @return [Symbol|nil]
+      attr_accessor :endpoint_operation
+
       def operations(&block)
         if block_given?
           @operations.each(&block)

data/lib/seahorse/model/operation.rb

@@ -22,6 +22,12 @@ module Seahorse
       # @return [Boolean]
       attr_accessor :deprecated
 
+      # @return [Boolean]
+      attr_accessor :endpoint_operation
+
+      # @return [Hash]
+      attr_accessor :endpoint_discovery
+
       # @return [String, nil]
       attr_accessor :documentation
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aws-sdk-core
 version: !ruby/object:Gem::Version
-  version: 3.36.0
+  version: 3.37.0
 platform: ruby
 authors:
 - Amazon Web Services
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-30 00:00:00.000000000 Z
+date: 2018-11-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jmespath
@@ -90,6 +90,7 @@ files:
 - lib/aws-sdk-core/deprecations.rb
 - lib/aws-sdk-core/eager_loader.rb
 - lib/aws-sdk-core/ecs_credentials.rb
+- lib/aws-sdk-core/endpoint_cache.rb
 - lib/aws-sdk-core/errors.rb
 - lib/aws-sdk-core/event_emitter.rb
 - lib/aws-sdk-core/ini_parser.rb
@@ -116,6 +117,7 @@ files:
 - lib/aws-sdk-core/plugins/client_metrics_plugin.rb
 - lib/aws-sdk-core/plugins/client_metrics_send_plugin.rb
 - lib/aws-sdk-core/plugins/credentials_configuration.rb
+- lib/aws-sdk-core/plugins/endpoint_discovery.rb
 - lib/aws-sdk-core/plugins/event_stream_configuration.rb
 - lib/aws-sdk-core/plugins/global_configuration.rb
 - lib/aws-sdk-core/plugins/helpful_socket_errors.rb