smithy-client 1.0.0.pre0 → 1.0.0.pre1

data/lib/smithy-client/log_formatter.rb

@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+require 'pathname'
+
+module Smithy
+  module Client
+    # A log formatter generates a string for logging from output. This is
+    # accomplished with a log pattern string:
+    #
+    #     pattern = ':operation :http_response_status_code :time'
+    #     formatter = Smithy::Client::LogFormatter.new(pattern)
+    #     formatter.format(response)
+    #     #=> 'list_cities 200 0.0352'
+    #
+    # # Canned Formatters
+    #
+    # Instead of providing your own pattern, you can choose a canned log
+    # formatter.
+    #
+    # * {Formatter.default}
+    # * {Formatter.colored}
+    # * {Formatter.short}
+    #
+    # # Pattern Substitutions
+    #
+    # You can put any of these placeholders into your pattern.
+    #
+    #   * `:client_class` - The name of the client class.
+    #   * `:operation` - The name of the client request method.
+    #   * `:request_params` - The user provided parameters. Long strings are truncated/summarized if
+    #      they exceed the `:max_string_size`. Other objects are inspected.
+    #   * `:time` - The total time in seconds spent on the request.
+    #      This includes client side time spent building the request and parsing the response.
+    #   * `:retries` - The number of times a client request was retried.
+    #   * `:http_request_method` - The http request verb, e.g., `POST`, `PUT`, `GET`, etc.
+    #   * `:http_request_endpoint` - The request endpoint. This includes the scheme, host and port, but not the path.
+    #   * `:http_request_scheme` - This is replaced by `http` or `https`.
+    #   * `:http_request_host` - The host name of the http request endpoint (e.g. 'example.com').
+    #   * `:http_request_port` - The port number (e.g. '443' or '80').
+    #   * `:http_request_headers` - The http request headers, inspected.
+    #   * `:http_request_body` - The http request payload.
+    #   * `:http_response_status_code` - The http response status code, e.g., `200`, `404`, `500`, etc.
+    #   * `:http_response_headers` - The http response headers, inspected.
+    #   * `:http_response_body` - The http response body contents.
+    #   * `:error_class` - The class of the error that occurred, if any.
+    #   * `:error_message` - The error message, if an error occurred.
+    #
+    class LogFormatter
+      # @param [String] pattern The log format pattern should be a string and may contain substitutions.
+      # @option options [Integer] :max_string_size (1000) When summarizing
+      #  request parameters, strings longer than this value will be truncated.
+      # @option options [Boolean] :filter_sensitive_params (true) When false, sensitive params will
+      #  not be filtered when logging `:params`.
+      def initialize(pattern, options = {})
+        @pattern = pattern
+        @log_param_formatter = LogParamFormatter.new(options)
+        @log_param_filter = LogParamFilter.new(options)
+      end
+
+      # @return [String]
+      attr_reader :pattern
+
+      # Given a response, this will format a log message and return it as a string according to {#pattern}.
+      # @param [Smithy::Client::Response] response
+      # @return [String]
+      def format(response)
+        pattern.gsub(/:(\w+)/) { |sym| send("_#{sym[1..]}", response) }
+      end
+
+      private
+
+      def _client_class(response)
+        response.context.client.class.name
+      end
+
+      def _operation(response)
+        response.context.operation_name
+      end
+
+      def _request_params(response)
+        params = response.context.params
+        input = response.context.operation.input
+        @log_param_formatter.summarize(@log_param_filter.filter(input, params))
+      end
+
+      def _time(response)
+        duration = response.context[:logging_completed_at] - response.context[:logging_started_at]
+        Kernel.format('%.06f', duration).sub(/0+$/, '')
+      end
+
+      def _retries(response)
+        response.context.retries
+      end
+
+      def _http_request_endpoint(response)
+        response.context.http_request.endpoint.to_s
+      end
+
+      def _http_request_scheme(response)
+        response.context.http_request.endpoint.scheme
+      end
+
+      def _http_request_host(response)
+        response.context.http_request.endpoint.host
+      end
+
+      def _http_request_port(response)
+        response.context.http_request.endpoint.port
+      end
+
+      def _http_request_method(response)
+        response.context.http_request.http_method
+      end
+
+      def _http_request_headers(response)
+        response.context.http_request.headers.inspect
+      end
+
+      def _http_request_body(response)
+        body = response.context.http_request.body
+        return '' unless body.respond_to?(:rewind)
+
+        body_contents = body.read
+        body.rewind
+        @log_param_formatter.summarize(body_contents)
+      end
+
+      def _http_response_status_code(response)
+        response.context.http_response.status_code
+      end
+
+      def _http_response_headers(response)
+        response.context.http_response.headers.inspect
+      end
+
+      def _http_response_body(response)
+        body = response.context.http_response.body
+        return '' unless body.respond_to?(:rewind)
+
+        body_contents = body.read
+        body.rewind
+        @log_param_formatter.summarize(body_contents)
+      end
+
+      def _error_class(response)
+        response.error ? response.error.class.name : ''
+      end
+
+      def _error_message(response)
+        response.error ? response.error.message : ''
+      end
+
+      class << self
+        # The default log format.
+        # @option (see #initialize)
+        # @example A sample of the default format.
+        #
+        #     [ClientClass 200 0.580066 0 retries] list_objects(:bucket_name => 'bucket')
+        #
+        # @return [Formatter]
+        def default(options = {})
+          pattern = []
+          pattern << '[:client_class'
+          pattern << ':http_response_status_code'
+          pattern << ':time'
+          pattern << ':retries retries]'
+          pattern << ':operation(:request_params)'
+          pattern << ':error_class'
+          pattern << ':error_message'
+          LogFormatter.new("#{pattern.join(' ')}\n", options)
+        end
+
+        # The short log format.  Similar to default, but it does not
+        # inspect the request params or report on retries.
+        # @option (see #initialize)
+        # @example A sample of the short format
+        #
+        #     [ClientClass 200 0.494532] list_buckets
+        #
+        # @return [Formatter]
+        def short(options = {})
+          pattern = []
+          pattern << '[:client_class'
+          pattern << ':http_response_status_code'
+          pattern << ':time]'
+          pattern << ':operation'
+          pattern << ':error_class'
+          LogFormatter.new("#{pattern.join(' ')}\n", options)
+        end
+
+        # The default log format with ANSI colors.
+        # @option (see #initialize)
+        # @example A sample of the colored format (sans the ansi colors).
+        #
+        #     [ClientClass 200 0.580066 0 retries] list_objects(:bucket_name => 'bucket')
+        #
+        # @return [Formatter]
+        def colored(options = {})
+          bold = "\x1b[1m"
+          color = "\x1b[34m"
+          reset = "\x1b[0m"
+          pattern = []
+          pattern << "#{bold}#{color}[:client_class"
+          pattern << ':http_response_status_code'
+          pattern << ':time'
+          pattern << ":retries retries]#{reset}#{bold}"
+          pattern << ':operation(:request_params)'
+          pattern << ':error_class'
+          pattern << ":error_message#{reset}"
+          LogFormatter.new("#{pattern.join(' ')}\n", options)
+        end
+      end
+    end
+  end
+end

data/lib/smithy-client/log_param_filter.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Smithy
+  module Client
+    # @api private
+    class LogParamFilter
+      include Schema::Shapes
+
+      def initialize(options = {})
+        @filter_sensitive_params = options.fetch(:filter_sensitive_params, true)
+      end
+
+      def filter(ref, values)
+        case ref.shape
+        when ListShape then list(ref, values)
+        when MapShape then map(ref, values)
+        when StructureShape then structure(ref, values)
+        when UnionShape then union(ref, values)
+        else scalar(ref, values)
+        end
+      end
+
+      private
+
+      def list(ref, values)
+        shape = ref.shape
+        return '[FILTERED]' if sensitive?(shape)
+
+        member_ref = shape.member
+        values.collect { |value| filter(member_ref, value) }
+      end
+
+      def map(ref, values)
+        shape = ref.shape
+        return '[FILTERED]' if sensitive?(shape)
+
+        filtered = {}
+        value_ref = shape.value
+        values.each_pair do |key, value|
+          filtered[key] = filter(value_ref, value)
+        end
+        filtered
+      end
+
+      def scalar(ref, value)
+        return '[FILTERED]' if sensitive?(ref.shape)
+
+        value
+      end
+
+      def structure(ref, values)
+        shape = ref.shape
+        return '[FILTERED]' if sensitive?(shape)
+
+        filtered = {}
+        values.each_pair do |key, value|
+          next unless shape.member?(key)
+
+          member_ref = shape.member(key)
+          filtered[key] = filter(member_ref, value)
+        end
+        filtered
+      end
+
+      def union(ref, values) # rubocop:disable Metrics/AbcSize
+        shape = ref.shape
+        return '[FILTERED]' if sensitive?(shape)
+
+        filtered = {}
+        if values.is_a?(Schema::Union)
+          name, member_ref = ref.shape.member_by_type(values.class)
+          filtered[name] = filter(member_ref, values.value)
+        else
+          key, value = values.first
+          if ref.shape.member?(key)
+            member_ref = ref.shape.member(key)
+            filtered[key] = filter(member_ref, value)
+          end
+        end
+        filtered
+      end
+
+      def sensitive?(shape)
+        @filter_sensitive_params && shape.traits.key?('smithy.api#sensitive')
+      end
+    end
+  end
+end

data/lib/smithy-client/log_param_formatter.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require 'pathname'
+
+module Smithy
+  module Client
+    # @api private
+    class LogParamFormatter
+      # String longer than the max string size are truncated
+      MAX_STRING_SIZE = 1000
+
+      def initialize(options = {})
+        @max_string_size = options[:max_string_size] || MAX_STRING_SIZE
+      end
+
+      def summarize(value)
+        case value
+        when Array then "[#{array(value)}]"
+        when File then file(value)
+        when Hash then "{ #{hash(value)} }"
+        when Pathname then pathname(value)
+        when String then string(value)
+        when Tempfile then tempfile(value)
+        else value.inspect
+        end
+      end
+
+      private
+
+      def string(str)
+        if str.size > @max_string_size
+          "#<String #{str[0...@max_string_size].inspect} ... (#{str.size} bytes)>"
+        else
+          str.inspect
+        end
+      end
+
+      def hash(hash)
+        hash.map do |key, value|
+          if key.is_a?(String)
+            "#{key.inspect} => #{summarize(value)}"
+          else
+            "#{key}: #{summarize(value)}"
+          end
+        end.join(', ')
+      end
+
+      def array(array)
+        array.map { |v| summarize(v) }.join(', ')
+      end
+
+      def file(file)
+        "#<File:#{file.path} (#{file.size} bytes)>"
+      end
+
+      def tempfile(file)
+        "#<Tempfile:#{file.path} (#{file.size} bytes)>"
+      end
+
+      def pathname(path)
+        "#<Pathname:#{path} (#{File.size(path)} bytes)>"
+      end
+    end
+  end
+end

data/lib/smithy-client/managed_file.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Smithy
+  module Client
+    # This utility class is used to track files opened by Smithy::Client.
+    # @api private
+    class ManagedFile < File
+      # @return [Boolean]
+      def open?
+        !closed?
+      end
+    end
+  end
+end

data/lib/smithy-client/net_http/connection_pool.rb

@@ -0,0 +1,297 @@
+# frozen_string_literal: true
+
+require 'cgi'
+require 'net/http'
+require 'net/https'
+require 'delegate'
+require 'logger'
+
+require_relative 'patches'
+
+Smithy::Client::NetHTTP::Patches.apply!
+
+module Smithy
+  module Client
+    module NetHTTP
+      # @api private
+      class ConnectionPool
+        @pools_mutex = Mutex.new
+        @pools = {}
+
+        class << self
+          # Returns a connection pool constructed from the given options.
+          # Calling this method twice with the same options will return
+          # the same pool.
+          #
+          # @option options [Numeric] :http_continue_timeout (nil)
+          #  @see https://docs.ruby-lang.org/en/master/Net/HTTP.html#attribute-i-continue_timeout
+          # @option options [Numeric] :http_keep_alive_timeout (nil)
+          #  @see https://docs.ruby-lang.org/en/master/Net/HTTP.html#attribute-i-keep_alive_timeout
+          # @option options [Numeric] :http_open_timeout (nil)
+          #  @see https://docs.ruby-lang.org/en/master/Net/HTTP.html#attribute-i-open_timeout
+          # @option options [Numeric] :http_read_timeout (nil)
+          #  @see https://docs.ruby-lang.org/en/master/Net/HTTP.html#attribute-i-read_timeout
+          # @option options [Numeric] :http_ssl_timeout (nil)
+          #  @see https://docs.ruby-lang.org/en/master/Net/HTTP.html#attribute-i-ssl_timeout
+          # @option options [Numeric] :http_write_timeout (nil)
+          #  @see https://docs.ruby-lang.org/en/master/Net/HTTP.html#attribute-i-write_timeout
+          # @option options [String] :http_ca_file (nil)
+          #  @see https://docs.ruby-lang.org/en/master/Net/HTTP.html#attribute-i-ca_file
+          # @option options [String] :http_ca_path (nil)
+          #  @see https://docs.ruby-lang.org/en/master/Net/HTTP.html#attribute-i-ca_path
+          # @option options [OpenSSL::X509::Certificate] :http_cert (nil)
+          #  @see https://docs.ruby-lang.org/en/master/Net/HTTP.html#attribute-i-cert
+          # @option options [OpenSSL::X509::Store] :http_cert_store (nil)
+          #  @see https://docs.ruby-lang.org/en/master/Net/HTTP.html#attribute-i-cert_store
+          # @option options [OpenSSL::PKey::RSA, OpenSSL::PKey::DSA] :http_key (nil)
+          #  @see https://docs.ruby-lang.org/en/master/Net/HTTP.html#attribute-i-key
+          # @option options [Integer] :http_verify_mode (OpenSSL::SSL::VERIFY_PEER)
+          #  @see https://docs.ruby-lang.org/en/master/Net/HTTP.html#attribute-i-verify_mode
+          # @option options [Boolean] :http_debug_output (false)
+          #  @see https://docs.ruby-lang.org/en/master/Net/HTTP.html#method-i-set_debug_output
+          # @option options [URI::HTTP, String] :http_proxy (nil)
+          #  @see https://docs.ruby-lang.org/en/master/Net/HTTP.html#attribute-i-proxy
+          # @option options [Logger] :logger (nil) Where debug output is sent.
+          #    Defaults to `Logger.new($stdout)` when `:http_wire_trace` is `true`.
+          # @return [ConnectionPool]
+          def for(options = {})
+            options = pool_options(options)
+            @pools_mutex.synchronize do
+              @pools[options] ||= new(options)
+            end
+          end
+
+          # @return [Array<ConnectionPool>] Returns a list of the
+          #  constructed connection pools.
+          def pools
+            @pools_mutex.synchronize do
+              @pools.values
+            end
+          end
+
+          private
+
+          # Filters an option hash, merging in default values.
+          # @return [Hash]
+          def pool_options(options)
+            options.delete(:logger) unless options[:http_debug_output]
+            options[:http_proxy] = URI.parse(options[:http_proxy].to_s)
+            options
+          end
+        end
+
+        OPTIONS = {
+          # Connections
+          http_continue_timeout: nil,
+          http_keep_alive_timeout: nil,
+          http_open_timeout: nil,
+          http_read_timeout: nil,
+          http_ssl_timeout: nil,
+          http_write_timeout: nil,
+          # Security
+          http_ca_file: nil,
+          http_ca_path: nil,
+          http_cert: nil,
+          http_cert_store: nil,
+          http_key: nil,
+          http_verify_mode: OpenSSL::SSL::VERIFY_PEER,
+          # Debugging
+          http_debug_output: nil,
+          # Proxies
+          http_proxy: nil,
+          # Other
+          logger: Logger.new($stdout)
+        }.freeze
+
+        # @api private
+        def initialize(options = {})
+          OPTIONS.each_pair do |opt_name, default_value|
+            value = options[opt_name].nil? ? default_value : options[opt_name]
+            instance_variable_set("@#{opt_name}", value)
+          end
+          @pool_mutex = Mutex.new
+          @pool = {}
+        end
+        private_class_method :new
+
+        OPTIONS.each_key do |attr_name|
+          attr_reader(attr_name)
+        end
+
+        # @param [URI::HTTP, URI::HTTPS] endpoint The HTTP(S) endpoint
+        #    to connect to (e.g. 'https://domain.com').
+        #
+        # @yieldparam [Net::HTTPSession] session
+        #
+        # @return [nil]
+        def session_for(endpoint)
+          endpoint = remove_path_and_query(endpoint)
+          session = nil
+
+          # attempt to recycle an already open session
+          @pool_mutex.synchronize do
+            _clean
+            session = @pool[endpoint].shift if @pool.key?(endpoint)
+          end
+
+          begin
+            session ||= start_session(endpoint)
+            yield(session)
+          rescue StandardError
+            session&.finish
+            raise
+          else
+            @pool_mutex.synchronize do
+              @pool[endpoint] = [] unless @pool.key?(endpoint)
+              @pool[endpoint] << session
+            end
+          end
+          nil
+        end
+
+        # @return [Integer] Returns the count of sessions currently in the
+        #  pool, not counting those currently in use.
+        def size
+          @pool_mutex.synchronize do
+            @pool.values.flatten.size
+          end
+        end
+
+        # Removes stale http sessions from the pool (that have exceeded the idle timeout).
+        # @return [nil]
+        def clean!
+          @pool_mutex.synchronize { _clean }
+          nil
+        end
+
+        # Closes and removes all sessions from the pool. If empty! is called while
+        # there are outstanding requests they may get checked back into the pool,
+        # leaving the pool in a non-empty state.
+        # @return [nil]
+        def empty!
+          @pool_mutex.synchronize do
+            @pool.values.flatten.map(&:finish)
+            @pool.clear
+          end
+          nil
+        end
+
+        private
+
+        def remove_path_and_query(endpoint)
+          endpoint.dup.tap do |e|
+            e.path = ''
+            e.query = nil
+          end.to_s
+        end
+
+        # Extract the parts of the http_proxy URI
+        # @return [Array<String>]
+        def http_proxy_parts
+          [
+            http_proxy.host,
+            http_proxy.port,
+            http_proxy.user && CGI.unescape(http_proxy.user),
+            http_proxy.password && CGI.unescape(http_proxy.password)
+          ]
+        end
+
+        # Starts and returns a new HTTP(S) session.
+        # @param [String] endpoint
+        # @return [Net::HTTPSession]
+        def start_session(endpoint)
+          endpoint = URI.parse(endpoint)
+
+          args = []
+          args << endpoint.host
+          args << endpoint.port
+          args += http_proxy_parts
+
+          http = ExtendedSession.new(Net::HTTP.new(*args.compact))
+          # Prefer SDK retries and do not retry at the http layer
+          http.max_retries = 0
+          configure_session(http, endpoint)
+          http.set_debug_output(logger) if http_debug_output
+
+          http.start
+          http
+        end
+
+        def configure_session(http, endpoint)
+          configure_http_connections(http)
+
+          if endpoint.scheme == 'https'
+            configure_ssl(http)
+          else
+            http.use_ssl = false
+          end
+        end
+
+        def configure_http_connections(http)
+          http.continue_timeout = http_continue_timeout if http_continue_timeout
+          http.keep_alive_timeout = http_keep_alive_timeout if http_keep_alive_timeout
+          http.open_timeout = http_open_timeout if http_open_timeout
+          http.read_timeout = http_read_timeout if http_read_timeout
+          http.write_timeout = http_write_timeout  if http_write_timeout
+        end
+
+        def configure_ssl(http)
+          http.use_ssl = true
+          http.ssl_timeout = http_ssl_timeout if http_ssl_timeout
+          return unless http_verify_mode == OpenSSL::SSL::VERIFY_PEER
+
+          configure_ssl_cert(http)
+        end
+
+        def configure_ssl_cert(http)
+          http.ca_file = http_ca_file if http_ca_file
+          http.ca_path = http_ca_path if http_ca_path
+          http.cert = http_cert if http_cert
+          http.cert_store = http_cert_store if http_cert_store
+          http.key = http_key if http_key
+        end
+
+        # Removes stale sessions from the pool. This method *must* be called.
+        # @note **Must** be called behind a `@pool_mutex` synchronize block.
+        def _clean
+          now = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
+          @pool.each_value do |sessions|
+            sessions.delete_if do |session|
+              if session.last_used.nil? ||
+                 (now - session.last_used > session.keep_alive_timeout * 1000)
+                session.finish
+                true
+              end
+            end
+          end
+        end
+
+        # Helper methods extended onto Net::HTTPSession objects opened by the
+        # connection pool.
+        # @api private
+        class ExtendedSession < SimpleDelegator
+          def initialize(http)
+            super
+            @http = http
+          end
+
+          # @return [Integer, nil]
+          attr_reader :last_used
+
+          # Sends the request and tracks that this session has been used.
+          def request(...)
+            @http.request(...)
+            @last_used = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
+          end
+
+          # Attempts to close/finish the session without raising an error.
+          def finish
+            @http.finish
+          rescue IOError
+            nil
+          end
+        end
+      end
+    end
+  end
+end