hearth 1.0.0.pre1 → 1.0.0.pre3

data/lib/hearth/http/client.rb

@@ -1,67 +1,111 @@
 # frozen_string_literal: true
 
+require 'delegate'
 require 'net/http'
 require 'logger'
 require 'openssl'
 
 module Hearth
   module HTTP
-    # Transmits an HTTP {Request} object, returning an HTTP {Response}.
-    # @api private
+    # An HTTP client that uses Net::HTTP to send requests.
     class Client
+      # @api private
+      OPTIONS = {
+        logger: nil,
+        debug_output: nil,
+        proxy: nil,
+        open_timeout: 15,
+        read_timeout: nil,
+        keep_alive_timeout: 5,
+        continue_timeout: 1,
+        write_timeout: nil,
+        ssl_timeout: nil,
+        verify_peer: true,
+        ca_file: nil,
+        ca_path: nil,
+        cert_store: nil,
+        host_resolver: nil
+      }.freeze
+
       # Initialize an instance of this HTTP client.
       #
       # @param [Hash] options The options for this HTTP Client
       #
-      # @option options [Boolean] :http_wire_trace (false) When `true`,
-      #   HTTP debug output will be sent to the `:logger`.
+      # @option options [Logger] :logger (nil) A logger used to log Net::HTTP
+      #   requests and responses when `:debug_output` is enabled.
       #
-      # @option options [Logger] :logger A logger where debug output is sent.
+      # @option options [Boolean] :debug_output (false) When `true`,
+      #   sets an output stream to the configured Logger (if any) for debugging.
       #
-      # @option options [URI::HTTP,String] :http_proxy A proxy to send
+      # @option options [String, URI] :proxy A proxy to send
       #   requests through. Formatted like 'http://proxy.com:123'.
       #
-      # @option options [Boolean] :ssl_verify_peer (true) When `true`,
+      # @option options [Float] :open_timeout (15) Number of seconds to
+      #   wait for the connection to open.
+      #
+      # @option options [Float] :read_timeout Number of seconds to wait
+      #   for one block to be read (via one read(2) call).
+      #
+      # @option options [Float] :keep_alive_timeout (5) Seconds to reuse the
+      #   connection of the previous request.
+      #
+      # @option options [Float] :continue_timeout (1) Seconds to wait for
+      #   100 Continue response.
+      #
+      # @option options [Float] :write_timeout Number of seconds to wait
+      #   for one block to be written (via one write(2) call).
+      #
+      # @option options [Float] :ssl_timeout Sets the SSL timeout seconds.
+      #
+      # @option options [Boolean] :verify_peer (true) When `true`,
       #   SSL peer certificates are verified when establishing a
       #   connection.
       #
-      # @option options [String] :ssl_ca_bundle Full path to the SSL
+      # @option options [String] :ca_file Full path to the SSL
       #   certificate authority bundle file that should be used when
       #   verifying peer certificates.  If you do not pass
-      #   `:ssl_ca_bundle` or `:ssl_ca_directory` the system default
+      #   `:ca_file` or `:ca_path` the system default
       #   will be used if available.
       #
-      # @option options [String] :ssl_ca_directory Full path of the
+      # @option options [String] :ca_path Full path of the
       #   directory that contains the unbundled SSL certificate
       #   authority files for verifying peer certificates.  If you do
-      #   not pass `:ssl_ca_bundle` or `:ssl_ca_directory` the
+      #   not pass `:ca_file` or `:ca_path` the
       #   system default will be used if available.
+      #
+      # @option options [OpenSSL::X509::Store] :cert_store An OpenSSL X509
+      #   certificate store that contains the SSL certificate authority.
+      #
+      # @option options [#resolve_address] :host_resolver
+      #   An object, such as {Hearth::DNS::HostResolver} that responds to
+      #   `#resolve_address`, returning an array of up to two IP addresses for
+      #   the given hostname, one IPv6 and one IPv4, in that order.
+      #   `#resolve_address` should take a nodename keyword argument and
+      #   optionally other keyword args similar to Addrinfo.getaddrinfo's
+      #   positional parameters.
       def initialize(options = {})
-        @http_wire_trace = options[:http_wire_trace]
-        @logger = options[:logger]
-        @http_proxy = options[:http_proxy]
-        @http_proxy = URI.parse(@http_proxy.to_s) if @http_proxy
-        @ssl_verify_peer = options[:ssl_verify_peer]
-        @ssl_ca_bundle = options[:ssl_ca_bundle]
-        @ssl_ca_directory = options[:ssl_ca_directory]
-        @ssl_ca_store = options[:ssl_ca_store]
+        unknown = options.keys - OPTIONS.keys
+        raise ArgumentError, "Unknown options: #{unknown}" unless unknown.empty?
+
+        OPTIONS.each_pair do |opt_name, default_value|
+          value = options.key?(opt_name) ? options[opt_name] : default_value
+          instance_variable_set("@#{opt_name}", value)
+        end
+      end
+
+      OPTIONS.each_key do |attr_name|
+        attr_reader(attr_name)
       end
 
       # @param [Request] request
       # @param [Response] response
+      # @param [Logger] logger (nil)
       # @return [Response]
-      def transmit(request:, response:)
-        uri = URI.parse(request.url)
-        http = create_http(uri)
-        http.set_debug_output(@logger) if @http_wire_trace
-
-        if uri.scheme == 'https'
-          configure_ssl(http)
-        else
-          http.use_ssl = false
+      def transmit(request:, response:, logger: nil)
+        net_request = build_net_request(request)
+        with_connection_pool(request.uri, logger) do |connection|
+          _transmit(connection, net_request, response)
         end
-
-        _transmit(http, request, response)
         response.body.rewind if response.body.respond_to?(:rewind)
         response
       rescue ArgumentError => e
@@ -73,37 +117,82 @@ module Hearth
 
       private
 
-      def _transmit(http, request, response)
-        http.start do |conn|
-          conn.request(build_net_request(request)) do |net_resp|
-            response.status = net_resp.code.to_i
-            response.headers = extract_headers(net_resp)
-            net_resp.read_body do |chunk|
-              response.body.write(chunk)
-            end
-          end
+      def with_connection_pool(endpoint, logger)
+        pool = ConnectionPool.for(pool_config)
+        connection = pool.connection_for(endpoint) do
+          new_connection(endpoint, logger)
+        end
+        yield connection
+        pool.offer(endpoint, connection)
+      rescue StandardError => e
+        connection&.finish
+        raise e
+      end
+
+      # Starts and returns a new HTTP connection.
+      # @return [Net::HTTP]
+      def new_connection(endpoint, logger)
+        http = create_http(endpoint)
+        http.set_debug_output(@logger || logger) if @debug_output
+        configure_timeouts(http)
+
+        if endpoint.scheme == 'https'
+          configure_ssl(http)
+        else
+          http.use_ssl = false
+        end
+
+        http.start
+        http
+      end
+
+      def _transmit(http, net_request, response)
+        # Inform monkey patch to use our DNS resolver
+        Thread.current[:net_http_hearth_dns_resolver] = @host_resolver
+        http.request(net_request) do |net_resp|
+          unpack_response(net_resp, response)
         end
+      ensure
+        # Restore the default DNS resolver
+        Thread.current[:net_http_hearth_dns_resolver] = nil
       end
 
-      # Creates an HTTP connection to the endpoint
-      # Applies proxy if set
+      def unpack_response(net_resp, response)
+        response.status = net_resp.code.to_i
+        net_resp.each_header { |k, v| response.headers[k] = v }
+        net_resp.read_body do |chunk|
+          response.body.write(chunk)
+        end
+      end
+
+      # Creates an HTTP connection to the endpoint.
+      # Applies proxy if set.
       def create_http(endpoint)
         args = []
         args << endpoint.host
         args << endpoint.port
-        args += http_proxy_parts if @http_proxy
+        args += proxy_parts if @proxy
         # Net::HTTP.new uses positional arguments: host, port, proxy_args....
-        Net::HTTP.new(*args.compact)
+        HTTP.new(Net::HTTP.new(*args.compact))
+      end
+
+      def configure_timeouts(http)
+        http.open_timeout = @open_timeout
+        http.keep_alive_timeout = @keep_alive_timeout
+        http.read_timeout = @read_timeout
+        http.continue_timeout = @continue_timeout
+        http.write_timeout = @write_timeout
       end
 
       # applies ssl settings to the HTTP object
       def configure_ssl(http)
         http.use_ssl = true
-        if @ssl_verify_peer
+        http.ssl_timeout = @ssl_timeout
+        if @verify_peer
           http.verify_mode = OpenSSL::SSL::VERIFY_PEER
-          http.ca_file = @ssl_ca_bundle if @ssl_ca_bundle
-          http.ca_path = @ssl_ca_directory if @ssl_ca_directory
-          http.cert_store = @ssl_ca_store if @ssl_ca_store
+          http.ca_file = @ca_file
+          http.ca_path = @ca_path
+          http.cert_store = @cert_store
         else
           http.verify_mode = OpenSSL::SSL::VERIFY_NONE
         end
@@ -115,15 +204,28 @@ module Hearth
       # @return [Net::HTTP::Request]
       def build_net_request(request)
         request_class = net_http_request_class(request)
-        req = request_class.new(request.url, request.headers.to_h)
-        req.body_stream = request.body
+        req = request_class.new(request.uri, net_headers_for(request))
+
+        # Net::HTTP adds a default Content-Type when a body is present.
+        # Set the body stream when it has an unknown size or when it is > 0.
+        # We instead add our own Content-Length header via Middleware.
+        if !request.body.respond_to?(:size) ||
+           (request.body.respond_to?(:size) && request.body.size.positive?)
+          req.body_stream = request.body
+        end
         req
       end
 
-      # @param [Net::HTTP::Response] response
+      # Validate that fields are not trailers and return a hash of headers.
+      # @param [HTTP::Request] request
       # @return [Hash<String, String>]
-      def extract_headers(response)
-        response.to_hash.transform_values(&:first)
+      def net_headers_for(request)
+        # Trailers are not supported in Net::HTTP
+        if request.trailers.any?
+          raise NotImplementedError, 'Trailers are not supported in Net::HTTP'
+        end
+
+        request.headers.to_h
       end
 
       # @param [Http::Request] request
@@ -137,16 +239,61 @@ module Hearth
         raise ArgumentError, msg
       end
 
-      # Extract the parts of the http_proxy URI
-      # @return [Array(String)]
-      def http_proxy_parts
+      # Extract the parts of the proxy URI
+      # @return [Array]
+      def proxy_parts
+        proxy = URI(@proxy)
         [
-          @http_proxy.host,
-          @http_proxy.port,
-          (@http_proxy.user && CGI.unescape(@http_proxy.user)),
-          (@http_proxy.password && CGI.unescape(@http_proxy.password))
+          proxy.host,
+          proxy.port,
+          proxy.user && CGI.unescape(proxy.user),
+          proxy.password && CGI.unescape(proxy.password)
         ]
       end
+
+      # Config options for the HTTP client used for connection pooling
+      # @return [Hash]
+      def pool_config
+        OPTIONS.each_key.with_object({}) do |option_name, hash|
+          hash[option_name] = instance_variable_get("@#{option_name}")
+        end
+      end
+
+      # Helper methods extended onto Net::HTTP objects.
+      # @api private
+      class HTTP < SimpleDelegator
+        def initialize(http)
+          super(http)
+          @http = http
+        end
+
+        # @return [Integer, nil]
+        attr_reader :last_used
+
+        # Sends the request and tracks that this connection has been used.
+        def request(...)
+          @http.request(...)
+          @last_used = monotonic_milliseconds
+        end
+
+        def stale?
+          @last_used.nil? ||
+            (monotonic_milliseconds - @last_used) > keep_alive_timeout * 1000
+        end
+
+        # Attempts to close/finish the connection without raising an error.
+        def finish
+          @http.finish
+        rescue IOError
+          nil
+        end
+
+        private
+
+        def monotonic_milliseconds
+          Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
+        end
+      end
     end
   end
 end

data/lib/hearth/http/error_inspector.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require 'time'
+
+module Hearth
+  module HTTP
+    # An HTTP error inspector, using hints from status code and headers.
+    class ErrorInspector
+      def initialize(error, http_response)
+        @error = error
+        @http_response = http_response
+      end
+
+      def retryable?
+        (modeled_retryable? ||
+          throttling? ||
+          transient? ||
+          server?) &&
+          # IO does not respond to #truncate and is not rewindable
+          @http_response.body.respond_to?(:truncate)
+      end
+
+      def error_type
+        if transient?
+          'Transient'
+        elsif throttling?
+          'Throttling'
+        elsif server?
+          'ServerError'
+        elsif client?
+          'ClientError'
+        else
+          'Unknown'
+        end
+      end
+
+      def hints
+        hints = {}
+        if (retry_after = retry_after_hint)
+          hints[:retry_after] = retry_after
+        end
+        hints
+      end
+
+      private
+
+      def transient?
+        @error.is_a?(Hearth::HTTP::NetworkingError)
+      end
+
+      def throttling?
+        @http_response.status == 429 || modeled_throttling?
+      end
+
+      def server?
+        (500..599).cover?(@http_response.status)
+      end
+
+      def client?
+        (400..499).cover?(@http_response.status)
+      end
+
+      def modeled_retryable?
+        @error.is_a?(Hearth::ApiError) && @error.retryable?
+      end
+
+      def modeled_throttling?
+        modeled_retryable? && @error.throttling?
+      end
+
+      def retry_after_hint
+        retry_after = @http_response.headers['retry-after']
+        Integer(retry_after)
+      rescue ArgumentError # string is present, assume it is a date
+        begin
+          Time.parse(retry_after) - Time.now
+        rescue ArgumentError # empty string, somehow
+          nil
+        end
+      rescue TypeError # header is not present
+        nil
+      end
+    end
+  end
+end

data/lib/hearth/http/error_parser.rb

@@ -9,17 +9,18 @@ module Hearth
     # @api private
     class ErrorParser
       # @api private
-      HTTP_3XX = (300..399).freeze
+      HTTP_3XX = (300..399)
 
       # @api private
-      HTTP_4XX = (400..499).freeze
+      HTTP_4XX = (400..499)
 
       # @api private
-      HTTP_5XX = (500..599).freeze
+      HTTP_5XX = (500..599)
 
       # @param [Module] error_module The code generated Errors module.
       #   Must contain service specific implementations of
-      #   ApiRedirectError, ApiClientError, and ApiServerError
+      #   ApiRedirectError, ApiClientError, and ApiServerError, and it
+      #   must have defined a `self.error_code(http_resp)` method.
       #
       # @param [Integer] success_status The status code of a
       #   successful response as defined by the model for
@@ -30,22 +31,21 @@ module Hearth
       #
       # @param [Array<Class<ApiError>>] errors Array of Error classes
       #   modeled for the operation.
-      #
-      # @param [callable] error_code_fn Protocol specific function
-      #   that will return the error code from a response, or nil if
-      #   there is none.
-      def initialize(error_module:, success_status:, errors:, error_code_fn:)
+      def initialize(error_module:, success_status:, errors:)
         @error_module = error_module
         @success_status = success_status
         @errors = errors
-        @error_code_fn = error_code_fn
       end
 
       # Parse and return the error if the response is not successful.
       #
-      # @param [Response] response The HTTP response
-      def parse(response)
-        extract_error(response) if error?(response)
+      # @param [HTTP::Response] http_resp The HTTP Response
+      # @param [Hash] metadata The metadata from {Hearth::Output}
+      def parse(http_resp, metadata)
+        error_code = @error_module.method(:error_code).call(http_resp)
+        return unless error?(error_code, http_resp)
+
+        create_error(error_code, http_resp, metadata)
       end
 
       private
@@ -59,20 +59,20 @@ module Hearth
       # 6. Response code 5xx -> unknown server error
       #   [MODIFIED, 3xx, 4xx, 5xx mapped, everything else is Generic ApiError]
       # 7. Everything else -> unknown client error
-      def error?(http_resp)
-        return true if @error_code_fn.call(http_resp)
+      def error?(error_code, http_resp)
+        return true if error_code
         return false if http_resp.status == @success_status
 
         !(200..299).cover?(http_resp.status)
       end
 
-      def extract_error(http_resp)
-        error_code = @error_code_fn.call(http_resp)
+      def create_error(error_code, http_resp, metadata)
         error_class = error_class(error_code) if error_code
 
         error_opts = {
           http_resp: http_resp,
           error_code: error_code,
+          metadata: metadata,
           message: error_code # default message
         }
 
@@ -84,9 +84,7 @@ module Hearth
       end
 
       def error_class(error_code)
-        @errors.find do |e|
-          e.name.include? error_code
-        end
+        @errors.find { |e| e.name.include? error_code }
       end
 
       def generic_error(error_opts)

data/lib/hearth/http/field.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Hearth
+  module HTTP
+    # Represents an HTTP field.
+    class Field
+      # @param [String] name The name of the field.
+      # @param [#to_s] value (nil) The value for the field. It can be any
+      #   object that responds to `#to_s`.
+      # @param [Symbol] kind The kind of field, either :header or :trailer.
+      def initialize(name, value = nil, kind: :header)
+        if name.nil? || name.empty?
+          raise ArgumentError, 'Field name must be a non-empty String'
+        end
+
+        @name = name
+        @value = value
+        @kind = kind
+      end
+
+      # @return [String]
+      attr_reader :name
+
+      # @return [Symbol]
+      attr_reader :kind
+
+      # Returns a string representation of the field.
+      # @return [String]
+      def value(encoding = nil)
+        encoding ? @value.to_s.encode(encoding) : @value.to_s
+      end
+
+      # @return [Boolean]
+      def header?
+        @kind == :header
+      end
+
+      # @return [Boolean]
+      def trailer?
+        @kind == :trailer
+      end
+
+      # @return [Hash]
+      def to_h
+        { @name => value }
+      end
+    end
+  end
+end

data/lib/hearth/http/fields.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module Hearth
+  module HTTP
+    # Provides Hash like access for Headers and Trailers with key normalization
+    class Fields
+      include Enumerable
+
+      # @param [Array<Field>] fields
+      # @param [String] encoding
+      def initialize(fields = [], encoding: 'utf-8')
+        unless fields.is_a?(Enumerable)
+          raise ArgumentError, 'fields must be an Enumerable of Field'
+        end
+
+        @entries = {}
+        fields.each { |field| self[field.name] = field }
+        @encoding = encoding
+      end
+
+      # @return [String]
+      attr_reader :encoding
+
+      # @param [String] key
+      def [](key)
+        @entries[key.downcase]
+      end
+
+      # @param [String] key
+      # @param [Field] value
+      def []=(key, value)
+        raise ArgumentError, 'value must be a Field' unless value.is_a?(Field)
+
+        @entries[key.downcase] = value
+      end
+
+      # @param [String] key
+      # @return [Boolean] Returns `true` if there is a Field with the given key.
+      def key?(key)
+        @entries.key?(key.downcase)
+      end
+
+      # @param [String] key
+      # @return [Field, nil] Returns the Field for the deleted Field key.
+      def delete(key)
+        @entries.delete(key.downcase)
+      end
+
+      # @return [Enumerable<Field>]
+      def each(&block)
+        @entries.values.each(&block)
+      end
+
+      # @return [Integer] Returns the number of Field entries.
+      def size
+        @entries.size
+      end
+
+      # @return [Hash]
+      def clear
+        @entries = {}
+      end
+
+      # @api private
+      def inspect
+        super.gsub(/ @entries={.*},/, '')
+      end
+
+      # Proxy class that wraps Fields to create Headers and Trailers
+      class Proxy
+        include Enumerable
+
+        def initialize(fields, kind)
+          @fields = fields
+          @kind = kind
+        end
+
+        # @param [String] key
+        def [](key)
+          @fields[key].value(@fields.encoding) if key?(key)
+        end
+
+        # @param [String] key
+        # @param [#to_s, Array<#to_s>] value
+        def []=(key, value)
+          @fields[key] = Field.new(key, value, kind: @kind)
+        end
+
+        # @param [String] key
+        # @return [Boolean] Returns `true` if there is a Field with the given
+        #   key and kind.
+        def key?(key)
+          @fields.key?(key) && @fields[key].kind == @kind
+        end
+
+        # @param [String] key
+        # @return [Field, nil] Returns the value for the deleted Field key.
+        def delete(key)
+          @fields.delete(key).value(@fields.encoding) if key?(key)
+        end
+
+        # @return [Enumerable<String,String>]
+        def each(&block)
+          @fields.filter { |f| f.kind == @kind }
+                 .to_h { |f| [f.name, f.value(@fields.encoding)] }
+                 .each(&block)
+        end
+        alias each_pair each
+
+        # @api private
+        def inspect
+          to_h
+        end
+      end
+    end
+  end
+end