hearth 1.0.0.pre1 → 1.0.0.pre2

data/lib/hearth/dns/host_resolver.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Hearth
+  module DNS
+    # Resolves a host name and service to an IP address. Can be used with the
+    # {HTTP::Client} host_resolver option. This implementation uses
+    # Addrinfo.getaddrinfo to resolve the host name.
+    #
+    # @see https://ruby-doc.org/stdlib-3.0.2/libdoc/socket/rdoc/Addrinfo.html
+    class HostResolver
+      # @param [Integer] service (443)
+      # @param [Integer] family (nil)
+      # @param [Symbol] socktype (:SOCK_STREAM)
+      # @param [Integer] protocol (nil)
+      # @param [Integer] flags (nil)
+      def initialize(service: 443, family: nil, socktype: :SOCK_STREAM,
+                     protocol: nil, flags: nil)
+        @service = service
+        @family = family
+        @socktype = socktype
+        @protocol = protocol
+        @flags = flags
+      end
+
+      # @return [Integer]
+      attr_reader :service
+
+      # @return [Integer]
+      attr_reader :family
+
+      # @return [Symbol]
+      attr_reader :socktype
+
+      # @return [Integer]
+      attr_reader :protocol
+
+      # @return [Integer]
+      attr_reader :flags
+
+      # @param [String] nodename
+      # @param (see Hearth::DNS::HostResolver#initialize)
+      def resolve_address(nodename:, **kwargs)
+        options = kwargs.merge(nodename: nodename)
+        addrinfo_list = addrinfo(options)
+        ipv6 = ipv6_addr(addrinfo_list, options) if use_ipv6?
+        ipv4 = ipv4_addr(addrinfo_list, options)
+        [ipv6, ipv4]
+      end
+
+      private
+
+      def addrinfo(options)
+        Addrinfo.getaddrinfo(
+          options[:nodename],
+          options.fetch(:service, @service),
+          options.fetch(:family, @family),
+          options.fetch(:socktype, @socktype),
+          options.fetch(:protocol, @protocol),
+          options.fetch(:flags, @flags)
+        )
+      end
+
+      def ipv4_addr(addrinfo_list, options)
+        addr = addrinfo_list.find(&:ipv4?)
+        return unless addr
+
+        HostAddress.new(
+          address_type: :A,
+          address: addr.ip_address,
+          hostname: options[:nodename]
+        )
+      end
+
+      def ipv6_addr(addrinfo_list, options)
+        addr = addrinfo_list.find(&:ipv6?)
+        return unless addr
+
+        HostAddress.new(
+          address_type: :AAAA,
+          address: addr.ip_address,
+          hostname: options[:nodename]
+        )
+      end
+
+      def use_ipv6?
+        Socket.ip_address_list.any? do |a|
+          a.ipv6? && !a.ipv6_loopback? && !a.ipv6_linklocal?
+        end
+      end
+    end
+  end
+end

data/lib/hearth/dns.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'socket'
+
+require_relative 'dns/host_address'
+require_relative 'dns/host_resolver'
+
+# These patches are based on resolv-replace
+# https://github.com/ruby/ruby/blob/master/lib/resolv-replace.rb
+# We cannot require resolv-replace because it would change DNS resolution
+# globally. When opening an HTTP request, we will set a thread local variable
+# to enable custom DNS resolution, and then disable it after the request is
+# complete. When the thread local variable is not set, we will use the default
+# Ruby DNS resolution, which may be Resolv or the system resolver.
+
+# Patch IPSocket
+# @api private
+class << IPSocket
+  alias original_hearth_getaddress getaddress
+
+  def getaddress(host)
+    unless (resolver = Thread.current[:net_http_hearth_dns_resolver])
+      return original_hearth_getaddress(host)
+    end
+
+    ipv6, ipv4 = resolver.resolve_address(nodename: host)
+    return ipv6.address if ipv6
+
+    ipv4.address
+  end
+end
+
+# Patch TCPSocket
+# @api private
+class TCPSocket < IPSocket
+  alias original_hearth_initialize initialize
+
+  # rubocop:disable Lint/MissingSuper
+  def initialize(host, serv, *rest)
+    if Thread.current[:net_http_hearth_dns_resolver]
+      rest[0] = IPSocket.getaddress(rest[0]) if rest[0]
+      original_hearth_initialize(IPSocket.getaddress(host), serv, *rest)
+    else
+      original_hearth_initialize(host, serv, *rest)
+    end
+  end
+  # rubocop:enable Lint/MissingSuper
+end

data/lib/hearth/http/api_error.rb

@@ -7,23 +7,19 @@ module Hearth
     class ApiError < Hearth::ApiError
       def initialize(http_resp:, **kwargs)
         @http_status = http_resp.status
-        @http_headers = http_resp.headers
+        @http_fields = http_resp.fields
         @http_body = http_resp.body
-        @request_id = http_resp.headers['x-request-id']
         super(**kwargs)
       end
 
       # @return [Integer]
       attr_reader :http_status
 
-      # @return [Hash<String, String>]
-      attr_reader :http_headers
+      # @return [Fields]
+      attr_reader :http_fields
 
-      # @return [String]
+      # @return [IO]
       attr_reader :http_body
-
-      # @return [String]
-      attr_reader :request_id
     end
   end
 end

data/lib/hearth/http/client.rb

@@ -1,67 +1,112 @@
 # frozen_string_literal: true
 
+require 'delegate'
 require 'net/http'
 require 'logger'
 require 'openssl'
 
 module Hearth
   module HTTP
-    # Transmits an HTTP {Request} object, returning an HTTP {Response}.
-    # @api private
+    # An HTTP client that uses Net::HTTP to send requests.
     class Client
+      # @api private
+      OPTIONS = {
+        logger: Logger.new($stdout),
+        debug_output: nil,
+        proxy: nil,
+        open_timeout: 15,
+        read_timeout: nil,
+        keep_alive_timeout: 5,
+        continue_timeout: 1,
+        write_timeout: nil,
+        ssl_timeout: nil,
+        verify_peer: true,
+        ca_file: nil,
+        ca_path: nil,
+        cert_store: nil,
+        host_resolver: nil
+      }.freeze
+
       # Initialize an instance of this HTTP client.
       #
       # @param [Hash] options The options for this HTTP Client
       #
-      # @option options [Boolean] :http_wire_trace (false) When `true`,
-      #   HTTP debug output will be sent to the `:logger`.
+      # @option options [Logger] :logger (Logger.new($stdout)) A logger
+      #   used to log Net::HTTP requests and responses when `:debug_output`
+      #   is enabled.
       #
-      # @option options [Logger] :logger A logger where debug output is sent.
+      # @option options [Boolean] :debug_output (false) When `true`,
+      #   sets an output stream to the configured Logger for debugging.
       #
-      # @option options [URI::HTTP,String] :http_proxy A proxy to send
+      # @option options [String, URI] :proxy A proxy to send
       #   requests through. Formatted like 'http://proxy.com:123'.
       #
-      # @option options [Boolean] :ssl_verify_peer (true) When `true`,
+      # @option options [Float] :open_timeout (15) Number of seconds to
+      #   wait for the connection to open.
+      #
+      # @option options [Float] :read_timeout Number of seconds to wait
+      #   for one block to be read (via one read(2) call).
+      #
+      # @option options [Float] :keep_alive_timeout (5) Seconds to reuse the
+      #   connection of the previous request.
+      #
+      # @option options [Float] :continue_timeout (1) Seconds to wait for
+      #   100 Continue response.
+      #
+      # @option options [Float] :write_timeout Number of seconds to wait
+      #   for one block to be written (via one write(2) call).
+      #
+      # @option options [Float] :ssl_timeout Sets the SSL timeout seconds.
+      #
+      # @option options [Boolean] :verify_peer (true) When `true`,
       #   SSL peer certificates are verified when establishing a
       #   connection.
       #
-      # @option options [String] :ssl_ca_bundle Full path to the SSL
+      # @option options [String] :ca_file Full path to the SSL
       #   certificate authority bundle file that should be used when
       #   verifying peer certificates.  If you do not pass
-      #   `:ssl_ca_bundle` or `:ssl_ca_directory` the system default
+      #   `:ca_file` or `:ca_path` the system default
       #   will be used if available.
       #
-      # @option options [String] :ssl_ca_directory Full path of the
+      # @option options [String] :ca_path Full path of the
       #   directory that contains the unbundled SSL certificate
       #   authority files for verifying peer certificates.  If you do
-      #   not pass `:ssl_ca_bundle` or `:ssl_ca_directory` the
+      #   not pass `:ca_file` or `:ca_path` the
       #   system default will be used if available.
+      #
+      # @option options [OpenSSL::X509::Store] :cert_store An OpenSSL X509
+      #   certificate store that contains the SSL certificate authority.
+      #
+      # @option options [#resolve_address] :host_resolver
+      #   An object, such as {Hearth::DNS::HostResolver} that responds to
+      #   `#resolve_address`, returning an array of up to two IP addresses for
+      #   the given hostname, one IPv6 and one IPv4, in that order.
+      #   `#resolve_address` should take a nodename keyword argument and
+      #   optionally other keyword args similar to Addrinfo.getaddrinfo's
+      #   positional parameters.
       def initialize(options = {})
-        @http_wire_trace = options[:http_wire_trace]
-        @logger = options[:logger]
-        @http_proxy = options[:http_proxy]
-        @http_proxy = URI.parse(@http_proxy.to_s) if @http_proxy
-        @ssl_verify_peer = options[:ssl_verify_peer]
-        @ssl_ca_bundle = options[:ssl_ca_bundle]
-        @ssl_ca_directory = options[:ssl_ca_directory]
-        @ssl_ca_store = options[:ssl_ca_store]
+        unknown = options.keys - OPTIONS.keys
+        raise ArgumentError, "Unknown options: #{unknown}" unless unknown.empty?
+
+        OPTIONS.each_pair do |opt_name, default_value|
+          value = options.key?(opt_name) ? options[opt_name] : default_value
+          instance_variable_set("@#{opt_name}", value)
+        end
+      end
+
+      OPTIONS.each_key do |attr_name|
+        attr_reader(attr_name)
       end
 
       # @param [Request] request
       # @param [Response] response
+      # @param [Logger] logger (nil)
       # @return [Response]
-      def transmit(request:, response:)
-        uri = URI.parse(request.url)
-        http = create_http(uri)
-        http.set_debug_output(@logger) if @http_wire_trace
-
-        if uri.scheme == 'https'
-          configure_ssl(http)
-        else
-          http.use_ssl = false
+      def transmit(request:, response:, logger: nil)
+        net_request = build_net_request(request)
+        with_connection_pool(request.uri, logger) do |connection|
+          _transmit(connection, net_request, response)
         end
-
-        _transmit(http, request, response)
         response.body.rewind if response.body.respond_to?(:rewind)
         response
       rescue ArgumentError => e
@@ -73,37 +118,83 @@ module Hearth
 
       private
 
-      def _transmit(http, request, response)
-        http.start do |conn|
-          conn.request(build_net_request(request)) do |net_resp|
-            response.status = net_resp.code.to_i
-            response.headers = extract_headers(net_resp)
-            net_resp.read_body do |chunk|
-              response.body.write(chunk)
-            end
-          end
+      def with_connection_pool(endpoint, logger)
+        pool = ConnectionPool.for(pool_config)
+        connection = pool.connection_for(endpoint) do
+          new_connection(endpoint, logger)
+        end
+        yield connection
+        pool.offer(endpoint, connection)
+      rescue StandardError => e
+        connection&.finish
+        raise e
+      end
+
+      # Starts and returns a new HTTP connection.
+      # @param [URI] endpoint
+      # @return [Net::HTTP]
+      def new_connection(endpoint, logger)
+        http = create_http(endpoint)
+        http.set_debug_output(logger || @logger) if @debug_output
+        configure_timeouts(http)
+
+        if endpoint.scheme == 'https'
+          configure_ssl(http)
+        else
+          http.use_ssl = false
+        end
+
+        http.start
+        http
+      end
+
+      def _transmit(http, net_request, response)
+        # Inform monkey patch to use our DNS resolver
+        Thread.current[:net_http_hearth_dns_resolver] = @host_resolver
+        http.request(net_request) do |net_resp|
+          unpack_response(net_resp, response)
         end
+      ensure
+        # Restore the default DNS resolver
+        Thread.current[:net_http_hearth_dns_resolver] = nil
       end
 
-      # Creates an HTTP connection to the endpoint
-      # Applies proxy if set
+      def unpack_response(net_resp, response)
+        response.status = net_resp.code.to_i
+        net_resp.each_header { |k, v| response.headers[k] = v }
+        net_resp.read_body do |chunk|
+          response.body.write(chunk)
+        end
+      end
+
+      # Creates an HTTP connection to the endpoint.
+      # Applies proxy if set.
       def create_http(endpoint)
         args = []
         args << endpoint.host
         args << endpoint.port
-        args += http_proxy_parts if @http_proxy
+        args += proxy_parts if @proxy
         # Net::HTTP.new uses positional arguments: host, port, proxy_args....
-        Net::HTTP.new(*args.compact)
+        HTTP.new(Net::HTTP.new(*args.compact))
+      end
+
+      def configure_timeouts(http)
+        http.open_timeout = @open_timeout
+        http.keep_alive_timeout = @keep_alive_timeout
+        http.read_timeout = @read_timeout
+        http.continue_timeout = @continue_timeout
+        http.write_timeout = @write_timeout
       end
 
       # applies ssl settings to the HTTP object
       def configure_ssl(http)
         http.use_ssl = true
-        if @ssl_verify_peer
+        http.ssl_timeout = @ssl_timeout
+        if @verify_peer
           http.verify_mode = OpenSSL::SSL::VERIFY_PEER
-          http.ca_file = @ssl_ca_bundle if @ssl_ca_bundle
-          http.ca_path = @ssl_ca_directory if @ssl_ca_directory
-          http.cert_store = @ssl_ca_store if @ssl_ca_store
+          http.ca_file = @ca_file
+          http.ca_path = @ca_path
+          http.cert_store = @cert_store
         else
           http.verify_mode = OpenSSL::SSL::VERIFY_NONE
         end
@@ -115,15 +206,28 @@ module Hearth
       # @return [Net::HTTP::Request]
       def build_net_request(request)
         request_class = net_http_request_class(request)
-        req = request_class.new(request.url, request.headers.to_h)
-        req.body_stream = request.body
+        req = request_class.new(request.uri, net_headers_for(request))
+
+        # Net::HTTP adds a default Content-Type when a body is present.
+        # Set the body stream when it has an unknown size or when it is > 0.
+        # We instead add our own Content-Length header via Middleware.
+        if !request.body.respond_to?(:size) ||
+           (request.body.respond_to?(:size) && request.body.size.positive?)
+          req.body_stream = request.body
+        end
         req
       end
 
-      # @param [Net::HTTP::Response] response
+      # Validate that fields are not trailers and return a hash of headers.
+      # @param [HTTP::Request] request
       # @return [Hash<String, String>]
-      def extract_headers(response)
-        response.to_hash.transform_values(&:first)
+      def net_headers_for(request)
+        # Trailers are not supported in Net::HTTP
+        if request.trailers.any?
+          raise NotImplementedError, 'Trailers are not supported in Net::HTTP'
+        end
+
+        request.headers.to_h
       end
 
       # @param [Http::Request] request
@@ -137,16 +241,61 @@ module Hearth
         raise ArgumentError, msg
       end
 
-      # Extract the parts of the http_proxy URI
-      # @return [Array(String)]
-      def http_proxy_parts
+      # Extract the parts of the proxy URI
+      # @return [Array]
+      def proxy_parts
+        proxy = URI(@proxy)
         [
-          @http_proxy.host,
-          @http_proxy.port,
-          (@http_proxy.user && CGI.unescape(@http_proxy.user)),
-          (@http_proxy.password && CGI.unescape(@http_proxy.password))
+          proxy.host,
+          proxy.port,
+          proxy.user && CGI.unescape(proxy.user),
+          proxy.password && CGI.unescape(proxy.password)
         ]
       end
+
+      # Config options for the HTTP client used for connection pooling
+      # @return [Hash]
+      def pool_config
+        OPTIONS.each_key.with_object({}) do |option_name, hash|
+          hash[option_name] = instance_variable_get("@#{option_name}")
+        end
+      end
+
+      # Helper methods extended onto Net::HTTP objects.
+      # @api private
+      class HTTP < SimpleDelegator
+        def initialize(http)
+          super(http)
+          @http = http
+        end
+
+        # @return [Integer, nil]
+        attr_reader :last_used
+
+        # Sends the request and tracks that this connection has been used.
+        def request(...)
+          @http.request(...)
+          @last_used = monotonic_milliseconds
+        end
+
+        def stale?
+          @last_used.nil? ||
+            (monotonic_milliseconds - @last_used) > keep_alive_timeout * 1000
+        end
+
+        # Attempts to close/finish the connection without raising an error.
+        def finish
+          @http.finish
+        rescue IOError
+          nil
+        end
+
+        private
+
+        def monotonic_milliseconds
+          Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
+        end
+      end
     end
   end
 end

data/lib/hearth/http/error_inspector.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require 'time'
+
+module Hearth
+  module HTTP
+    # An HTTP error inspector, using hints from status code and headers.
+    # @api private
+    class ErrorInspector
+      def initialize(error, http_response)
+        @error = error
+        @http_response = http_response
+      end
+
+      def retryable?
+        (modeled_retryable? ||
+          throttling? ||
+          transient? ||
+          server?) &&
+          @http_response.body.respond_to?(:truncate)
+      end
+
+      def error_type
+        if transient?
+          'Transient'
+        elsif throttling?
+          'Throttling'
+        elsif server?
+          'ServerError'
+        elsif client?
+          'ClientError'
+        else
+          'Unknown'
+        end
+      end
+
+      def hints
+        hints = {}
+        if (retry_after = retry_after_hint)
+          hints[:retry_after] = retry_after
+        end
+        hints
+      end
+
+      private
+
+      def transient?
+        @error.is_a?(Hearth::HTTP::NetworkingError)
+      end
+
+      def throttling?
+        @http_response.status == 429 || modeled_throttling?
+      end
+
+      def server?
+        (500..599).cover?(@http_response.status)
+      end
+
+      def client?
+        (400..499).cover?(@http_response.status)
+      end
+
+      def modeled_retryable?
+        @error.is_a?(Hearth::ApiError) && @error.retryable?
+      end
+
+      def modeled_throttling?
+        modeled_retryable? && @error.throttling?
+      end
+
+      def retry_after_hint
+        retry_after = @http_response.headers['retry-after']
+        Integer(retry_after)
+      rescue ArgumentError # string is present, assume it is a date
+        begin
+          Time.parse(retry_after) - Time.now
+        rescue ArgumentError # empty string, somehow
+          nil
+        end
+      rescue TypeError # header is not prseent
+        nil
+      end
+    end
+  end
+end

data/lib/hearth/http/error_parser.rb

@@ -9,17 +9,18 @@ module Hearth
     # @api private
     class ErrorParser
       # @api private
-      HTTP_3XX = (300..399).freeze
+      HTTP_3XX = (300..399)
 
       # @api private
-      HTTP_4XX = (400..499).freeze
+      HTTP_4XX = (400..499)
 
       # @api private
-      HTTP_5XX = (500..599).freeze
+      HTTP_5XX = (500..599)
 
       # @param [Module] error_module The code generated Errors module.
       #   Must contain service specific implementations of
-      #   ApiRedirectError, ApiClientError, and ApiServerError
+      #   ApiRedirectError, ApiClientError, and ApiServerError, and it
+      #   must have defined a `self.error_code(http_resp)` method.
       #
       # @param [Integer] success_status The status code of a
       #   successful response as defined by the model for
@@ -30,22 +31,21 @@ module Hearth
       #
       # @param [Array<Class<ApiError>>] errors Array of Error classes
       #   modeled for the operation.
-      #
-      # @param [callable] error_code_fn Protocol specific function
-      #   that will return the error code from a response, or nil if
-      #   there is none.
-      def initialize(error_module:, success_status:, errors:, error_code_fn:)
+      def initialize(error_module:, success_status:, errors:)
         @error_module = error_module
         @success_status = success_status
         @errors = errors
-        @error_code_fn = error_code_fn
       end
 
       # Parse and return the error if the response is not successful.
       #
-      # @param [Response] response The HTTP response
-      def parse(response)
-        extract_error(response) if error?(response)
+      # @param [HTTP::Response] http_resp The HTTP Response
+      # @param [Hash] metadata The metadata from {Hearth::Output}
+      def parse(http_resp, metadata)
+        error_code = @error_module.method(:error_code).call(http_resp)
+        return unless error?(error_code, http_resp)
+
+        create_error(error_code, http_resp, metadata)
       end
 
       private
@@ -59,20 +59,20 @@ module Hearth
       # 6. Response code 5xx -> unknown server error
       #   [MODIFIED, 3xx, 4xx, 5xx mapped, everything else is Generic ApiError]
       # 7. Everything else -> unknown client error
-      def error?(http_resp)
-        return true if @error_code_fn.call(http_resp)
+      def error?(error_code, http_resp)
+        return true if error_code
         return false if http_resp.status == @success_status
 
         !(200..299).cover?(http_resp.status)
       end
 
-      def extract_error(http_resp)
-        error_code = @error_code_fn.call(http_resp)
+      def create_error(error_code, http_resp, metadata)
         error_class = error_class(error_code) if error_code
 
         error_opts = {
           http_resp: http_resp,
           error_code: error_code,
+          metadata: metadata,
           message: error_code # default message
         }
 
@@ -84,9 +84,7 @@ module Hearth
       end
 
       def error_class(error_code)
-        @errors.find do |e|
-          e.name.include? error_code
-        end
+        @errors.find { |e| e.name.include? error_code }
       end
 
       def generic_error(error_opts)