httpx 1.3.4 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 59239a38c693a32e279f32e5f153f606fdc83e0cdce02e6079947cc8f8d6e0d0
-  data.tar.gz: c386fbdd0409186f50390a2379c3945add29a7cafac805bbd0c780992e933437
+  metadata.gz: 7edc783221b5d919a1c788bf360f0cc2d43ee1556eb893c6b5d377d1bd3b4241
+  data.tar.gz: 14dc4593ae5c46acb33a4f242a0058e75eddec70bd22fdf75c05496240de3abc
 SHA512:
-  metadata.gz: 5d142fdfe649e0447f3043e5eebd52776c71c44c4b141607b061895c520e32a5d1da0c5c678c6fd6cdb9f572e1d3c43165d8c26bf006420c2b89ce66c547c986
-  data.tar.gz: 77435864f6afd01fdb7fbc8e1019928411a50032c11751220062f1fff804355d0220823f18cbef430e47bc26f3c28fab826513a242222ccdb003eae949a41b97
+  metadata.gz: 181bc9e6155708c2d1cdaee3dc7c5c15cd5e2b1ba8d7911b6dd5a79e1cebe38558766ec89d8446483d434c7387e330874a70633b2ed5fb8f6a9d8a3401a4abf4
+  data.tar.gz: 49f57c406a2a5c5be83c018c2151ef2e842c266e93d9609dff15684e662bb25580e7c84e2e96a32d1a45094c9baf8ad18f89dbab1e8235127f4d40429f733572

data/doc/release_notes/1_4_0.md

@@ -0,0 +1,43 @@
+# 1.4.0
+
+## Features
+
+### `:content_digest` plugin
+
+The `:content_digest` can be used to calculate the digest of request payloads and set them in the `"content-digest"` header; it can also validate the integrity of responses which declare the same `"content-digest"` header.
+
+More info under https://honeyryderchuck.gitlab.io/httpx/wiki/Content-Digest
+
+## Per-session connection pools
+
+This architectural changes moves away from per-thread shared connection pools, and into per-session (also thread-safe) connection pools. Unlike before, this enables connections from a session to be reused across threads, as well as limiting the number of connections that can be open on a given origin peer. This fixes long-standing issues, such as reusing connections under a fiber scheduler loop (such as the one from the gem `async`).
+
+A new `:pool_options` option is introduced, which can be passed an hash with the following sub-options:
+
+* `:max_connections_per_origin`: maximum number of connections a pool allows (unbounded by default, for backwards compatibility).
+* `:pool_timeout`: the number of seconds a session will wait for a connection to be checked out (default: 5)
+
+More info under https://honeyryderchuck.gitlab.io/httpx/wiki/Connection-Pools
+
+
+## Improvements
+
+* `:aws_sigv4` plugin: improved digest calculation on compressed request bodies by buffering content to a tempfile.
+* `HTTPX::Response#json` will parse payload from extended json MIME types (like `application/ld+json`, `application/hal+json`, ...).
+
+## Bugfixes
+
+* `:aws_sigv4` plugin: do not try to rewind a request body which yields chunks.
+* fixed request encoding when `:json` param is passed, and the `oj` gem is used (by using the `:compat` flag).
+* native resolver: on message truncation, bubble up tcp handshake errors as resolve errors.
+* allow `HTTPX::Response#json` to accept extended JSON mime types (such as responses with `content-type: application/ld+json`)
+
+## Chore
+
+* default options are now fully frozen (in case anyone relies on overriding them).
+
+### `:xml` plugin
+
+XML encoding/decoding (via `:xml` request param, and `HTTPX::Response#xml`) is now available via the `:xml` plugin.
+
+Using `HTTPX::Response#xml` without the plugin will issue a deprecation warning.

data/lib/httpx/adapters/faraday.rb

@@ -108,9 +108,11 @@ module Faraday
             ssl_options
           end
         else
+          # :nocov:
           def ssl_options_from_env(*)
             {}
           end
+          # :nocov:
         end
       end
 

data/lib/httpx/adapters/webmock.rb

@@ -52,16 +52,18 @@ module WebMock
       end
 
       module InstanceMethods
-        def init_connection(*)
-          connection = super
+        private
+
+        def do_init_connection(connection, selector)
+          super
+
           connection.once(:unmock_connection) do
             unless connection.addresses
               connection.__send__(:callbacks)[:connect_error].clear
-              pool.__send__(:unregister_connection, connection)
+              deselect_connection(connection, selector)
             end
-            pool.__send__(:resolve_connection, connection)
+            resolve_connection(connection, selector)
           end
-          connection
         end
       end
 
@@ -100,6 +102,10 @@ module WebMock
           super
         end
 
+        def terminate
+          force_reset
+        end
+
         def send(request)
           request_signature = Plugin.build_webmock_request_signature(request)
           WebMock::RequestRegistry.instance.requested_signatures.put(request_signature)

data/lib/httpx/callbacks.rb

@@ -15,11 +15,6 @@ module HTTPX
       self
     end
 
-    def only(type, &block)
-      callbacks(type).clear
-      on(type, &block)
-    end
-
     def emit(type, *args)
       callbacks(type).delete_if { |pr| :delete == pr.call(*args) } # rubocop:disable Style/YodaCondition
     end

data/lib/httpx/chainable.rb

@@ -73,7 +73,7 @@ module HTTPX
         ].include?(callback)
 
         warn "DEPRECATION WARNING: calling `.#{meth}` on plain HTTPX sessions is deprecated. " \
-             "Use HTTPX.plugin(:callbacks).#{meth} instead."
+             "Use `HTTPX.plugin(:callbacks).#{meth}` instead."
 
         plugin(:callbacks).__send__(meth, *args, **options, &blk)
       else
@@ -101,4 +101,6 @@ module HTTPX
       end
     end
   end
+
+  extend Chainable
 end

data/lib/httpx/connection/http2.rb

@@ -98,10 +98,10 @@ module HTTPX
         @streams.size < @max_requests
     end
 
-    def send(request)
+    def send(request, head = false)
       unless can_buffer_more_requests?
-        @pending << request
-        return
+        head ? @pending.unshift(request) : @pending << request
+        return false
       end
       unless (stream = @streams[request])
         stream = @connection.new_stream
@@ -113,6 +113,7 @@ module HTTPX
       true
     rescue ::HTTP2::Error::StreamLimitExceeded
       @pending.unshift(request)
+      false
     end
 
     def consume
@@ -154,8 +155,7 @@ module HTTPX
 
     def send_pending
       while (request = @pending.shift)
-        # TODO: this request should go back to top of stack
-        break unless send(request)
+        break unless send(request, true)
       end
     end
 
@@ -327,10 +327,14 @@ module HTTPX
         end
       end
       send(@pending.shift) unless @pending.empty?
+
       return unless @streams.empty? && exhausted?
 
-      close
-      emit(:exhausted) unless @pending.empty?
+      if @pending.empty?
+        close
+      else
+        emit(:exhausted)
+      end
     end
 
     def on_frame(bytes)

data/lib/httpx/connection.rb

@@ -43,13 +43,14 @@ module HTTPX
 
     attr_reader :type, :io, :origin, :origins, :state, :pending, :options, :ssl_session
 
-    attr_writer :timers
+    attr_writer :current_selector, :coalesced_connection
 
-    attr_accessor :family
+    attr_accessor :current_session, :family
 
     def initialize(uri, options)
-      @origins = [uri.origin]
-      @origin = Utils.to_uri(uri.origin)
+      @current_session = @current_selector = @coalesced_connection = nil
+      @exhausted = @cloned = false
+
       @options = Options.new(options)
       @type = initialize_type(uri, @options)
       @origins = [uri.origin]
@@ -58,6 +59,7 @@ module HTTPX
       @read_buffer = Buffer.new(@options.buffer_size)
       @write_buffer = Buffer.new(@options.buffer_size)
       @pending = []
+
       on(:error, &method(:on_error))
       if @options.io
         # if there's an already open IO, get its
@@ -68,6 +70,31 @@ module HTTPX
       else
         transition(:idle)
       end
+      on(:activate) do
+        @current_session.select_connection(self, @current_selector)
+      end
+      on(:close) do
+        next if @exhausted # it'll reset
+
+        # may be called after ":close" above, so after the connection has been checked back in.
+        # next unless @current_session
+
+        next unless @current_session
+
+        @current_session.deselect_connection(self, @current_selector, @cloned)
+      end
+      on(:terminate) do
+        next if @exhausted # it'll reset
+
+        # may be called after ":close" above, so after the connection has been checked back in.
+        next unless @current_session
+
+        @current_session.deselect_connection(self, @current_selector)
+      end
+
+      on(:altsvc) do |alt_origin, origin, alt_params|
+        build_altsvc_connection(alt_origin, origin, alt_params)
+      end
 
       @inflight = 0
       @keep_alive_timeout = @options.timeout[:keep_alive_timeout]
@@ -77,6 +104,10 @@ module HTTPX
       self.addresses = @options.addresses if @options.addresses
     end
 
+    def peer
+      @origin
+    end
+
     # this is a semi-private method, to be used by the resolver
     # to initiate the io object.
     def addresses=(addrs)
@@ -168,7 +199,12 @@ module HTTPX
     end
 
     def inflight?
-      @parser && !@parser.empty? && !@write_buffer.empty?
+      @parser && (
+        # parser may be dealing with other requests (possibly started from a different fiber)
+        !@parser.empty? ||
+        # connection may be doing connection termination handshake
+        !@write_buffer.empty?
+      )
     end
 
     def interests
@@ -184,6 +220,9 @@ module HTTPX
 
       return @parser.interests if @parser
 
+      nil
+    rescue StandardError => e
+      emit(:error, e)
       nil
     end
 
@@ -205,6 +244,9 @@ module HTTPX
         consume
       end
       nil
+    rescue StandardError => e
+      emit(:error, e)
+      raise e
     end
 
     def close
@@ -221,8 +263,9 @@ module HTTPX
 
     # bypasses the state machine to force closing of connections still connecting.
     # **only** used for Happy Eyeballs v2.
-    def force_reset
+    def force_reset(cloned = false)
       @state = :closing
+      @cloned = cloned
       transition(:closed)
     end
 
@@ -235,6 +278,8 @@ module HTTPX
     end
 
     def send(request)
+      return @coalesced_connection.send(request) if @coalesced_connection
+
       if @parser && !@write_buffer.full?
         if @response_received_at && @keep_alive_timeout &&
            Utils.elapsed_time(@response_received_at) > @keep_alive_timeout
@@ -255,6 +300,8 @@ module HTTPX
     end
 
     def timeout
+      return if @state == :closed || @state == :inactive
+
       return @timeout if @timeout
 
       return @options.timeout[:connect_timeout] if @state == :idle
@@ -301,6 +348,12 @@ module HTTPX
       transition(:open)
     end
 
+    def disconnect
+      emit(:close)
+      @current_session = nil
+      @current_selector = nil
+    end
+
     def consume
       return unless @io
 
@@ -475,8 +528,25 @@ module HTTPX
         request.emit(:promise, parser, stream)
       end
       parser.on(:exhausted) do
+        @exhausted = true
+        current_session = @current_session
+        current_selector = @current_selector
+        parser.close
         @pending.concat(parser.pending)
-        emit(:exhausted)
+        case @state
+        when :closed
+          idling
+          @exhausted = false
+          @current_session = current_session
+          @current_selector = current_selector
+        when :closing
+          once(:close) do
+            idling
+            @exhausted = false
+            @current_session = current_session
+            @current_selector = current_selector
+          end
+        end
       end
       parser.on(:origin) do |origin|
         @origins |= [origin]
@@ -492,8 +562,14 @@ module HTTPX
       end
       parser.on(:reset) do
         @pending.concat(parser.pending) unless parser.empty?
+        current_session = @current_session
+        current_selector = @current_selector
         reset
-        idling unless @pending.empty?
+        unless @pending.empty?
+          idling
+          @current_session = current_session
+          @current_selector = current_selector
+        end
       end
       parser.on(:current_timeout) do
         @current_timeout = @timeout = parser.timeout
@@ -504,7 +580,15 @@ module HTTPX
       parser.on(:error) do |request, ex|
         case ex
         when MisdirectedRequestError
-          emit(:misdirected, request)
+          current_session = @current_session
+          current_selector = @current_selector
+          parser.close
+
+          other_connection = current_session.find_connection(@origin, current_selector,
+                                                             @options.merge(ssl: { alpn_protocols: %w[http/1.1] }))
+          other_connection.merge(self)
+          request.transition(:idle)
+          other_connection.send(request)
         else
           response = ErrorResponse.new(request, ex)
           request.response = response
@@ -531,13 +615,13 @@ module HTTPX
       error.set_backtrace(e.backtrace)
       connecting? && callbacks_for?(:connect_error) ? emit(:connect_error, error) : handle_error(error)
       @state = :closed
-      emit(:close)
+      disconnect
     rescue TLSError, ::HTTP2::Error::ProtocolError, ::HTTP2::Error::HandshakeError => e
       # connect errors, exit gracefully
       handle_error(e)
       connecting? && callbacks_for?(:connect_error) ? emit(:connect_error, e) : handle_error(e)
       @state = :closed
-      emit(:close)
+      disconnect
     end
 
     def handle_transition(nextstate)
@@ -582,7 +666,7 @@ module HTTPX
         return unless @write_buffer.empty?
 
         purge_after_closed
-        emit(:close) if @pending.empty?
+        disconnect if @pending.empty?
       when :already_open
         nextstate = :open
         # the first check for given io readiness must still use a timeout.
@@ -617,12 +701,40 @@ module HTTPX
       end
     end
 
+    # returns an HTTPX::Connection for the negotiated Alternative Service (or none).
+    def build_altsvc_connection(alt_origin, origin, alt_params)
+      # do not allow security downgrades on altsvc negotiation
+      return if @origin.scheme == "https" && alt_origin.scheme != "https"
+
+      altsvc = AltSvc.cached_altsvc_set(origin, alt_params.merge("origin" => alt_origin))
+
+      # altsvc already exists, somehow it wasn't advertised, probably noop
+      return unless altsvc
+
+      alt_options = @options.merge(ssl: @options.ssl.merge(hostname: URI(origin).host))
+
+      connection = @current_session.find_connection(alt_origin, @current_selector, alt_options)
+
+      # advertised altsvc is the same origin being used, ignore
+      return if connection == self
+
+      connection.extend(AltSvc::ConnectionMixin) unless connection.is_a?(AltSvc::ConnectionMixin)
+
+      log(level: 1) { "#{origin} alt-svc: #{alt_origin}" }
+
+      connection.merge(self)
+      terminate
+    rescue UnsupportedSchemeError
+      altsvc["noop"] = true
+      nil
+    end
+
     def build_socket(addrs = nil)
       case @type
       when "tcp"
-        TCP.new(@origin, addrs, @options)
+        TCP.new(peer, addrs, @options)
       when "ssl"
-        SSL.new(@origin, addrs, @options) do |sock|
+        SSL.new(peer, addrs, @options) do |sock|
           sock.ssl_session = @ssl_session
           sock.session_new_cb do |sess|
             @ssl_session = sess
@@ -635,7 +747,7 @@ module HTTPX
 
         path = String(path) if path
 
-        UNIX.new(@origin, path, @options)
+        UNIX.new(peer, path, @options)
       else
         raise Error, "unsupported transport (#{@type})"
       end
@@ -734,7 +846,7 @@ module HTTPX
 
     def set_request_timeout(request, timeout, start_event, finish_events, &callback)
       request.once(start_event) do
-        interval = @timers.after(timeout, callback)
+        interval = @current_selector.after(timeout, callback)
 
         Array(finish_events).each do |event|
           # clean up request timeouts if the connection errors out

data/lib/httpx/errors.rb

@@ -29,6 +29,18 @@ module HTTPX
     end
   end
 
+  # Raise when it can't acquire a connection for a given origin.
+  class PoolTimeoutError < TimeoutError
+    attr_reader :origin
+
+    # initializes the +origin+ it refers to, and the
+    # +timeout+ causing the error.
+    def initialize(origin, timeout)
+      @origin = origin
+      super(timeout, "Timed out after #{timeout} seconds while waiting for a connection to #{origin}")
+    end
+  end
+
   # Error raised when there was a timeout establishing the connection to a server.
   # This may be raised due to timeouts during TCP and TLS (when applicable) connection
   # establishment.

data/lib/httpx/loggable.rb

@@ -13,11 +13,14 @@ module HTTPX
       white: 37,
     }.freeze
 
+    USE_DEBUG_LOG = ENV.key?("HTTPX_DEBUG")
+
     def log(level: @options.debug_level, color: nil, &msg)
-      return unless @options.debug
       return unless @options.debug_level >= level
 
-      debug_stream = @options.debug
+      debug_stream = @options.debug || ($stderr if USE_DEBUG_LOG)
+
+      return unless debug_stream
 
       message = (+"" << msg.call << "\n")
       message = "\e[#{COLORS[color]}m#{message}\e[0m" if color && debug_stream.respond_to?(:isatty) && debug_stream.isatty
@@ -25,9 +28,6 @@ module HTTPX
     end
 
     def log_exception(ex, level: @options.debug_level, color: nil)
-      return unless @options.debug
-      return unless @options.debug_level >= level
-
       log(level: level, color: color) { ex.full_message }
     end
   end

data/lib/httpx/options.rb

@@ -25,14 +25,14 @@ module HTTPX
       end
     rescue NotImplementedError
       [Socket::AF_INET]
-    end
+    end.freeze
 
     DEFAULT_OPTIONS = {
       :max_requests => Float::INFINITY,
-      :debug => ENV.key?("HTTPX_DEBUG") ? $stderr : nil,
+      :debug => nil,
       :debug_level => (ENV["HTTPX_DEBUG"] || 1).to_i,
-      :ssl => {},
-      :http2_settings => { settings_enable_push: 0 },
+      :ssl => EMPTY_HASH,
+      :http2_settings => { settings_enable_push: 0 }.freeze,
       :fallback_protocol => "http/1.1",
       :supported_compression_formats => %w[gzip deflate],
       :decompress_response_body => true,
@@ -56,13 +56,15 @@ module HTTPX
       :response_class => Class.new(Response),
       :request_body_class => Class.new(Request::Body),
       :response_body_class => Class.new(Response::Body),
+      :pool_class => Class.new(Pool),
       :connection_class => Class.new(Connection),
       :options_class => Class.new(self),
       :transport => nil,
       :addresses => nil,
       :persistent => false,
       :resolver_class => (ENV["HTTPX_RESOLVER"] || :native).to_sym,
-      :resolver_options => { cache: true },
+      :resolver_options => { cache: true }.freeze,
+      :pool_options => EMPTY_HASH,
       :ip_families => ip_address_families,
     }.freeze
 
@@ -110,6 +112,7 @@ module HTTPX
     # :request_body_class :: class used to instantiate a request body
     # :response_body_class :: class used to instantiate a response body
     # :connection_class :: class used to instantiate connections
+    # :pool_class :: class used to instantiate the session connection pool
     # :options_class :: class used to instantiate options
     # :transport :: type of transport to use (set to "unix" for UNIX sockets)
     # :addresses :: bucket of peer addresses (can be a list of IP addresses, a hash of domain to list of adddresses;
@@ -118,7 +121,8 @@ module HTTPX
     # :persistent :: whether to persist connections in between requests (defaults to <tt>true</tt>)
     # :resolver_class :: which resolver to use (defaults to <tt>:native</tt>, can also be <tt>:system<tt> for
     #                    using getaddrinfo or <tt>:https</tt> for DoH resolver, or a custom class)
-    # :resolver_options :: hash of options passed to the resolver
+    # :resolver_options :: hash of options passed to the resolver. Accepted keys depend on the resolver type.
+    # :pool_options :: hash of options passed to the connection pool (See Pool#initialize).
     # :ip_families :: which socket families are supported (system-dependent)
     # :origin :: HTTP origin to set on requests with relative path (ex: "https://api.serv.com")
     # :base_path :: path to prefix given relative paths with (ex: "/v2")
@@ -215,6 +219,7 @@ module HTTPX
       ssl http2_settings
       request_class response_class headers_class request_body_class
       response_body_class connection_class options_class
+      pool_class pool_options
       io fallback_protocol debug debug_level resolver_class resolver_options
       compress_request_body decompress_response_body
       persistent
@@ -246,14 +251,6 @@ module HTTPX
       end
     end
 
-    OTHER_LOOKUP = ->(obj, k, ivar_map) {
-      case obj
-      when Hash
-        obj[ivar_map[k]]
-      else
-        obj.instance_variable_get(k)
-      end
-    }
     def merge(other)
       ivar_map = nil
       other_ivars = case other
@@ -266,12 +263,12 @@ module HTTPX
 
       return self if other_ivars.empty?
 
-      return self if other_ivars.all? { |ivar| instance_variable_get(ivar) == OTHER_LOOKUP[other, ivar, ivar_map] }
+      return self if other_ivars.all? { |ivar| instance_variable_get(ivar) == access_option(other, ivar, ivar_map) }
 
       opts = dup
 
       other_ivars.each do |ivar|
-        v = OTHER_LOOKUP[other, ivar, ivar_map]
+        v = access_option(other, ivar, ivar_map)
 
         unless v
           opts.instance_variable_set(ivar, v)
@@ -322,6 +319,10 @@ module HTTPX
         @response_body_class.__send__(:include, pl::ResponseBodyMethods) if defined?(pl::ResponseBodyMethods)
         @response_body_class.extend(pl::ResponseBodyClassMethods) if defined?(pl::ResponseBodyClassMethods)
       end
+      if defined?(pl::PoolMethods)
+        @pool_class = @pool_class.dup
+        @pool_class.__send__(:include, pl::PoolMethods)
+      end
       if defined?(pl::ConnectionMethods)
         @connection_class = @connection_class.dup
         @connection_class.__send__(:include, pl::ConnectionMethods)
@@ -346,5 +347,14 @@ module HTTPX
         instance_variable_set(:"@#{k}", value)
       end
     end
+
+    def access_option(obj, k, ivar_map)
+      case obj
+      when Hash
+        obj[ivar_map[k]]
+      else
+        obj.instance_variable_get(k)
+      end
+    end
   end
 end