httpx 1.7.2 → 1.7.4

data/lib/httpx/plugins/tracing.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module HTTPX::Plugins
+  #
+  # This plugin adds a simple interface to integrate request tracing SDKs.
+  #
+  # An example of such an integration is the datadog adapter.
+  #
+  # https://gitlab.com/os85/httpx/wikis/Tracing
+  #
+  module Tracing
+    class Wrapper
+      attr_reader :tracers
+      protected :tracers
+
+      def initialize(*tracers)
+        @tracers = tracers.flat_map do |tracer|
+          case tracer
+          when Wrapper
+            tracer.tracers
+          else
+            tracer
+          end
+        end.uniq
+      end
+
+      def merge(tracer)
+        case tracer
+        when Wrapper
+          Wrapper.new(*@tracers, *tracer.tracers)
+        else
+          Wrapper.new(*@tracers, tracer)
+        end
+      end
+
+      def freeze
+        @tracers.each(&:freeze).freeze
+        super
+      end
+
+      %i[start finish reset enabled?].each do |callback|
+        class_eval(<<-OUT, __FILE__, __LINE__ + 1)
+          # proxies ##{callback} calls to wrapper tracers.
+          def #{callback}(*args)                        # def start(*args)
+            @tracers.each { |t| t.#{callback}(*args) }  # @tracers.each { |t| t.start(*args) }
+          end                                           # end
+        OUT
+      end
+    end
+
+    # adds support for the following options:
+    #
+    # :tracer :: object which responds to #start, #finish and #reset.
+    module OptionsMethods
+      private
+
+      def option_tracer(tracer)
+        unless tracer.respond_to?(:start) &&
+               tracer.respond_to?(:finish) &&
+               tracer.respond_to?(:reset) &&
+               tracer.respond_to?(:enabled?)
+          raise TypeError, "#{tracer} must to respond to `#start(r)`, `#finish` and `#reset` and `#enabled?"
+        end
+
+        tracer = Wrapper.new(@tracer, tracer) if @tracer
+        tracer
+      end
+    end
+
+    module RequestMethods
+      attr_accessor :init_time
+
+      # intercepts request initialization to inject the tracing logic.
+      def initialize(*)
+        super
+
+        @init_time = nil
+
+        tracer = @options.tracer
+
+        return unless tracer && tracer.enabled?(self)
+
+        on(:idle) do
+          tracer.reset(self)
+
+          # request is reset when it's retried.
+          @init_time = nil
+        end
+        on(:headers) do
+          # the usual request init time (when not including the connection handshake)
+          # should be the time the request is buffered the first time.
+          @init_time ||= ::Time.now
+
+          tracer.start(self)
+        end
+        on(:response) { |response| tracer.finish(self, response) }
+      end
+
+      def response=(*)
+        # init_time should be set when it's send to a connection.
+        # However, there are situations where connection initialization fails.
+        # Example is the :ssrf_filter plugin, which raises an error on
+        # initialize if the host is an IP which matches against the known set.
+        # in such cases, we'll just set here right here.
+        @init_time ||= ::Time.now
+
+        super
+      end
+    end
+
+    # Connection mixin
+    module ConnectionMethods
+      def initialize(*)
+        super
+
+        @init_time = ::Time.now
+      end
+
+      def send(request)
+        # request init time is only the same as the connection init time
+        # if the connection is going through the connection handshake.
+        request.init_time ||= @init_time unless open?
+
+        super
+      end
+
+      def idling
+        super
+
+        # time of initial request(s) is accounted from the moment
+        # the connection is back to :idle, and ready to connect again.
+        @init_time = ::Time.now
+      end
+    end
+  end
+  register_plugin :tracing, Tracing
+end

data/lib/httpx/request.rb

@@ -91,7 +91,7 @@ module HTTPX
       raise UnsupportedSchemeError, "#{@uri}: #{@uri.scheme}: unsupported URI scheme" unless ALLOWED_URI_SCHEMES.include?(@uri.scheme)
 
       @state = :idle
-      @response = @peer_address = @informational_status = nil
+      @response = @drainer = @peer_address = @informational_status = nil
       @ping = false
       @persistent = @options.persistent
       @active_timeouts = []

data/lib/httpx/resolver/multi.rb

@@ -71,14 +71,7 @@ module HTTPX
 
     def lazy_resolve(connection)
       @resolvers.each do |resolver|
-        conn_to_resolve = @current_session.try_clone_connection(connection, @current_selector, resolver.family)
-        resolver << conn_to_resolve
-
-        next if resolver.empty?
-
-        # both the resolver and the connection it's resolving must be pineed to the session
-        @current_session.pin(conn_to_resolve, @current_selector)
-        @current_session.select_resolver(resolver, @current_selector)
+        resolver.lazy_resolve(connection)
       end
     end
 

data/lib/httpx/resolver/native.rb

@@ -529,7 +529,7 @@ module HTTPX
 
         resolve if @queries.empty? && !@connections.empty?
       when :closed
-        return unless @state == :open
+        return if @state == :closed
 
         @io.close if @io
         @start_timeout = nil

data/lib/httpx/resolver/resolver.rb

@@ -119,6 +119,11 @@ module HTTPX
       end
     end
 
+    def on_io_error(e)
+      on_error(e)
+      force_close(true)
+    end
+
     def on_error(error)
       handle_error(error)
       disconnect
@@ -138,6 +143,19 @@ module HTTPX
       true
     end
 
+    def lazy_resolve(connection)
+      return unless @current_session && @current_selector
+
+      conn_to_resolve = @current_session.try_clone_connection(connection, @current_selector, @family)
+      self << conn_to_resolve
+
+      return if empty?
+
+      # both the resolver and the connection it's resolving must be pinned to the session
+      @current_session.pin(conn_to_resolve, @current_selector)
+      @current_session.select_resolver(self, @current_selector)
+    end
+
     private
 
     def emit_resolved_connection(connection, addresses, early_resolve)
@@ -181,9 +199,10 @@ module HTTPX
     end
 
     def disconnect
-      return if closed?
-
       close
+
+      return unless closed?
+
       @current_session.deselect_resolver(self, @current_selector)
     end
   end

data/lib/httpx/resolver/system.rb

@@ -116,7 +116,9 @@ module HTTPX
       @current_session.select_resolver(self, @current_selector)
     end
 
-    def early_resolve(connection, **); end
+    def early_resolve(_, **) # rubocop:disable Naming/PredicateMethod
+      false
+    end
 
     def handle_socket_timeout(interval)
       error = HTTPX::ResolveTimeoutError.new(interval, "timed out while waiting on select")

data/lib/httpx/selector.rb

@@ -204,8 +204,7 @@ module HTTPX
       rescue IOError => e
         (Array(r) + Array(w)).each do |sel|
           # TODO: is there a way to cheaply find the IO associated with the error?
-          sel.on_error(e)
-          sel.force_close(true)
+          sel.on_io_error(e)
         end
       rescue StandardError => e
         (Array(r) + Array(w)).each do |sel|
@@ -249,8 +248,9 @@ module HTTPX
           when :rw then rw_wait(io, interval)
           end
       rescue IOError => e
-        io.on_error(e)
-        io.force_close(true)
+        io.on_io_error(e)
+
+        return
       rescue StandardError => e
         io.on_error(e)
 

data/lib/httpx/session.rb

@@ -140,9 +140,10 @@ module HTTPX
       end
       selector.deregister(connection)
 
+      # do not check-in connections only created for Happy Eyeballs
       return if cloned
 
-      return if @closing && connection.state == :closed
+      return if @closing && connection.state == :closed && !connection.used?
 
       connection.log(level: 2) { "check-in connection##{connection.object_id}(#{connection.state}) in pool##{@pool.object_id}" }
       @pool.checkin_connection(connection)
@@ -177,16 +178,15 @@ module HTTPX
 
     # returns the HTTPX::Connection through which the +request+ should be sent through.
     def find_connection(request_uri, selector, options)
-      log(level: 2) { "finding connection for #{request_uri}..." }
       if (connection = selector.find_connection(request_uri, options))
         connection.idling if connection.state == :closed
-        connection.log(level: 2) { "found connection##{connection.object_id}(#{connection.state}) in selector##{selector.object_id}" }
+        log(level: 2) { "found connection##{connection.object_id}(#{connection.state}) in selector##{selector.object_id}" }
         return connection
       end
 
       connection = @pool.checkout_connection(request_uri, options)
 
-      connection.log(level: 2) { "found connection##{connection.object_id}(#{connection.state}) in pool##{@pool.object_id}" }
+      log(level: 2) { "found connection##{connection.object_id}(#{connection.state}) in pool##{@pool.object_id}" }
 
       case connection.state
       when :idle
@@ -240,7 +240,7 @@ module HTTPX
 
       return unless response && response.finished?
 
-      log(level: 2) { "response fetched" }
+      log(level: 2) { "response##{response.object_id} fetched" }
 
       response
     end
@@ -249,6 +249,7 @@ module HTTPX
     def send_request(request, selector, options = request.options)
       error = begin
         catch(:resolve_error) do
+          log(level: 2) { "finding connection for request##{request.object_id}..." }
           connection = find_connection(request.uri, selector, options)
           connection.send(request)
         end
@@ -391,7 +392,11 @@ module HTTPX
       resolver = find_resolver_for(connection, selector)
 
       pin(connection, selector)
-      resolver.early_resolve(connection) || resolver.lazy_resolve(connection)
+      early_resolve(resolver, connection) || resolver.lazy_resolve(connection)
+    end
+
+    def early_resolve(resolver, connection)
+      resolver.early_resolve(connection)
     end
 
     def on_resolver_connection(connection, selector)

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "1.7.2"
+  VERSION = "1.7.4"
 end

data/sig/chainable.rbs

@@ -29,7 +29,7 @@ module HTTPX
               | (:proxy, ?options) -> (Plugins::sessionProxy & Plugins::httpProxy)
               | (:push_promise, ?options) -> Plugins::sessionPushPromise
               | (:retries, ?options) -> Plugins::sessionRetries
-              | (:rate_limiter, ?options) -> Session
+              | (:rate_limiter, ?options) -> Plugins::sessionRateLimiter
               | (:stream, ?options) -> Plugins::sessionStream
               | (:stream_bidi, ?options) -> Plugins::sessionStreamBidi
               | (:aws_sigv4, ?options) -> Plugins::awsSigV4Session
@@ -43,6 +43,7 @@ module HTTPX
               | (:webdav, ?options) -> Plugins::sessionWebDav
               | (:xml, ?options) -> Plugins::sessionXML
               | (:query, ?options) -> Plugins::sessionQuery
+              | (:tracing, ?options) -> Plugins::sessionTracing
               | (Symbol | Module, ?options) { (Class) -> void } -> Session
               | (Symbol | Module, ?options) -> Session
 

data/sig/connection/http1.rbs

@@ -26,6 +26,8 @@ module HTTPX
 
     def reset: () -> void
 
+    def reset_requests: () -> void
+
     def close: () -> void
 
     def empty?: () -> bool

data/sig/connection/http2.rbs

@@ -6,6 +6,7 @@ module HTTPX
     MAX_CONCURRENT_REQUESTS: Integer
 
     attr_reader streams: Hash[Request, ::HTTP2::Stream]
+
     attr_reader pending: Array[Request]
 
     @connection: HTTP2::Client
@@ -43,6 +44,8 @@ module HTTPX
 
     def timeout: () -> Numeric?
 
+    def reset_requests: () -> void
+
     private
 
     def initialize: (Buffer buffer, Options options) -> untyped
@@ -79,6 +82,8 @@ module HTTPX
 
     def on_stream_refuse: (::HTTP2::Stream stream, Request request, StandardError error) -> void
 
+    def on_stream_half_close: (::HTTP2::Stream stream, Request request) -> void
+
     def on_stream_close: (::HTTP2::Stream stream, Request request, (Symbol | StandardError)? error) -> void
 
     def on_frame: (string bytes) -> void
@@ -87,13 +92,15 @@ module HTTPX
 
     def on_close: (Integer last_frame, Symbol? error, String? payload) -> void
 
-    def on_frame_sent: (::HTTP2::frame) -> void
+    def on_frame_sent: (::HTTP2::frame frame) -> void
+
+    def frame_with_extra_info: (::HTTP2::frame frame) -> Hash[Symbol, untyped]
 
-    def on_frame_received: (::HTTP2::frame) -> void
+    def on_frame_received: (::HTTP2::frame frame) -> void
 
-    def on_altsvc: (String origin, ::HTTP2::frame) -> void
+    def on_altsvc: (String origin, ::HTTP2::frame frame) -> void
 
-    def on_promise: (::HTTP2::Stream) -> void
+    def on_promise: (::HTTP2::Stream stream) -> void
 
     def on_origin: (String) -> void
 

data/sig/connection.rbs

@@ -37,6 +37,7 @@ module HTTPX
     @read_buffer: Buffer
     @write_buffer: Buffer
     @inflight: Integer
+    @max_concurrent_requests: Integer?
     @keep_alive_timeout: Numeric?
     @timeout: Numeric?
     @current_timeout: Numeric?
@@ -119,6 +120,10 @@ module HTTPX
 
     def on_error: (HTTPX::TimeoutError | Error | StandardError error, ?Request? request) -> void
 
+    def on_io_error: (IOError error) -> void
+
+    def on_connect_error: (Exception error) -> void
+
     private
 
     def initialize: (http_uri uri, Options options) -> void
@@ -153,6 +158,8 @@ module HTTPX
 
     def handle_error: (StandardError error, ?Request? request) -> void
 
+    def force_purge: () -> void
+
     def close_sibling: () -> void
 
     def purge_after_closed: () -> void

data/sig/plugins/brotli.rbs

@@ -5,17 +5,22 @@ module HTTPX
 
       def self?.encode: (body_encoder body) -> Deflater
 
-      def self?.decode: (HTTPX::Response response, ?bytesize: Integer) -> Transcoder::_Decoder
+      def self?.decode: (HTTPX::Response response, ?bytesize: Integer | Float) -> Inflater
 
-      class Deflater < Transcoder::Deflater
+      class Error < ::HTTPX::Error
       end
 
-      module RequestBodyClassMethods
-        def initialize_deflater_body: (body_encoder body, Encoding | String encoding) -> body_encoder
+      class Deflater < Transcoder::Deflater
+        @compressor: ::Brotli::Compressor
       end
 
-      module ResponseBodyClassMethods
-        def initialize_inflater_by_encoding: (Encoding | String encoding, Response response, ?bytesize: Integer) -> (Transcoder::_Decoder | Transcoder::GZIP::Inflater)
+      class Inflater
+        @inflater: ::Brotli::Decompressor
+        @bytesize: Integer | Float
+
+        def initialize: (Integer | Float bytesize) -> void
+
+        def call: (String chunk) -> String
       end
     end
   end

data/sig/plugins/cookies/cookie.rbs

@@ -31,6 +31,8 @@ module HTTPX
       def cookie_value: () -> String
       alias to_s cookie_value
 
+      def match?: (String | cookie_attributes name_or_options) -> bool
+
       def valid_for_uri?: (http_uri uri) -> bool
 
       def self.new: (Cookie) -> instance
@@ -41,8 +43,7 @@ module HTTPX
 
       private
 
-      def initialize: (cookie_attributes) -> untyped
-                    | (_ToS, _ToS, ?cookie_attributes) -> untyped
+      def initialize: (_ToS name, _ToS value, ?cookie_attributes) -> void
 
       def acceptable_from_uri?: (uri) -> bool
 

data/sig/plugins/cookies/jar.rbs

@@ -6,11 +6,20 @@ module HTTPX
       include Enumerable[Cookie]
 
       @cookies: Array[Cookie]
+      @mtx: Thread::Mutex
 
       def parse: (String set_cookie) -> void
 
+      def get: (String | cookie_attributes name_or_options) -> Cookie?
+
+      def get_all: (String | cookie_attributes name_or_options) -> Array[Cookie]
+
+      def set: (_ToS | Cookie name, ?(cookie_attributes | _ToS) value_or_options) -> void
+
       def add: (Cookie name, ?String path) -> void
 
+      def delete: (String | Cookie | cookie_attributes name_or_options) -> void
+
       def []: (http_uri) -> Array[Cookie]
 
       def each: (?http_uri?) { (Cookie) -> void } -> void
@@ -21,6 +30,8 @@ module HTTPX
       private
 
       def initialize: (?_Each[cookie] cookies) -> untyped
+
+      def synchronize: [T] { () -> T } -> T
     end
   end
 end

data/sig/plugins/cookies.rbs

@@ -15,6 +15,8 @@ module HTTPX
 
       module InstanceMethods
         def cookies: () -> Jar
+
+        def make_jar: (*untyped) -> Jar
       end
 
       module HeadersMethods

data/sig/plugins/expect.rbs

@@ -2,14 +2,29 @@ module HTTPX
   module Plugins
     module Expect
       EXPECT_TIMEOUT: Integer
+      NOEXPECT_STORE_MUTEX: Thread::Mutex
+
+      self.@no_expect_store: Store
+      def self.no_expect_store: () -> Store
+
+      def self.extra_options: (Options) -> (Options & _ExpectOptions)
+
+      class Store
+        @store: Array[String]
+        @mutex: Thread::Mutex
+
+        def include?: (String host) -> bool
+
+        def add: (String host) -> void
+
+        def delete: (String host) -> void
+      end
 
       interface _ExpectOptions
         def expect_timeout: () -> Integer?
 
         def expect_threshold_size: () -> Integer?
       end
-
-      def self.extra_options: (Options) -> (Options & _ExpectOptions)
     end
   end
 end

data/sig/plugins/proxy/socks4.rbs

@@ -5,6 +5,10 @@ module HTTPX
   module Plugins
     module Proxy
       module Socks4
+        VERSION: Integer
+        CONNECT: Integer
+        GRANTED: Integer
+        PROTOCOLS: Array[String]
 
         module ConnectionMethods
           def __socks4_proxy_connect: () ->  void

data/sig/plugins/rate_limiter.rbs

@@ -5,11 +5,11 @@ module HTTPX
 
       def self.load_dependencies: (singleton(Session)) -> void
 
-      def self.retry_after_rate_limit: (untyped, response) -> Numeric?
-
       module InstanceMethods
         def rate_limit_error?: (response response) -> bool
       end
     end
+
+    type sessionRateLimiter = Session & RateLimiter::InstanceMethods
   end
 end

data/sig/plugins/response_cache.rbs

@@ -51,7 +51,7 @@ module HTTPX
       module ResponseMethods
         attr_writer original_request: cacheRequest
 
-        @cache: bool
+        @cached: bool
         @cache_control: Array[String]?
         @vary: Array[String]?
         @date: Time?
@@ -66,9 +66,9 @@ module HTTPX
 
         def fresh?: () -> bool
 
-        def cache_control: () -> Array[String]?
+         %a{pure} def cache_control: () -> Array[String]?
 
-        def vary: () -> Array[String]?
+         %a{pure} def vary: () -> Array[String]?
 
         private
 

data/sig/plugins/retries.rbs

@@ -8,16 +8,16 @@ module HTTPX
       DEFAULT_JITTER: ^(Numeric) -> Numeric
       BACKOFF_ALGORITHMS: Array[Symbol]
 
-      def self?.retry_after_polynomial_backoff: (retriesRequest request, response response) -> Numeric
+      def self?.retry_after_polynomial_backoff: (retriesRequest request, retriesResponse response) -> Numeric
 
-      def self?.retry_after_exponential_backoff: (retriesRequest request, response response) -> Numeric
+      def self?.retry_after_exponential_backoff: (retriesRequest request, retriesResponse response) -> Numeric
 
       interface _RetryCallback
-        def call: (response response) -> bool?
+        def call: (retriesResponse response) -> bool?
       end
 
       interface _RetriesOptions
-        def retry_after: () -> (^(retriesRequest request, response response) -> Numeric | Numeric)?
+        def retry_after: () -> (^(retriesRequest request, retriesResponse response) -> Numeric | Numeric)?
 
         def retry_jitter: () -> ^(Numeric jitter) -> Numeric
 
@@ -35,17 +35,19 @@ module HTTPX
 
         private
 
-        def fetch_response: (retriesRequest request, Selector selector, retriesOptions options) -> (retriesResponse | ErrorResponse)?
+        def fetch_response: (retriesRequest request, Selector selector, retriesOptions options) -> retriesResponse?
 
-        def retryable_request?: (retriesRequest request, response response, retriesOptions options) -> boolish
+        def retryable_request?: (retriesRequest request, retriesResponse response, retriesOptions options) -> boolish
 
-        def retryable_response?: (response response, retriesOptions options) -> boolish
+        def retryable_response?: (retriesResponse response, retriesOptions options) -> boolish
 
         def retryable_error?: (_Exception error, Options options) -> boolish
 
-        def try_partial_retry: (retriesRequest request, (retriesResponse | ErrorResponse) response) -> void
+        def try_partial_retry: (retriesRequest request, retriesResponse response) -> void
 
-        def prepare_to_retry: (Request & RequestMethods request, response response) -> void
+        def prepare_to_retry: (Request & RequestMethods request, retriesResponse response) -> void
+
+        def when_to_retry: (Request & RequestMethods request, retriesResponse response, retriesOptions options) -> void
       end
 
       module RequestMethods
@@ -53,20 +55,22 @@ module HTTPX
 
         attr_accessor retries: Integer
 
-        attr_writer partial_response: Response?
+        attr_writer partial_response: retriesResponse?
 
-        def response=: (retriesResponse | ErrorResponse response) -> void
+        def response=: (retriesResponse response) -> void
       end
 
       module ResponseMethods
-        def from_partial_response: (Response response) -> void
+        def from_partial_response: (retriesHTTPResponse response) -> void
       end
 
       type retriesOptions = Options & _RetriesOptions
 
       type retriesRequest = Request & RequestMethods
 
-      type retriesResponse = Response & ResponseMethods
+      type retriesHTTPResponse = Response & ResponseMethods
+
+      type retriesResponse = retriesHTTPResponse | ErrorResponse
     end
 
     type sessionRetries = Session & Retries::InstanceMethods