httpx 1.7.4 → 1.7.6

data/lib/httpx/io/unix.rb

@@ -14,16 +14,20 @@ module HTTPX
       @state = :idle
       @options = options
       @fallback_protocol = @options.fallback_protocol
-      if @options.io
-        @io = case @options.io
-              when Hash
-                @options.io[origin.authority]
-              else
-                @options.io
-        end
-        raise Error, "Given IO objects do not match the request authority" unless @io
+      if (io = @options.io)
+        io =
+          case io
+          when Hash
+            io[origin.authority]
+          else
+            io
+          end
+        raise Error, "Given IO objects do not match the request authority" unless io
+
+        # @type var io: UNIXSocket
 
-        @path = @io.path
+        _, @path = io.addr
+        @io = io
         @keep_open = true
         @state = :connected
       elsif path

data/lib/httpx/options.rb

@@ -202,16 +202,19 @@ module HTTPX
 
     REQUEST_BODY_IVARS = %i[@headers].freeze
 
-    def ==(other)
-      super || options_equals?(other)
-    end
+    # checks whether +other+ matches the same connection-level options
+    def connection_options_match?(other, ignore_ivars = nil)
+      return true if self == other
 
-    # checks whether +other+ is equal by comparing the session options
-    def options_equals?(other, ignore_ivars = REQUEST_BODY_IVARS)
       # headers and other request options do not play a role, as they are
       # relevant only for the request.
-      ivars = instance_variables - ignore_ivars
-      other_ivars = other.instance_variables - ignore_ivars
+      ivars = instance_variables
+      ivars.reject! { |iv| REQUEST_BODY_IVARS.include?(iv) }
+      ivars.reject! { |iv| ignore_ivars.include?(iv) } if ignore_ivars
+
+      other_ivars = other.instance_variables
+      other_ivars.reject! { |iv| REQUEST_BODY_IVARS.include?(iv) }
+      other_ivars.reject! { |iv| ignore_ivars.include?(iv) } if ignore_ivars
 
       return false if ivars.size != other_ivars.size
 
@@ -222,6 +225,19 @@ module HTTPX
       end
     end
 
+    RESOLVER_IVARS = %i[
+      @resolver_class @resolver_cache @resolver_options
+      @resolver_native_class @resolver_system_class @resolver_https_class
+    ].freeze
+
+    # checks whether +other+ matches the same resolver-level options
+    def resolver_options_match?(other)
+      self == other ||
+        RESOLVER_IVARS.all? do |ivar|
+          instance_variable_get(ivar) == other.instance_variable_get(ivar)
+        end
+    end
+
     # returns a HTTPX::Options instance resulting of the merging of +other+ with self.
     # it may return self if +other+ is self or equal to self.
     def merge(other)

data/lib/httpx/parser/http1.rb

@@ -14,6 +14,8 @@ module HTTPX
         @state = :idle
         @buffer = "".b
         @headers = {}
+        @content_length = nil
+        @_has_trailers = @upgrade = false
       end
 
       def <<(chunk)
@@ -25,7 +27,8 @@ module HTTPX
         @state = :idle
         @headers = {}
         @content_length = nil
-        @_has_trailers = nil
+        @_has_trailers = @upgrade = false
+        @buffer = @buffer.to_s
         @buffer.clear
       end
 
@@ -34,7 +37,7 @@ module HTTPX
       end
 
       def upgrade_data
-        @buffer
+        @buffer.to_s
       end
 
       private
@@ -55,6 +58,7 @@ module HTTPX
       end
 
       def parse_headline
+        #: @type ivar @buffer: String
         idx = @buffer.index("\n")
         return unless idx
 
@@ -75,6 +79,8 @@ module HTTPX
         headers = @headers
         buffer = @buffer
 
+        #: @type var buffer: String
+
         while (idx = buffer.index("\n"))
           # @type var line: String
           line = buffer.byteslice(0..idx)
@@ -118,17 +124,20 @@ module HTTPX
 
       def parse_data
         if @buffer.respond_to?(:each)
+          # @type ivar @buffer: Transcoder::Chunker::Decoder
           @buffer.each do |chunk|
             @observer.on_data(chunk)
           end
         elsif @content_length
-          # @type var data: String
+          # @type ivar @buffer: String
           data = @buffer.byteslice(0, @content_length)
+          # @type var data: String
           @buffer = @buffer.byteslice(@content_length..-1) || "".b
           @content_length -= data.bytesize
           @observer.on_data(data)
           data.clear
         else
+          # @type ivar @buffer: String
           @observer.on_data(@buffer)
           @buffer.clear
         end
@@ -152,7 +161,7 @@ module HTTPX
             tr_encoding.split(/ *, */).each do |encoding|
               case encoding
               when "chunked"
-                @buffer = Transcoder::Chunker::Decoder.new(@buffer, @_has_trailers)
+                @buffer = Transcoder::Chunker::Decoder.new(@buffer.to_s, @_has_trailers)
               end
             end
           end
@@ -165,6 +174,7 @@ module HTTPX
         if @content_length
           @content_length <= 0
         elsif @buffer.respond_to?(:finished?)
+          # @type ivar @buffer: Transcoder::Chunker::Decoder
           @buffer.finished?
         else
           false

data/lib/httpx/plugins/auth.rb

@@ -44,6 +44,7 @@ module HTTPX
           super
 
           @auth_header_value = nil
+          @auth_header_value_mtx = Thread::Mutex.new
           @skip_auth_header_value = false
         end
 
@@ -63,7 +64,9 @@ module HTTPX
         end
 
         def reset_auth_header_value!
-          @auth_header_value = nil
+          @auth_header_value_mtx.synchronize do
+            @auth_header_value = nil
+          end
         end
 
         private
@@ -71,9 +74,11 @@ module HTTPX
         def send_request(request, *)
           return super if @skip_auth_header_value || request.authorized?
 
-          @auth_header_value ||= generate_auth_token
+          auth_header_value = @auth_header_value_mtx.synchronize do
+            @auth_header_value ||= generate_auth_token
+          end
 
-          request.authorize(@auth_header_value) if @auth_header_value
+          request.authorize(auth_header_value) if auth_header_value
 
           super
         end
@@ -92,9 +97,11 @@ module HTTPX
       end
 
       module RequestMethods
+        attr_reader :auth_token_value
+
         def initialize(*)
           super
-          @auth_token_value = nil
+          @auth_token_value = @auth_header_value = nil
         end
 
         def authorized?
@@ -102,19 +109,20 @@ module HTTPX
         end
 
         def unauthorize!
-          return unless (auth_value = @auth_token_value)
+          return unless (auth_value = @auth_header_value)
 
           @headers.get("authorization").delete(auth_value)
 
-          @auth_token_value = nil
+          @auth_token_value = @auth_header_value = nil
         end
 
         def authorize(auth_value)
+          @auth_header_value = auth_value
           if (auth_type = @options.auth_header_type)
-            auth_value = "#{auth_type} #{auth_value}"
+            @auth_header_value = "#{auth_type} #{@auth_header_value}"
           end
 
-          @headers.add("authorization", auth_value)
+          @headers.add("authorization", @auth_header_value)
 
           @auth_token_value = auth_value
         end
@@ -138,8 +146,14 @@ module HTTPX
             return unless auth_error?(response, request.options) ||
                           (@options.generate_auth_value_on_retry && @options.generate_auth_value_on_retry.call(response))
 
+            # regenerate token before retry, but only if it's the first request from batch failing.
+            # otherwise, it means that the first request already passed here, so this request should
+            # use whatever was generated for it.
+            @auth_header_value_mtx.synchronize do
+              @auth_header_value = generate_auth_token if request.auth_token_value == @auth_header_value
+            end
+
             request.unauthorize!
-            @auth_header_value = generate_auth_token
           end
 
           def auth_error?(response, options)

data/lib/httpx/plugins/expect.rb

@@ -70,6 +70,9 @@ module HTTPX
       module RequestMethods
         def initialize(*)
           super
+
+          @informational_status = nil
+
           return if @body.empty?
 
           threshold = @options.expect_threshold_size

data/lib/httpx/plugins/retries.rb

@@ -148,7 +148,7 @@ module HTTPX
             log { "failed to get response, #{request.retries} tries to go..." }
             prepare_to_retry(request, response)
 
-            if (retry_after = when_to_retry(request, response, options))
+            if (retry_after = when_to_retry(request, response, options)) && retry_after.positive?
               # apply jitter
               if (jitter = request.options.retry_jitter)
                 retry_after = jitter.call(retry_after)

data/lib/httpx/plugins/tracing.rb

@@ -89,7 +89,7 @@ module HTTPX::Plugins
         on(:headers) do
           # the usual request init time (when not including the connection handshake)
           # should be the time the request is buffered the first time.
-          @init_time ||= ::Time.now
+          @init_time ||= ::Time.now.utc
 
           tracer.start(self)
         end
@@ -102,7 +102,7 @@ module HTTPX::Plugins
         # Example is the :ssrf_filter plugin, which raises an error on
         # initialize if the host is an IP which matches against the known set.
         # in such cases, we'll just set here right here.
-        @init_time ||= ::Time.now
+        @init_time ||= ::Time.now.utc
 
         super
       end
@@ -113,7 +113,7 @@ module HTTPX::Plugins
       def initialize(*)
         super
 
-        @init_time = ::Time.now
+        @init_time = ::Time.now.utc
       end
 
       def send(request)
@@ -129,7 +129,7 @@ module HTTPX::Plugins
 
         # time of initial request(s) is accounted from the moment
         # the connection is back to :idle, and ready to connect again.
-        @init_time = ::Time.now
+        @init_time = ::Time.now.utc
       end
     end
   end

data/lib/httpx/pool.rb

@@ -122,12 +122,6 @@ module HTTPX
 
         @max_connections_cond.signal
         @origin_conds[connection.origin.to_s].signal
-
-        # Observed situations where a session handling multiple requests in a loop
-        # across multiple threads checks the same connection in and out, while another
-        # thread which is waiting on the same connection never gets the chance to pick
-        # it up, because ruby's thread scheduler never switched on to it in the process.
-        Thread.pass
       end
     end
 
@@ -153,16 +147,20 @@ module HTTPX
         resolvers = @resolvers[resolver_type]
 
         idx = resolvers.find_index do |res|
-          res.options == options
+          res.options.resolver_options_match?(options)
         end
         resolvers.delete_at(idx) if idx
       end || checkout_new_resolver(resolver_type, options)
     end
 
     def checkin_resolver(resolver)
-      resolver_class = resolver.class
+      if resolver.is_a?(Resolver::Multi)
+        resolver_class = resolver.resolvers.first.class
+      else
+        resolver_class = resolver.class
 
-      resolver = resolver.multi
+        resolver = resolver.multi
+      end
 
       # a multi requires all sub-resolvers being closed in order to be
       # correctly checked back in.

data/lib/httpx/request.rb

@@ -42,6 +42,9 @@ module HTTPX
     # The IP address from the peer server.
     attr_accessor :peer_address
 
+    # the connection the request is currently being sent to (none if before or after transaction)
+    attr_writer :connection
+
     attr_writer :persistent
 
     attr_reader :active_timeouts
@@ -91,7 +94,7 @@ module HTTPX
       raise UnsupportedSchemeError, "#{@uri}: #{@uri.scheme}: unsupported URI scheme" unless ALLOWED_URI_SCHEMES.include?(@uri.scheme)
 
       @state = :idle
-      @response = @drainer = @peer_address = @informational_status = nil
+      @connection = @response = @drainer = @peer_address = @informational_status = nil
       @ping = false
       @persistent = @options.persistent
       @active_timeouts = []
@@ -274,8 +277,7 @@ module HTTPX
       when :idle
         @body.rewind
         @ping = false
-        @response = nil
-        @drainer = nil
+        @response = @drainer = nil
         @active_timeouts.clear
       when :headers
         return unless @state == :idle
@@ -321,6 +323,16 @@ module HTTPX
         callbacks(event).delete(clb)
       end
     end
+
+    def handle_error(error)
+      if (connection = @connection)
+        connection.on_error(error, self)
+      else
+        response = ErrorResponse.new(self, error)
+        self.response = response
+        emit(:response, response)
+      end
+    end
   end
 end
 

data/lib/httpx/resolver/native.rb

@@ -66,7 +66,7 @@ module HTTPX
     end
 
     def closed?
-      @state == :closed
+      @state == :idle || @state == :closed
     end
 
     def to_io

data/lib/httpx/response.rb

@@ -112,6 +112,7 @@ module HTTPX
     def finish!
       @finished = true
       @headers.freeze
+      @request.connection = nil
     end
 
     # returns whether the response contains body payload.
@@ -282,6 +283,7 @@ module HTTPX
       @error = error
       @options = request.options
       log_exception(@error)
+      finish!
     end
 
     # returns the exception full message.
@@ -299,7 +301,9 @@ module HTTPX
       true
     end
 
-    def finish!; end
+    def finish!
+      @request.connection = nil
+    end
 
     # raises the wrapped exception.
     def raise_for_status

data/lib/httpx/selector.rb

@@ -29,7 +29,7 @@ module HTTPX
 
     def_delegator :@timers, :after
 
-    def_delegator :@selectables, :empty?
+    def_delegator :@selectables, :each
 
     def initialize
       @timers = Timers.new
@@ -37,8 +37,8 @@ module HTTPX
       @is_timer_interval = false
     end
 
-    def each(&blk)
-      @selectables.each(&blk)
+    def empty?
+      @selectables.empty? && @timers.empty?
     end
 
     def next_tick
@@ -81,7 +81,8 @@ module HTTPX
 
     def find_resolver(options)
       res = @selectables.find do |c|
-        c.is_a?(Resolver::Resolver) && options == c.options
+        c.is_a?(Resolver::Resolver) &&
+          options.resolver_options_match?(c.options)
       end
 
       res.multi if res
@@ -148,16 +149,18 @@ module HTTPX
 
         next(is_closed) if is_closed
 
-        io.log(level: 2) do
-          "[#{io.state}] registering in selector##{object_id} for select (#{interests})#{" for #{interval} seconds" unless interval.nil?}"
-        end
+        if interests
+          io.log(level: 2) do
+            "[#{io.state}] registering in selector##{object_id} for select (#{interests})#{" for #{interval} seconds" unless interval.nil?}"
+          end
 
-        if READABLE.include?(interests)
-          r = r.nil? ? io : (Array(r) << io)
-        end
+          if READABLE.include?(interests)
+            r = r.nil? ? io : (Array(r) << io)
+          end
 
-        if WRITABLE.include?(interests)
-          w = w.nil? ? io : (Array(w) << io)
+          if WRITABLE.include?(interests)
+            w = w.nil? ? io : (Array(w) << io)
+          end
         end
 
         is_closed

data/lib/httpx/session.rb

@@ -325,53 +325,34 @@ module HTTPX
 
     # returns the array of HTTPX::Response objects corresponding to the array of HTTPX::Request +requests+.
     def receive_requests(requests, selector)
-      responses = [] # : Array[response]
-
-      # guarantee ordered responses
-      loop do
-        request = requests.first
-
-        return responses unless request
-
-        catch(:coalesced) { selector.next_tick } until (response = fetch_response(request, selector, request.options))
-        request.complete!(response)
-
-        responses << response
-        requests.shift
+      waiting = 0
+      responses = requests.map do |request|
+        fetch_response(request, selector, request.options).tap do |response|
+          waiting += 1 if response.nil?
+        end
+      end
 
-        break if requests.empty?
+      until waiting.zero? || selector.empty?
+        # loop on selector until at least one response has been received.
+        catch(:coalesced) { selector.next_tick }
 
-        next unless selector.empty?
+        responses.each_with_index do |response, idx|
+          next unless response.nil?
 
-        # in some cases, the pool of connections might have been drained because there was some
-        # handshake error, and the error responses have already been emitted, but there was no
-        # opportunity to traverse the requests, hence we're returning only a fraction of the errors
-        # we were supposed to. This effectively fetches the existing responses and return them.
-        exit_from_loop = true
+          request = requests[idx]
 
-        requests_to_remove = [] # : Array[Request]
+          response = fetch_response(request, selector, request.options)
 
-        requests.each do |req|
-          response = fetch_response(req, selector, request.options)
+          next unless response
 
-          if exit_from_loop && response
-            req.complete!(response)
-            responses << response
-            requests_to_remove << req
-          else
-            # fetch_response may resend requests. when that happens, we need to go back to the initial
-            # loop and process the selector. we still do a pass-through on the remainder of requests, so
-            # that every request that need to be resent, is resent.
-            exit_from_loop = false
-
-            raise Error, "something went wrong, responses not found and requests not resent" if selector.empty?
-          end
+          request.complete!(response)
+          responses[idx] = response
+          waiting -= 1
         end
+      end
 
-        break if exit_from_loop
+      raise Error, "something went wrong, responses not found and requests not resent" unless waiting.zero?
 
-        requests -= requests_to_remove
-      end
       responses
     end
 
@@ -392,7 +373,11 @@ module HTTPX
       resolver = find_resolver_for(connection, selector)
 
       pin(connection, selector)
-      early_resolve(resolver, connection) || resolver.lazy_resolve(connection)
+      if early_resolve(resolver, connection)
+        @pool.checkin_resolver(resolver)
+      else
+        resolver.lazy_resolve(connection)
+      end
     end
 
     def early_resolve(resolver, connection)

data/lib/httpx/timers.rb

@@ -6,6 +6,10 @@ module HTTPX
       @intervals = []
     end
 
+    def empty?
+      @intervals.empty?
+    end
+
     def after(interval_in_secs, cb = nil, &blk)
       callback = cb || blk
 

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "1.7.4"
+  VERSION = "1.7.6"
 end

data/sig/altsvc.rbs

@@ -3,6 +3,8 @@ module HTTPX
     module ConnectionMixin
       H2_ALTSVC_SCHEMES: Array[String]
 
+      ALTSVC_IGNORE_IVARS: Array[Symbol]
+
       def send: (Request request) -> void
 
       def match?: (http_uri uri, Options options) -> bool

data/sig/connection/http1.rbs

@@ -52,7 +52,7 @@ module HTTPX
 
     def on_complete: () -> void
 
-    def dispatch: () -> void
+    def dispatch: (Request request) -> void
 
     def ping: () -> void
 

data/sig/connection.rbs

@@ -50,6 +50,7 @@ module HTTPX
     @altsvc_connection: instance?
     @sibling: instance?
     @main_sibling: bool
+    @no_more_requests_counter: Integer
 
 
     def addresses: () -> Array[Resolver::Entry]?
@@ -156,6 +157,12 @@ module HTTPX
 
     def build_socket: (?Array[Resolver::Entry]? addrs) -> (TCP | SSL | UNIX)
 
+    def ping: () -> void
+
+    def pong: () -> void
+
+    def no_more_requests_loop_check: () -> void
+
     def handle_error: (StandardError error, ?Request? request) -> void
 
     def force_purge: () -> void
@@ -172,9 +179,9 @@ module HTTPX
 
     def set_request_request_timeout: (Request request) -> void
 
-    def write_timeout_callback: (Request request, Numeric write_timeout) -> void
+    def write_timeout_callback: (Request request, Numeric timeout) -> void
 
-    def read_timeout_callback: (Request request, Numeric read_timeout, ?singleton(RequestTimeoutError) error_type) -> void
+    def read_timeout_callback: (Request request, Numeric timeout, ?singleton(RequestTimeoutError) error_type) -> void
 
     def set_request_timeout: (Symbol label, Request request, Numeric timeout, Symbol start_event, Symbol | Array[Symbol] finish_events) { () -> void } -> void
 

data/sig/io/ssl.rbs

@@ -8,6 +8,7 @@ module HTTPX
 
     @ctx: OpenSSL::SSL::SSLContext
     @verify_hostname: bool
+    @sni_hostname: String
 
     attr_writer ssl_session: OpenSSL::SSL::Session?
 

data/sig/io/tcp.rbs

@@ -46,7 +46,7 @@ module HTTPX
 
     public
 
-    def read: (Integer size, ?(Buffer | String) buffer) -> (0 | nil | untyped)
+    def read: (Integer size, Buffer | String buffer) -> (0 | nil | untyped)
 
     def write: (Buffer buffer) -> Integer?
 
@@ -67,6 +67,6 @@ module HTTPX
 
     def do_transition: (Symbol nextstate) -> void
 
-    def log_transition_state: (Symbol nextstate) -> void
+    def log_transition_state: (Symbol nextstate) -> String
   end
 end