httpx 1.6.3 → 1.7.0

data/lib/httpx/plugins/stream_bidi.rb

@@ -26,6 +26,8 @@ module HTTPX
           class_eval(<<-METH, __FILE__, __LINE__ + 1)
             # lock.aware version of +#{lock_meth}+
             def #{lock_meth}(*)                # def close(*)
+              return super unless @options.stream
+
               return super if @lock.owned?
 
               # small race condition between
@@ -43,6 +45,8 @@ module HTTPX
           class_eval(<<-METH, __FILE__, __LINE__ + 1)
             # lock.aware version of +#{lock_meth}+
             private def #{lock_meth}(*)                # private def join_headers(*)
+              return super unless @options.stream
+
               return super if @lock.owned?
 
               # small race condition between
@@ -55,6 +59,8 @@ module HTTPX
         end
 
         def handle_stream(stream, request)
+          return super unless @options.stream
+
           request.on(:body) do
             next unless request.headers_sent
 
@@ -67,6 +73,8 @@ module HTTPX
 
         # when there ain't more chunks, it makes the buffer as full.
         def send_chunk(request, stream, chunk, next_chunk)
+          return super unless @options.stream
+
           super
 
           return if next_chunk
@@ -77,39 +85,60 @@ module HTTPX
 
         # sets end-stream flag when the request is closed.
         def end_stream?(request, next_chunk)
+          return super unless @options.stream
+
           request.closed? && next_chunk.nil?
         end
       end
 
-      # BidiBuffer is a Buffer which can be receive data from threads othr
-      # than the thread of the corresponding Connection/Session.
+      # BidiBuffer is a thread-safe Buffer which can receive data from any thread.
+      #
+      # It uses a dual-buffer strategy with mutex protection:
+      # - +@buffer+ is the main buffer, protected by +@buffer_mutex+
+      # - +@oob_buffer+ receives data when +@buffer_mutex+ is contended
       #
-      # It synchronizes access to a secondary internal +@oob_buffer+, which periodically
-      # is reconciled to the main internal +@buffer+.
+      # This allows non-blocking writes from any thread while maintaining thread safety.
       class BidiBuffer < Buffer
         def initialize(*)
           super
-          @parent_thread = Thread.current
+          @buffer_mutex = Thread::Mutex.new
           @oob_mutex = Thread::Mutex.new
           @oob_buffer = "".b
         end
 
-        # buffers the +chunk+ to be sent
+        # buffers the +chunk+ to be sent (thread-safe, non-blocking)
         def <<(chunk)
-          return super if Thread.current == @parent_thread
-
-          @oob_mutex.synchronize { @oob_buffer << chunk }
+          if @buffer_mutex.try_lock
+            begin
+              super
+            ensure
+              @buffer_mutex.unlock
+            end
+          else
+            # another thread holds the lock, use OOB buffer to avoid blocking
+            @oob_mutex.synchronize { @oob_buffer << chunk }
+          end
         end
 
-        # reconciles the main and secondary buffer (which receives data from other threads).
+        # reconciles the main and secondary buffer (thread-safe, callable from any thread).
         def rebuffer
-          raise Error, "can only rebuffer while waiting on a response" unless Thread.current == @parent_thread
+          @buffer_mutex.synchronize do
+            @oob_mutex.synchronize do
+              return if @oob_buffer.empty?
 
-          @oob_mutex.synchronize do
-            @buffer << @oob_buffer
-            @oob_buffer.clear
+              @buffer << @oob_buffer
+              @oob_buffer.clear
+            end
           end
         end
+
+        Buffer.instance_methods - Object.instance_methods - %i[<<].each do |meth|
+          class_eval(<<-MOD, __FILE__, __LINE__ + 1)
+            def #{meth}(*) # def empty?
+              @buffer_mutex.synchronize { super }
+            end
+          MOD
+        end
       end
 
       # Proxy to wake up the session main loop when one
@@ -156,7 +185,13 @@ module HTTPX
 
         def timeout; end
 
+        def inflight?
+          !@closed
+        end
+
         def terminate
+          return if @closed
+
           @pipe_write.close
           @pipe_read.close
           @closed = true
@@ -190,16 +225,20 @@ module HTTPX
         def close(selector = Selector.new)
           @signal.terminate
           selector.deregister(@signal)
-          super(selector)
+          super
         end
 
         def select_connection(connection, selector)
+          return super unless connection.options.stream
+
           super
           selector.register(@signal)
           connection.signal = @signal
         end
 
         def deselect_connection(connection, *)
+          return super unless connection.options.stream
+
           super
 
           connection.signal = nil
@@ -219,10 +258,14 @@ module HTTPX
         end
 
         def closed?
+          return super unless @options.stream
+
           @closed
         end
 
         def can_buffer?
+          return super unless @options.stream
+
           super && @state != :waiting_for_chunk
         end
 
@@ -230,6 +273,8 @@ module HTTPX
         # +:waiting_for_chunk+ state, which the request transitions to once payload
         # is buffered.
         def transition(nextstate)
+          return super unless @options.stream
+
           headers_sent = @headers_sent
 
           case nextstate
@@ -264,6 +309,8 @@ module HTTPX
         end
 
         def close
+          return super unless @options.stream
+
           @mutex.synchronize do
             return if @closed
 
@@ -278,10 +325,12 @@ module HTTPX
       module RequestBodyMethods
         def initialize(*, **)
           super
-          @headers.delete("content-length")
+          @headers.delete("content-length") if @options.stream
         end
 
         def empty?
+          return super unless @options.stream
+
           false
         end
       end
@@ -293,18 +342,23 @@ module HTTPX
 
         def initialize(*)
           super
+
+          return unless @options.stream
+
           @write_buffer = BidiBuffer.new(@options.buffer_size)
         end
 
         # rebuffers the +@write_buffer+ before calculating interests.
         def interests
+          return super unless @options.stream
+
           @write_buffer.rebuffer
 
           super
         end
 
         def call
-          return super unless (error = @signal.error)
+          return super unless @options.stream && (error = @signal.error)
 
           on_error(error)
         end
@@ -312,6 +366,8 @@ module HTTPX
         private
 
         def set_parser_callbacks(parser)
+          return super unless @options.stream
+
           super
           parser.on(:flush_buffer) do
             @signal.wakeup if @signal

data/lib/httpx/pool.rb

@@ -122,6 +122,12 @@ module HTTPX
 
         @max_connections_cond.signal
         @origin_conds[connection.origin.to_s].signal
+
+        # Observed situations where a session handling multiple requests in a loop
+        # across multiple threads checks the same connection in and out, while another
+        # thread which is waiting on the same connection never gets the chance to pick
+        # it up, because ruby's thread scheduler never switched on to it in the process.
+        Thread.pass
       end
     end
 
@@ -193,15 +199,19 @@ module HTTPX
     end
 
     def checkout_new_connection(uri, options)
-      options.connection_class.new(uri, options)
+      connection = options.connection_class.new(uri, options)
+      connection.log(level: 2) { "created connection##{connection.object_id} in pool##{object_id}" }
+      connection
     end
 
     def checkout_new_resolver(resolver_type, options)
-      if resolver_type.multi?
+      resolver = if resolver_type.multi?
         Resolver::Multi.new(resolver_type, options)
       else
         resolver_type.new(options)
       end
+      resolver.log(level: 2) { "created resolver##{resolver.object_id} in pool##{object_id}" }
+      resolver
     end
 
     # drops and returns the +connection+ from the connection pool; if +connection+ is <tt>nil</tt> (default),

data/lib/httpx/request.rb

@@ -10,6 +10,7 @@ module HTTPX
     extend Forwardable
     include Loggable
     include Callbacks
+
     using URIExtensions
 
     ALLOWED_URI_SCHEMES = %w[https http].freeze
@@ -90,12 +91,20 @@ module HTTPX
       raise UnsupportedSchemeError, "#{@uri}: #{@uri.scheme}: unsupported URI scheme" unless ALLOWED_URI_SCHEMES.include?(@uri.scheme)
 
       @state = :idle
-      @response = @peer_address = @context = @informational_status = nil
+      @response = @peer_address = @informational_status = nil
       @ping = false
       @persistent = @options.persistent
       @active_timeouts = []
     end
 
+    # dupped initialization
+    def initialize_dup(orig)
+      super
+      @uri = orig.instance_variable_get(:@uri).dup
+      @headers = orig.instance_variable_get(:@headers).dup
+      @body = orig.instance_variable_get(:@body).dup
+    end
+
     def complete!(response = @response)
       emit(:complete, response)
     end

data/lib/httpx/resolver/https.rb

@@ -12,6 +12,7 @@ module HTTPX
   #
   class Resolver::HTTPS < Resolver::Resolver
     extend Forwardable
+
     using URIExtensions
 
     module DNSExtensions
@@ -38,10 +39,12 @@ module HTTPX
       @resolver_options = DEFAULTS.merge(@options.resolver_options)
       @queries = {}
       @requests = {}
+      @_timeouts = Array(@resolver_options[:timeouts])
+      @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
       @uri = URI(@resolver_options[:uri])
-      @uri_addresses = nil
+      @name = @uri_addresses = nil
       @resolver = Resolv::DNS.new
-      @resolver.timeouts = @resolver_options.fetch(:timeouts, Resolver::RESOLVE_TIMEOUT)
+      @resolver.timeouts = @_timeouts.empty? ? Resolver::RESOLVE_TIMEOUT : @_timeouts
       @resolver.lazy_initialize
     end
 
@@ -96,21 +99,26 @@ module HTTPX
       else
         @queries[hostname] = connection
       end
+
+      @name = hostname
+
       log { "resolver #{FAMILY_TYPES[@record_type]}: query for #{hostname}" }
 
-      begin
-        request = build_request(hostname)
-        request.on(:response, &method(:on_response).curry(2)[request])
-        request.on(:promise, &method(:on_promise))
-        @requests[request] = hostname
-        resolver_connection.send(request)
-        @connections << connection
-      rescue ResolveError, Resolv::DNS::EncodeError => e
-        reset_hostname(hostname)
-        throw(:resolve_error, e) if connection.pending.empty?
-        emit_resolve_error(connection, connection.peer.host, e)
-        close_or_resolve
-      end
+      send_request(hostname, connection)
+    end
+
+    def send_request(hostname, connection)
+      request = build_request(hostname)
+      request.on(:response, &method(:on_response).curry(2)[request])
+      request.on(:promise, &method(:on_promise))
+      @requests[request] = hostname
+      resolver_connection.send(request)
+      @connections << connection
+    rescue ResolveError, Resolv::DNS::EncodeError => e
+      reset_hostname(hostname)
+      throw(:resolve_error, e) if connection.pending.empty?
+      emit_resolve_error(connection, connection.peer.host, e)
+      close_or_resolve
     end
 
     def on_response(request, response)
@@ -122,6 +130,18 @@ module HTTPX
       close_or_resolve
     else
       # @type var response: HTTPX::Response
+      if response.status.between?(300, 399) && response.headers.key?("location")
+        hostname = @requests[request]
+        connection = @queries[hostname]
+        location_uri = URI(response.headers["location"])
+        location_uri = response.uri.merge(location_uri) if location_uri.relative?
+
+        # we assume that the DNS server URI changed permanently and move on
+        @uri = location_uri
+        send_request(hostname, connection)
+        return
+      end
+
       parse(request, response)
     ensure
       @requests.delete(request)
@@ -133,6 +153,10 @@ module HTTPX
     end
 
     def parse(request, response)
+      hostname = @name
+
+      @name = nil
+
       code, result = decode_response_body(response)
 
       case code
@@ -151,6 +175,23 @@ module HTTPX
         end
 
         resolve
+      when :retriable_error
+        timeouts = @timeouts[hostname]
+
+        unless timeouts.empty?
+          log { "resolver #{FAMILY_TYPES[@record_type]}: failed, but will retry..." }
+
+          connection = @queries[hostname]
+
+          resolve(connection, hostname)
+          return
+        end
+
+        host = @requests.delete(request)
+        connection = reset_hostname(host)
+
+        emit_resolve_error(connection)
+        close_or_resolve
       when :dns_error
         host = @requests.delete(request)
         connection = reset_hostname(host)
@@ -217,15 +258,18 @@ module HTTPX
       uri = @uri.dup
       rklass = @options.request_class
       payload = Resolver.encode_dns_query(hostname, type: @record_type)
+      timeouts = @timeouts[hostname]
+      request_timeout = timeouts.shift
+      options = @options.merge(timeout: { request_timeout: request_timeout })
 
       if @resolver_options[:use_get]
         params = URI.decode_www_form(uri.query.to_s)
         params << ["type", FAMILY_TYPES[@record_type]]
         params << ["dns", Base64.urlsafe_encode64(payload, padding: false)]
         uri.query = URI.encode_www_form(params)
-        request = rklass.new("GET", uri, @options)
+        request = rklass.new("GET", uri, options)
       else
-        request = rklass.new("POST", uri, @options, body: [payload])
+        request = rklass.new("POST", uri, options, body: [payload])
         request.headers["content-type"] = "application/dns-message"
       end
       request.headers["accept"] = "application/dns-message"
@@ -243,6 +287,7 @@ module HTTPX
     end
 
     def reset_hostname(hostname, reset_candidates: true)
+      @timeouts.delete(hostname)
       connection = @queries.delete(hostname)
 
       return connection unless connection && reset_candidates
@@ -250,6 +295,8 @@ module HTTPX
       # eliminate other candidates
       candidates = @queries.select { |_, conn| connection == conn }.keys
       @queries.delete_if { |h, _| candidates.include?(h) }
+      # reset timeouts
+      @timeouts.delete_if { |h, _| candidates.include?(h) }
 
       connection
     end
@@ -259,6 +306,9 @@ module HTTPX
       @connections.shift until @connections.empty? || @connections.first.state != :closed
 
       if (@connections - @queries.values).empty?
+        # the same resolver connection may be serving different https resolvers (AAAA and A).
+        return if inflight?
+
         if should_deactivate
           deactivate
         else

data/lib/httpx/resolver/multi.rb

@@ -23,6 +23,10 @@ module HTTPX
       @errors = Hash.new { |hs, k| hs[k] = [] }
     end
 
+    def state
+      @resolvers.map(&:state).uniq.join(",")
+    end
+
     def current_selector=(s)
       @current_selector = s
       @resolvers.each { |r| r.current_selector = s }

data/lib/httpx/resolver/native.rb

@@ -9,6 +9,7 @@ module HTTPX
   #
   class Resolver::Native < Resolver::Resolver
     extend Forwardable
+
     using URIExtensions
 
     DEFAULTS = {
@@ -298,16 +299,14 @@ module HTTPX
     end
 
     def parse(buffer)
-      @timer.cancel
-
-      @timer = @name = nil
-
       code, result = Resolver.decode_dns_answer(buffer)
 
       case code
       when :ok
+        reset_query
         parse_addresses(result)
       when :no_domain_found
+        reset_query
         # Indicates no such domain was found.
         hostname, connection = @queries.first
         reset_hostname(hostname, reset_candidates: false)
@@ -324,6 +323,7 @@ module HTTPX
           close_or_resolve
         end
       when :message_truncated
+        reset_query
         # TODO: what to do if it's already tcp??
         return if @socket_type == :tcp
 
@@ -332,13 +332,29 @@ module HTTPX
         hostname, _ = @queries.first
         reset_hostname(hostname)
         transition(:closed)
+      when :retriable_error
+        if @name && @timer
+          log { "resolver #{FAMILY_TYPES[@record_type]}: failed, but will retry..." }
+          return
+        end
+        # retry now!
+        # connection = @queries[@name].shift
+        # @timer.fire
+        reset_query
+        hostname, connection = @queries.first
+        reset_hostname(hostname)
+        @connections.delete(connection)
+        ex = NativeResolveError.new(connection, connection.peer.host, "unknown DNS error (error code #{result})")
+        raise ex
       when :dns_error
+        reset_query
         hostname, connection = @queries.first
         reset_hostname(hostname)
         @connections.delete(connection)
         ex = NativeResolveError.new(connection, connection.peer.host, "unknown DNS error (error code #{result})")
         raise ex
       when :decode_error
+        reset_query
         hostname, connection = @queries.first
         reset_hostname(hostname)
         @connections.delete(connection)
@@ -529,6 +545,12 @@ module HTTPX
       on_error(e)
     end
 
+    def reset_query
+      @timer.cancel
+
+      @timer = @name = nil
+    end
+
     def reset_hostname(hostname, connection: @queries.delete(hostname), reset_candidates: true)
       @timeouts.delete(hostname)
 

data/lib/httpx/resolver/resolver.rb

@@ -75,7 +75,7 @@ module HTTPX
 
       # double emission check, but allow early resolution to work
       conn_addrs = connection.addresses
-      return if !early_resolve && conn_addrs && (!conn_addrs.empty? && !addresses.intersect?(conn_addrs))
+      return if !early_resolve && conn_addrs && !conn_addrs.empty? && !addresses.intersect?(conn_addrs)
 
       log do
         "resolver #{FAMILY_TYPES[RECORD_TYPES[family]]}: " \
@@ -124,7 +124,7 @@ module HTTPX
       disconnect
     end
 
-    def early_resolve(connection, hostname: connection.peer.host)
+    def early_resolve(connection, hostname: connection.peer.host) # rubocop:disable Naming/PredicateMethod
       addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
 
       return false unless addresses