httpx 1.7.5 → 1.7.7

data/lib/httpx/connection.rb

@@ -62,6 +62,7 @@ module HTTPX
       @pending = []
       @inflight = 0
       @keep_alive_timeout = @options.timeout[:keep_alive_timeout]
+      @no_more_requests_counter = 0
 
       if @options.io
         # if there's an already open IO, get its
@@ -106,7 +107,7 @@ module HTTPX
         # origin came from an ORIGIN frame, we're going to verify the hostname with the
         # SSL certificate
         (@origins.size == 1 || @origin == uri.origin || (@io.is_a?(SSL) && @io.verify_hostname(uri.host))) &&
-        @options == options
+        @options.connection_options_match?(options)
     end
 
     def mergeable?(connection)
@@ -117,7 +118,7 @@ module HTTPX
       (
         (open? && @origin == connection.origin) ||
         !(@io.addresses & (connection.addresses || [])).empty?
-      ) && @options == connection.options
+      ) && @options.connection_options_match?(connection.options)
     end
 
     # coalesces +self+ into +connection+.
@@ -287,6 +288,10 @@ module HTTPX
     def reset
       return if @state == :closing || @state == :closed
 
+      # do not reset a connection which may have restarted back to :idle, such when the parser resets
+      # (example: HTTP/1 parser disabling pipelining)
+      return if @state == :idle && @pending.any?
+
       parser = @parser
 
       if parser && parser.respond_to?(:max_concurrent_requests)
@@ -313,8 +318,8 @@ module HTTPX
           log(level: 3) { "keep alive timeout expired, pinging connection..." }
           @pending << request
           transition(:active) if @state == :inactive
-          parser.ping
           request.ping!
+          ping
           return
         end
 
@@ -455,11 +460,11 @@ module HTTPX
           #
           # this condition takes into account:
           #
-          # * the number of inflight requests
           # * the number of pending requests
+          # * the number of inflight requests
           # * whether the write buffer has bytes (i.e. for close handshake)
           if @pending.empty? && @inflight.zero? && @write_buffer.empty?
-            log(level: 3) { "NO MORE REQUESTS..." } if @parser && @parser.pending.any?
+            no_more_requests_loop_check if @parser && @parser.pending.any?
 
             # terminate if an altsvc connection has been established
             terminate if @altsvc_connection
@@ -509,7 +514,7 @@ module HTTPX
 
             # exit #consume altogether if all outstanding requests have been dealt with
             if @pending.empty? && @inflight.zero? && @write_buffer.empty? # rubocop:disable Style/Next
-              log(level: 3) { "NO MORE REQUESTS..." } if @parser && @parser.pending.any?
+              no_more_requests_loop_check if @parser && @parser.pending.any?
 
               # terminate if an altsvc connection has been established
               terminate if @altsvc_connection
@@ -635,15 +640,16 @@ module HTTPX
           build_altsvc_connection(alt_origin, origin, alt_params)
         end
         @response_received_at = Utils.now
+        @no_more_requests_counter = 0
         @inflight -= 1
         response.finish!
-        request.emit(:response, response)
+        request.emit_response(response)
       end
       parser.on(:altsvc) do |alt_origin, origin, alt_params|
         build_altsvc_connection(alt_origin, origin, alt_params)
       end
 
-      parser.on(:pong, &method(:send_pending))
+      parser.on(:pong, &method(:pong))
 
       parser.on(:promise) do |request, stream|
         request.emit(:promise, parser, stream)
@@ -701,7 +707,7 @@ module HTTPX
         @inflight -= 1
         response = ErrorResponse.new(request, error)
         request.response = response
-        request.emit(:response, response)
+        request.emit_response(response)
       end
     end
 
@@ -755,16 +761,6 @@ module HTTPX
         return if @inflight.positive? || @parser.waiting_for_ping?
       when :closing
         return unless connecting? || @state == :open
-
-        unless @write_buffer.empty?
-          # preset state before handshake, as error callbacks
-          # may take it back here.
-          @state = nextstate
-          # handshakes, try sending
-          consume
-          @write_buffer.clear
-          return
-        end
       when :closed
         return unless @state == :closing
         return unless @write_buffer.empty?
@@ -790,6 +786,12 @@ module HTTPX
       case nextstate
       when :inactive
         disconnect
+      when :closing
+        return if @write_buffer.empty?
+
+        # try flushing termination handshakes
+        consume
+        @write_buffer.clear
       when :closed
         # TODO: should this raise an error instead?
         return unless @pending.empty?
@@ -813,16 +815,18 @@ module HTTPX
     end
 
     def close_sibling
-      return unless @sibling
+      sibling = @sibling
 
-      if @sibling.io_connected?
+      return unless sibling
+
+      if sibling.io_connected?
         reset
         # TODO: transition connection to closed
       end
 
-      unless @sibling.state == :closed
-        merge(@sibling) unless @main_sibling
-        @sibling.force_reset(true)
+      unless sibling.state == :closed
+        merge(sibling) unless @main_sibling
+        sibling.force_reset(true)
       end
 
       @sibling = nil
@@ -902,28 +906,61 @@ module HTTPX
       end
     end
 
+    def ping
+      return if parser.waiting_for_ping?
+
+      parser.ping
+      call
+    end
+
+    def pong
+      @response_received_at = Utils.now
+      @no_more_requests_counter = 0
+      send_pending
+    end
+
+    def no_more_requests_loop_check
+      log(level: 3) { "NO MORE REQUESTS..." }
+      @no_more_requests_counter += 1
+
+      return if @no_more_requests_counter < 50
+
+      raise Error, "connection corrupted, aborted after looping for a while, " \
+                   "please report this https://gitlab.com/os85/httpx/-/work_items " \
+                   "along with debug logs"
+    end
+
     # recover internal state and emit all relevant error responses when +error+ was raised.
     # this takes an optiona +request+ which may have already been handled and can be opted out
     # in the state recovery process.
     def handle_error(error, request = nil)
-      parser.handle_error(error, request) if @parser && @parser.respond_to?(:handle_error)
-      while (req = @pending.shift)
-        next if request && req == request
+      if request
+        @inflight -= 1
+        response = ErrorResponse.new(request, error)
+        request.response = response
+        request.emit_response(response)
+      end
 
-        response = ErrorResponse.new(req, error)
-        req.response = response
-        req.emit(:response, response)
+      pending = @pending
+      if (parser = @parser) && parser.respond_to?(:handle_error)
+        # parser.handle_error may disconnect the connection
+        pending = @pending.dup
+        @pending = []
+
+        parser.handle_error(error, request)
       end
 
-      return unless request
+      while (req = pending.shift)
+        next if request && req == request
 
-      @inflight -= 1
-      response = ErrorResponse.new(request, error)
-      request.response = response
-      request.emit(:response, response)
+        resp = ErrorResponse.new(req, error)
+        req.response = resp
+        req.emit_response(resp)
+      end
     end
 
     def set_request_timeouts(request)
+      request.connection = self
       set_request_write_timeout(request)
       set_request_read_timeout(request)
       set_request_request_timeout(request)
@@ -959,29 +996,29 @@ module HTTPX
       end
     end
 
-    def write_timeout_callback(request, write_timeout)
+    def write_timeout_callback(request, timeout)
       return if request.state == :done
 
       @write_buffer.clear
-      error = WriteTimeoutError.new(request, nil, write_timeout)
+      error = WriteTimeoutError.new(request, nil, timeout)
 
-      on_error(error, request)
+      request.handle_error(error)
     end
 
-    def read_timeout_callback(request, read_timeout, error_type = ReadTimeoutError)
+    def read_timeout_callback(request, timeout, error_type = ReadTimeoutError)
       response = request.response
 
       return if response && response.finished?
 
       @write_buffer.clear
-      error = error_type.new(request, request.response, read_timeout)
+      error = error_type.new(request, response, timeout)
 
-      on_error(error, request)
+      request.handle_error(error)
     end
 
     def set_request_timeout(label, request, timeout, start_event, finish_events, &callback)
       request.set_timeout_callback(start_event) do
-        unless @current_selector
+        unless (selector = @current_selector)
           raise Error, "request has been resend to an out-of-session connection, and this " \
                        "should never happen!!! Please report this error! " \
                        "(state:#{@state}, " \
@@ -992,7 +1029,7 @@ module HTTPX
                        "coalesced?:#{coalesced?})"
         end
 
-        timer = @current_selector.after(timeout, callback)
+        timer = selector.after(timeout, callback)
         request.active_timeouts << label
 
         Array(finish_events).each do |event|

data/lib/httpx/io/ssl.rb

@@ -51,6 +51,7 @@ module HTTPX
     end
 
     def protocol
+      # @type ivar @io: OpenSSL::SSL::SSLSocket
       @io.alpn_protocol || super
     rescue StandardError
       super
@@ -60,6 +61,7 @@ module HTTPX
       # in jruby, alpn_protocol may return ""
       # https://github.com/jruby/jruby-openssl/issues/287
       def protocol
+        # @type ivar @io: OpenSSL::SSL::SSLSocket
         proto = @io.alpn_protocol
 
         return super if proto.nil? || proto.empty?
@@ -76,9 +78,10 @@ module HTTPX
 
     def verify_hostname(host)
       return false if @ctx.verify_mode == OpenSSL::SSL::VERIFY_NONE
-      return false if !@io.respond_to?(:peer_cert) || @io.peer_cert.nil?
+      # @type ivar @io: OpenSSL::SSL::SSLSocket
+      return false if !@io.respond_to?(:peer_cert) || (peer_cert = @io.peer_cert).nil?
 
-      OpenSSL::SSL.verify_certificate_identity(@io.peer_cert, host)
+      OpenSSL::SSL.verify_certificate_identity(peer_cert, host)
     end
 
     def connected?
@@ -86,7 +89,9 @@ module HTTPX
     end
 
     def ssl_session_expired?
-      @ssl_session.nil? || Process.clock_gettime(Process::CLOCK_REALTIME) >= (@ssl_session.time.to_f + @ssl_session.timeout)
+      ssl_session = @ssl_session
+
+      ssl_session.nil? || Process.clock_gettime(Process::CLOCK_REALTIME) >= (ssl_session.time.to_f + ssl_session.timeout)
     end
 
     def connect
@@ -97,6 +102,8 @@ module HTTPX
         return unless @state == :connected
       end
 
+      # @type ivar @io: OpenSSL::SSL::SSLSocket
+
       unless @io.is_a?(OpenSSL::SSL::SSLSocket)
         if (hostname_is_ip = (@ip == @sni_hostname)) && @ctx.verify_hostname
           # IPv6 address would be "[::1]", must turn to "0000:0000:0000:0000:0000:0000:0000:0001" for cert SAN check
@@ -105,16 +112,19 @@ module HTTPX
           @ctx.verify_hostname = false
         end
 
-        @io = OpenSSL::SSL::SSLSocket.new(@io, @ctx)
+        ssl = OpenSSL::SSL::SSLSocket.new(@io, @ctx)
 
-        @io.hostname = @sni_hostname unless hostname_is_ip
-        @io.session = @ssl_session unless ssl_session_expired?
-        @io.sync_close = true
+        ssl.hostname = @sni_hostname unless hostname_is_ip
+        ssl.session = @ssl_session unless ssl_session_expired?
+        ssl.sync_close = true
+
+        @io = ssl
       end
       try_ssl_connect
     end
 
     def try_ssl_connect
+      # @type ivar @io: OpenSSL::SSL::SSLSocket
       ret = @io.connect_nonblock(exception: false)
       log(level: 3, color: :cyan) { "TLS CONNECT: #{ret}..." }
       case ret
@@ -147,16 +157,18 @@ module HTTPX
     def log_transition_state(nextstate)
       return super unless nextstate == :negotiated
 
-      server_cert = @io.peer_cert
+      # @type ivar @io: OpenSSL::SSL::SSLSocket
+
+      server_cert = @io.peer_cert #: OpenSSL::X509::Certificate
 
       "#{super}\n\n" \
         "SSL connection using #{@io.ssl_version} / #{Array(@io.cipher).first}\n" \
         "ALPN, server accepted to use #{protocol}\n" \
         "Server certificate:\n " \
-        "subject: #{server_cert.subject}\n " \
-        "start date: #{server_cert.not_before}\n " \
-        "expire date: #{server_cert.not_after}\n " \
-        "issuer: #{server_cert.issuer}\n " \
+        "subject: #{log_redact(server_cert.subject)}\n " \
+        "start date: #{log_redact(server_cert.not_before)}\n " \
+        "expire date: #{log_redact(server_cert.not_after)}\n " \
+        "issuer: #{log_redact(server_cert.issuer)}\n " \
         "SSL certificate verify ok."
     end
   end

data/lib/httpx/io/tcp.rb

@@ -23,18 +23,21 @@ module HTTPX
       @fallback_protocol = @options.fallback_protocol
       @port = origin.port
       @interests = :w
-      if @options.io
-        @io = case @options.io
-              when Hash
-                @options.io[origin.authority]
-              else
-                @options.io
-        end
-        raise Error, "Given IO objects do not match the request authority" unless @io
-
-        _, _, _, ip = @io.addr
-        @ip = Resolver::Entry.new(ip)
-        @addresses << @ip
+      if (io = @options.io)
+        io =
+          case io
+          when Hash
+            io[origin.authority]
+          else
+            io
+          end
+        raise Error, "Given IO objects do not match the request authority" unless io
+
+        # @type var io: TCPSocket | OpenSSL::SSL::SSLSocket
+
+        _, _, _, ip = io.addr
+        @io = io
+        @addresses << (@ip = Resolver::Entry.new(ip))
         @keep_open = true
         @state = :connected
       else
@@ -183,6 +186,9 @@ module HTTPX
 
       begin
         @io.close
+      rescue StandardError => e
+        log { "error closing socket" }
+        log { e.full_message(highlight: false) }
       ensure
         transition(:closed)
       end

data/lib/httpx/io/unix.rb

@@ -14,16 +14,20 @@ module HTTPX
       @state = :idle
       @options = options
       @fallback_protocol = @options.fallback_protocol
-      if @options.io
-        @io = case @options.io
-              when Hash
-                @options.io[origin.authority]
-              else
-                @options.io
-        end
-        raise Error, "Given IO objects do not match the request authority" unless @io
+      if (io = @options.io)
+        io =
+          case io
+          when Hash
+            io[origin.authority]
+          else
+            io
+          end
+        raise Error, "Given IO objects do not match the request authority" unless io
+
+        # @type var io: UNIXSocket
 
-        @path = @io.path
+        _, @path = io.addr
+        @io = io
         @keep_open = true
         @state = :connected
       elsif path

data/lib/httpx/loggable.rb

@@ -57,7 +57,7 @@ module HTTPX
       log_redact(text, @options.debug_redact == :body)
     end
 
-    def log_redact(text, should_redact)
+    def log_redact(text, should_redact = nil)
       should_redact ||= @options.debug_redact == true
 
       return text.to_s unless should_redact

data/lib/httpx/options.rb

@@ -202,16 +202,19 @@ module HTTPX
 
     REQUEST_BODY_IVARS = %i[@headers].freeze
 
-    def ==(other)
-      super || options_equals?(other)
-    end
+    # checks whether +other+ matches the same connection-level options
+    def connection_options_match?(other, ignore_ivars = nil)
+      return true if self == other
 
-    # checks whether +other+ is equal by comparing the session options
-    def options_equals?(other, ignore_ivars = REQUEST_BODY_IVARS)
       # headers and other request options do not play a role, as they are
       # relevant only for the request.
-      ivars = instance_variables - ignore_ivars
-      other_ivars = other.instance_variables - ignore_ivars
+      ivars = instance_variables
+      ivars.reject! { |iv| REQUEST_BODY_IVARS.include?(iv) }
+      ivars.reject! { |iv| ignore_ivars.include?(iv) } if ignore_ivars
+
+      other_ivars = other.instance_variables
+      other_ivars.reject! { |iv| REQUEST_BODY_IVARS.include?(iv) }
+      other_ivars.reject! { |iv| ignore_ivars.include?(iv) } if ignore_ivars
 
       return false if ivars.size != other_ivars.size
 
@@ -222,6 +225,19 @@ module HTTPX
       end
     end
 
+    RESOLVER_IVARS = %i[
+      @resolver_class @resolver_cache @resolver_options
+      @resolver_native_class @resolver_system_class @resolver_https_class
+    ].freeze
+
+    # checks whether +other+ matches the same resolver-level options
+    def resolver_options_match?(other)
+      self == other ||
+        RESOLVER_IVARS.all? do |ivar|
+          instance_variable_get(ivar) == other.instance_variable_get(ivar)
+        end
+    end
+
     # returns a HTTPX::Options instance resulting of the merging of +other+ with self.
     # it may return self if +other+ is self or equal to self.
     def merge(other)

data/lib/httpx/parser/http1.rb

@@ -14,6 +14,8 @@ module HTTPX
         @state = :idle
         @buffer = "".b
         @headers = {}
+        @content_length = nil
+        @_has_trailers = @upgrade = false
       end
 
       def <<(chunk)
@@ -25,8 +27,8 @@ module HTTPX
         @state = :idle
         @headers = {}
         @content_length = nil
-        @_has_trailers = nil
-        @buffer.clear
+        @_has_trailers = @upgrade = false
+        @buffer = @buffer.to_s
       end
 
       def upgrade?
@@ -34,7 +36,7 @@ module HTTPX
       end
 
       def upgrade_data
-        @buffer
+        @buffer.to_s
       end
 
       private
@@ -55,6 +57,7 @@ module HTTPX
       end
 
       def parse_headline
+        #: @type ivar @buffer: String
         idx = @buffer.index("\n")
         return unless idx
 
@@ -75,6 +78,8 @@ module HTTPX
         headers = @headers
         buffer = @buffer
 
+        #: @type var buffer: String
+
         while (idx = buffer.index("\n"))
           # @type var line: String
           line = buffer.byteslice(0..idx)
@@ -118,17 +123,20 @@ module HTTPX
 
       def parse_data
         if @buffer.respond_to?(:each)
+          # @type ivar @buffer: Transcoder::Chunker::Decoder
           @buffer.each do |chunk|
             @observer.on_data(chunk)
           end
         elsif @content_length
-          # @type var data: String
+          # @type ivar @buffer: String
           data = @buffer.byteslice(0, @content_length)
+          # @type var data: String
           @buffer = @buffer.byteslice(@content_length..-1) || "".b
           @content_length -= data.bytesize
           @observer.on_data(data)
           data.clear
         else
+          # @type ivar @buffer: String
           @observer.on_data(@buffer)
           @buffer.clear
         end
@@ -152,7 +160,7 @@ module HTTPX
             tr_encoding.split(/ *, */).each do |encoding|
               case encoding
               when "chunked"
-                @buffer = Transcoder::Chunker::Decoder.new(@buffer, @_has_trailers)
+                @buffer = Transcoder::Chunker::Decoder.new(@buffer.to_s, @_has_trailers)
               end
             end
           end
@@ -165,6 +173,7 @@ module HTTPX
         if @content_length
           @content_length <= 0
         elsif @buffer.respond_to?(:finished?)
+          # @type ivar @buffer: Transcoder::Chunker::Decoder
           @buffer.finished?
         else
           false

data/lib/httpx/plugins/auth/digest.rb

@@ -24,6 +24,11 @@ module HTTPX
 
         def authenticate(request, authenticate)
           "Digest #{generate_header(request.verb, request.path, authenticate)}"
+        rescue StandardError => e
+          response = ErrorResponse.new(request, e)
+          request.response = response
+          request.emit_response(response)
+          nil
         end
 
         private
@@ -77,6 +82,7 @@ module HTTPX
 
           if params["algorithm"] =~ /(.*?)(-sess)?$/
             alg = Regexp.last_match(1)
+            raise_format_error unless alg
             algorithm = ::Digest.const_get(alg)
             raise Error, "unknown algorithm \"#{alg}\"" unless algorithm
 

data/lib/httpx/plugins/auth.rb

@@ -44,6 +44,7 @@ module HTTPX
           super
 
           @auth_header_value = nil
+          @auth_header_value_mtx = Thread::Mutex.new
           @skip_auth_header_value = false
         end
 
@@ -63,7 +64,9 @@ module HTTPX
         end
 
         def reset_auth_header_value!
-          @auth_header_value = nil
+          @auth_header_value_mtx.synchronize do
+            @auth_header_value = nil
+          end
         end
 
         private
@@ -71,9 +74,11 @@ module HTTPX
         def send_request(request, *)
           return super if @skip_auth_header_value || request.authorized?
 
-          @auth_header_value ||= generate_auth_token
+          auth_header_value = @auth_header_value_mtx.synchronize do
+            @auth_header_value ||= generate_auth_token
+          end
 
-          request.authorize(@auth_header_value) if @auth_header_value
+          request.authorize(auth_header_value) if auth_header_value
 
           super
         end
@@ -92,9 +97,11 @@ module HTTPX
       end
 
       module RequestMethods
+        attr_reader :auth_token_value
+
         def initialize(*)
           super
-          @auth_token_value = nil
+          @auth_token_value = @auth_header_value = nil
         end
 
         def authorized?
@@ -102,19 +109,20 @@ module HTTPX
         end
 
         def unauthorize!
-          return unless (auth_value = @auth_token_value)
+          return unless (auth_value = @auth_header_value)
 
           @headers.get("authorization").delete(auth_value)
 
-          @auth_token_value = nil
+          @auth_token_value = @auth_header_value = nil
         end
 
         def authorize(auth_value)
+          @auth_header_value = auth_value
           if (auth_type = @options.auth_header_type)
-            auth_value = "#{auth_type} #{auth_value}"
+            @auth_header_value = "#{auth_type} #{@auth_header_value}"
           end
 
-          @headers.add("authorization", auth_value)
+          @headers.add("authorization", @auth_header_value)
 
           @auth_token_value = auth_value
         end
@@ -138,8 +146,14 @@ module HTTPX
             return unless auth_error?(response, request.options) ||
                           (@options.generate_auth_value_on_retry && @options.generate_auth_value_on_retry.call(response))
 
+            # regenerate token before retry, but only if it's the first request from batch failing.
+            # otherwise, it means that the first request already passed here, so this request should
+            # use whatever was generated for it.
+            @auth_header_value_mtx.synchronize do
+              @auth_header_value = generate_auth_token if request.auth_token_value == @auth_header_value
+            end
+
             request.unauthorize!
-            @auth_header_value = generate_auth_token
           end
 
           def auth_error?(response, options)