httpx 1.7.8 → 1.8.1

data/lib/httpx/connection.rb

@@ -47,8 +47,8 @@ module HTTPX
     def initialize(uri, options)
       @current_session = @current_selector = @max_concurrent_requests =
                            @parser = @sibling = @coalesced_connection = @altsvc_connection =
-                                                  @family = @io = @ssl_session = @timeout =
-                                                                    @connected_at = @response_received_at = nil
+                                                  @ping_timer = @family = @io = @ssl_session =
+                                                                            @timeout = @connected_at = @response_received_at = nil
 
       @exhausted = @cloned = @main_sibling = false
 
@@ -161,15 +161,17 @@ module HTTPX
     end
 
     def purge_pending(&block)
-      pendings = []
       if @parser
         pending = @parser.pending
         @inflight -= pending.size
-        pendings << pending
+        pending.reject! do |req|
+          block.call(req)
+          true
+        end
       end
-      pendings << @pending
-      pendings.each do |pending|
-        pending.reject!(&block)
+      @pending.reject! do |req|
+        block.call(req)
+        true
       end
     end
 
@@ -203,7 +205,7 @@ module HTTPX
       return @parser.interests if @parser
 
       nil
-    rescue StandardError => e
+    rescue Error => e
       on_error(e)
       nil
     end
@@ -215,6 +217,8 @@ module HTTPX
     def call
       case @state
       when :idle
+        return if no_more_requests?
+
         connect
 
         # when opening the tcp or ssl socket fails
@@ -222,18 +226,39 @@ module HTTPX
 
         consume
       when :closed
-        return
+        return if no_more_requests?
+
+        # there are pending requests to send, restart the state machine.
+        idling
+
+        # @fiber-switch-guard
+        # fiber may have switch after ensuring that @io is closed.
+        return unless @state == :idle
+
+        call
       when :closing
         consume
         transition(:closed)
+
+        # @fiber-switch-guard
+        # fiber may have switch while closing @io.
+        return if @state == :closed &&
+                  # only remain here if there are pending requests.
+                  @pending.empty?
+
+        call
       when :open
         consume
       end
       nil
-    rescue IOError => e
+    rescue Errno::ECONNRESET,
+           Errno::EINVAL,
+           SocketError,
+           IOError,
+           TLSError => e
       @write_buffer.clear
       on_io_error(e)
-    rescue StandardError => e
+    rescue Error => e
       @write_buffer.clear
       on_error(e)
     rescue Exception => e # rubocop:disable Lint/RescueException
@@ -241,6 +266,10 @@ module HTTPX
       raise e
     end
 
+    def initial_call
+      call
+    end
+
     def close
       transition(:active) if @state == :inactive
 
@@ -251,7 +280,12 @@ module HTTPX
       case @state
       when :idle
         purge_after_closed
-        disconnect
+
+        # @fiber-switch-guard
+        if @io.can_disconnect? && @pending.empty?
+          disconnect
+          return
+        end
       when :closed
         @connected_at = nil
       end
@@ -292,6 +326,11 @@ module HTTPX
       # (example: HTTP/1 parser disabling pipelining)
       return if @state == :idle && @pending.any?
 
+      if @ping_timer
+        @ping_timer.cancel
+        @ping_timer = nil
+      end
+
       parser = @parser
 
       if parser && parser.respond_to?(:max_concurrent_requests)
@@ -319,7 +358,7 @@ module HTTPX
           @pending << request
           transition(:active) if @state == :inactive
           request.ping!
-          ping
+          ping(request)
           return
         end
 
@@ -341,6 +380,9 @@ module HTTPX
 
     def idling
       purge_after_closed
+
+      return unless @state == :closed
+
       @write_buffer.clear
       transition(:idle)
       return unless @parser
@@ -408,6 +450,11 @@ module HTTPX
 
     def on_io_error(e)
       on_error(e)
+
+      # do not force close if parser resets the connection.
+      # can happen i.e. when HTTP/1.1 pipelining is disabled.
+      return if @state == :idle && @pending.any?
+
       force_close(true)
     end
 
@@ -421,6 +468,8 @@ module HTTPX
         if @timeout
           @timeout -= error.timeout
           return unless @timeout <= 0
+
+          @timeout = nil
         end
 
         error = error.to_connection_error if connecting?
@@ -429,7 +478,7 @@ module HTTPX
       reset
     end
 
-    # :nocov:
+    # simplecov:disable
     def inspect
       "#<#{self.class}:#{object_id} " \
         "@origin=#{@origin} " \
@@ -437,7 +486,7 @@ module HTTPX
         "@pending=#{@pending.size} " \
         "@io=#{@io}>"
     end
-    # :nocov:
+    # simplecov:enable
 
     private
 
@@ -463,7 +512,7 @@ module HTTPX
           # * the number of pending requests
           # * the number of inflight requests
           # * whether the write buffer has bytes (i.e. for close handshake)
-          if @pending.empty? && @inflight.zero? && @write_buffer.empty?
+          if no_more_requests? && @write_buffer.empty?
             no_more_requests_loop_check if @parser && @parser.pending.any?
 
             # terminate if an altsvc connection has been established
@@ -513,7 +562,7 @@ module HTTPX
             break if @state == :closing || @state == :closed
 
             # exit #consume altogether if all outstanding requests have been dealt with
-            if @pending.empty? && @inflight.zero? && @write_buffer.empty? # rubocop:disable Style/Next
+            if no_more_requests? && @write_buffer.empty? # rubocop:disable Style/Next
               no_more_requests_loop_check if @parser && @parser.pending.any?
 
               # terminate if an altsvc connection has been established
@@ -660,7 +709,12 @@ module HTTPX
         @exhausted = true
         parser.close
 
+        # @fiber-switch-guard
+        # fiber may have switched while closing @io, check whether still in the exhausted loop.
+        next unless @exhausted
+
         idling
+
         @exhausted = false
       end
       parser.on(:origin) do |origin|
@@ -676,6 +730,9 @@ module HTTPX
         enqueue_pending_requests_from_parser(parser)
 
         reset
+
+        next unless @state == :closed
+
         # :reset event only fired in http/1.1, so this guarantees
         # that the connection will be closed here.
         idling unless @pending.empty?
@@ -766,6 +823,9 @@ module HTTPX
         return unless @write_buffer.empty?
 
         purge_after_closed
+
+        # @fiber-switch-guard
+        return unless @state == :closing && (@io.nil? || @io.can_disconnect?)
       when :already_open
         nextstate = :open
         # the first check for given io readiness must still use a timeout.
@@ -833,7 +893,15 @@ module HTTPX
     end
 
     def purge_after_closed
-      @io.close if @io
+      if @io
+        @io.close
+
+        # @fiber-switch-guard
+        # due to fiber scheduler, multiple fibers may be listening on the same connection
+        # and moving the state machine forward; in such cases, when the control flow reaches
+        # this line, the io object may not be closed anymore.
+        return unless @io&.can_disconnect?
+      end
       @read_buffer.clear
       @timeout = nil
     end
@@ -906,14 +974,25 @@ module HTTPX
       end
     end
 
-    def ping
+    def ping(_request)
       return if parser.waiting_for_ping?
 
       parser.ping
+
+      ping_timeout = @options.timeout[:ping_timeout]
+
+      @ping_timer = @current_selector.after(ping_timeout) do
+        log(level: 3) { "ping timeout expired..." }
+        error = PingTimeoutError.new(ping_timeout, "Timed out after #{ping_timeout} seconds")
+        on_error(error)
+      end
+
       call
     end
 
     def pong
+      @ping_timer.cancel
+      @ping_timer = nil
       @response_received_at = Utils.now
       @no_more_requests_counter = 0
       send_pending
@@ -930,6 +1009,11 @@ module HTTPX
                    "along with debug logs"
     end
 
+    # true when there are no more pending nor inflight (in parser) requests
+    def no_more_requests?
+      @pending.empty? && @inflight.zero?
+    end
+
     # recover internal state and emit all relevant error responses when +error+ was raised.
     # this takes an optiona +request+ which may have already been handled and can be opted out
     # in the state recovery process.
@@ -964,6 +1048,7 @@ module HTTPX
       set_request_write_timeout(request)
       set_request_read_timeout(request)
       set_request_request_timeout(request)
+      set_request_total_request_timeout(request)
     end
 
     def set_request_read_timeout(request)
@@ -1016,6 +1101,18 @@ module HTTPX
       request.handle_error(error)
     end
 
+    def set_request_total_request_timeout(request)
+      return if request.started?
+
+      total_request_timeout = request.total_request_timeout
+
+      return if total_request_timeout.nil? || total_request_timeout.infinite?
+
+      set_request_timeout(:total_request_timeout, request, total_request_timeout, :headers, :complete) do
+        read_timeout_callback(request, total_request_timeout, TotalRequestTimeoutError)
+      end
+    end
+
     def set_request_timeout(label, request, timeout, start_event, finish_events, &callback)
       request.set_timeout_callback(start_event) do
         unless (selector = @current_selector)
@@ -1030,13 +1127,14 @@ module HTTPX
         end
 
         timer = selector.after(timeout, callback)
-        request.active_timeouts << label
+        timer.label = label
+        request.active_timeouts << timer
 
         Array(finish_events).each do |event|
           # clean up request timeouts if the connection errors out
           request.set_timeout_callback(event) do
             timer.cancel
-            request.active_timeouts.delete(label)
+            request.active_timeouts.delete(timer)
           end
         end
       end

data/lib/httpx/errors.rb

@@ -62,15 +62,22 @@ module HTTPX
   # Error raised when there was a timeout while sending a request from the server.
   class WriteTimeoutError < RequestTimeoutError; end
 
+  # Error raised when a response couldn't be received for a request after multiple interactions.
+  # This error should not be retriable.
+  class TotalRequestTimeoutError < RequestTimeoutError; end
+
   # Error raised when there was a timeout while waiting for the HTTP/2 settings frame from the server.
   class SettingsTimeoutError < TimeoutError; end
 
   # Error raised when there was a timeout while resolving a domain to an IP.
   class ResolveTimeoutError < TimeoutError; end
 
-  # Error raise when there was a timeout waiting for readiness of the socket the request is related to.
+  # Error raised when there was a timeout waiting for readiness of the socket the request is related to.
   class OperationTimeoutError < TimeoutError; end
 
+  # Error raised when a connection liveness probe (aka ping) times out.
+  class PingTimeoutError < TimeoutError; end
+
   # Error raised when there was an error while resolving a domain to an IP.
   class ResolveError < Error; end
 

data/lib/httpx/headers.rb

@@ -140,12 +140,12 @@ module HTTPX
       @headers.to_s
     end
 
-    # :nocov:
+    # simplecov:disable
     def inspect
       "#<#{self.class}:#{object_id} " \
         "#{to_hash.inspect}>"
     end
-    # :nocov:
+    # simplecov:enable
 
     # this is internal API and doesn't abide to other public API
     # guarantees, like downcasing strings.

data/lib/httpx/io/ssl.rb

@@ -20,8 +20,9 @@ module HTTPX
       super
 
       @ssl_session = nil
-      ctx_options = TLS_OPTIONS.merge(options.ssl)
-      @sni_hostname = ctx_options.delete(:hostname) || @hostname
+      ctx_options = TLS_OPTIONS
+      ctx_options = ctx_options.merge(options.ssl) if options.ssl && !options.ssl.empty?
+      @sni_hostname = (ctx_options.delete(:hostname) if ctx_options.key?(:hostname)) || @hostname
 
       if @keep_open && @io.is_a?(OpenSSL::SSL::SSLSocket)
         # externally initiated ssl socket
@@ -29,7 +30,7 @@ module HTTPX
         @state = :negotiated
       else
         @ctx = OpenSSL::SSL::SSLContext.new
-        @ctx.set_params(ctx_options) unless ctx_options.empty?
+        @ctx.set_params(ctx_options)
         unless @ctx.session_cache_mode.nil? # a dummy method on JRuby
           @ctx.session_cache_mode =
             OpenSSL::SSL::SSLContext::SESSION_CACHE_CLIENT | OpenSSL::SSL::SSLContext::SESSION_CACHE_NO_INTERNAL_STORE
@@ -51,24 +52,22 @@ module HTTPX
     end
 
     def protocol
-      # @type ivar @io: OpenSSL::SSL::SSLSocket
+      return super unless @io.is_a?(OpenSSL::SSL::SSLSocket)
+
       @io.alpn_protocol || super
-    rescue StandardError
-      super
     end
 
     if RUBY_ENGINE == "jruby"
       # in jruby, alpn_protocol may return ""
       # https://github.com/jruby/jruby-openssl/issues/287
       def protocol
-        # @type ivar @io: OpenSSL::SSL::SSLSocket
+        return super unless @io.is_a?(OpenSSL::SSL::SSLSocket)
+
         proto = @io.alpn_protocol
 
         return super if proto.nil? || proto.empty?
 
         proto
-      rescue StandardError
-        super
       end
     end
 

data/lib/httpx/io/tcp.rb

@@ -182,18 +182,35 @@ module HTTPX
     end
 
     def close
-      return if @keep_open || closed?
+      return if @keep_open
+
+      # mark tcp as closed, so that it can be disconnected.
+      # this bypasses the state machine API as not not allow the transition
+      # from idle to closed in normal circumstances.
+      @state = :closed if @state == :idle
+
+      return if closed?
 
       begin
         @io.close
-      rescue StandardError => e
+      rescue IOError => e
         log { "error closing socket" }
         log { e.full_message(highlight: false) }
       ensure
-        transition(:closed)
+        # @fiber-switch-guard
+        # ensure that all :closed IOs don't leave dangling sockets
+        # behind. This may happen in a fiber scheduler scenario where
+        # connection is reused across fibers.
+        transition(:closed) if @io.closed?
       end
     end
 
+    # signals that the connection that contains this IO can be checked back into the pool.
+    # that includes sockets opened outside of the scope of the session, or closed IOs.
+    def can_disconnect?
+      @keep_open || @state == :closed
+    end
+
     def connected?
       @state == :connected
     end
@@ -202,7 +219,7 @@ module HTTPX
       @state == :idle || @state == :closed
     end
 
-    # :nocov:
+    # simplecov:disable
     def inspect
       "#<#{self.class}:#{object_id} " \
         "#{@ip}:#{@port} " \
@@ -211,7 +228,7 @@ module HTTPX
         "@addresses=#{@addresses} " \
         "@state=#{@state}>"
     end
-    # :nocov:
+    # simplecov:enable
 
     private
 

data/lib/httpx/io/unix.rb

@@ -60,11 +60,11 @@ module HTTPX
       true
     end
 
-    # :nocov:
+    # simplecov:disable
     def inspect
       "#<#{self.class}:#{object_id} @path=#{@path}) @state=#{@state})>"
     end
-    # :nocov:
+    # simplecov:enable
 
     private
 

data/lib/httpx/loggable.rb

@@ -17,6 +17,12 @@ module HTTPX
 
     USE_DEBUG_LOG = ENV.key?("HTTPX_DEBUG")
 
+    def self.log_identifiers
+      "pid=#{Process.pid} " \
+        "tid=#{Thread.current.object_id} " \
+        "fid=#{Fiber.current.object_id}"
+    end
+
     def log(
       level: @options.debug_level,
       color: nil,
@@ -36,10 +42,9 @@ module HTTPX
         klass = klass.superclass
       end
 
-      message = +"(time:#{Time.now.utc}, pid:#{Process.pid}, " \
-                 "tid:#{Thread.current.object_id}, " \
-                 "fid:#{Fiber.current.object_id}, " \
-                 "self:#{class_name}##{object_id}) "
+      message = +"(time=#{Time.now.utc} " \
+                 "#{Loggable.log_identifiers} " \
+                 "self=#{class_name}##{object_id}) "
       message << msg.call << "\n"
       message = "\e[#{COLORS[color]}m#{message}\e[0m" if color && debug_stream.respond_to?(:isatty) && debug_stream.isatty
       debug_stream << message
@@ -49,6 +54,8 @@ module HTTPX
       log(level: level, color: color, debug_level: debug_level, debug: debug) { ex.full_message }
     end
 
+    private
+
     def log_redact_headers(text)
       log_redact(text, @options.debug_redact == :headers)
     end

data/lib/httpx/options.rb

@@ -8,12 +8,12 @@ module HTTPX
     WINDOW_SIZE = 1 << 14 # 16K
     MAX_BODY_THRESHOLD_SIZE = (1 << 10) * 112 # 112K
     KEEP_ALIVE_TIMEOUT = 20
+    PING_TIMEOUT = 2
     SETTINGS_TIMEOUT = 10
     CLOSE_HANDSHAKE_TIMEOUT = 10
     CONNECT_TIMEOUT = READ_TIMEOUT = WRITE_TIMEOUT = 60
-    REQUEST_TIMEOUT = OPERATION_TIMEOUT = nil
+    REQUEST_TIMEOUT = OPERATION_TIMEOUT = TOTAL_REQUEST_TIMEOUT = nil
     RESOLVER_TYPES = %i[memory file].freeze
-
     # default value used for "user-agent" header, when not overridden.
     USER_AGENT = "httpx.rb/#{VERSION}".freeze # rubocop:disable Style/RedundantFreeze
 
@@ -79,9 +79,15 @@ module HTTPX
     # :decompress_response_body :: whether to auto-decompress response body (defaults to <tt>true</tt>).
     # :compress_request_body :: whether to auto-decompress response body (defaults to <tt>true</tt>)
     # :timeout :: hash of timeout configurations (supports <tt>:connect_timeout</tt>, <tt>:settings_timeout</tt>,
-    #             <tt>:operation_timeout</tt>, <tt>:keep_alive_timeout</tt>,  <tt>:read_timeout</tt>,  <tt>:write_timeout</tt>
-    #             and <tt>:request_timeout</tt>
+    #             <tt>:operation_timeout</tt>, <tt>:keep_alive_timeout</tt>, <tt>:read_timeout</tt>,  <tt>:write_timeout</tt>,
+    #             <tt>:request_timeout</tt>, <tt>:total_request_timeout</tt> and <tt>:ping_timeout</tt>,
     # :headers :: hash of HTTP headers (ex: <tt>{ "x-custom-foo" => "bar" }</tt>)
+    # :max_response_body_size :: maximum size (in bytes) that the response body can consume (no threshold by default), after which an
+    #                            error is raised.
+    # :max_response_headers :: maximum number of header fields that a response can receive, after which an error is raised.
+    # :max_response_header_value_size :: maximum size (in bytes) a header value can have (no threshold by default).
+    #                                    for cases where the value is broken into multiple header fields (such as "cookie" or "set-cookie"),
+    #                                    this is the total aggregated size.
     # :window_size :: number of bytes to read from a socket
     # :buffer_size :: internal read and write buffer size in bytes
     # :body_threshold_size :: maximum size in bytes of response payload that is buffered in memory.
@@ -387,6 +393,7 @@ module HTTPX
     # number options
     %i[
       max_concurrent_requests max_requests window_size buffer_size
+      max_response_body_size max_response_headers max_response_header_value_size
       body_threshold_size debug_level
     ].each do |option|
       class_eval(<<-OUT, __FILE__, __LINE__ + 1)
@@ -553,15 +560,20 @@ module HTTPX
       :supported_compression_formats => %w[gzip deflate],
       :decompress_response_body => true,
       :compress_request_body => true,
+      :max_response_headers => 1000,
+      :max_response_header_value_size => nil,
+      :max_response_body_size => Float::INFINITY,
       :timeout => {
         connect_timeout: CONNECT_TIMEOUT,
         settings_timeout: SETTINGS_TIMEOUT,
         close_handshake_timeout: CLOSE_HANDSHAKE_TIMEOUT,
         operation_timeout: OPERATION_TIMEOUT,
         keep_alive_timeout: KEEP_ALIVE_TIMEOUT,
+        ping_timeout: PING_TIMEOUT,
         read_timeout: READ_TIMEOUT,
         write_timeout: WRITE_TIMEOUT,
         request_timeout: REQUEST_TIMEOUT,
+        total_request_timeout: TOTAL_REQUEST_TIMEOUT,
       }.freeze,
       :headers_class => Class.new(Headers, &SET_TEMPORARY_NAME),
       :headers => EMPTY_HASH,

data/lib/httpx/parser/http1.rb

@@ -9,11 +9,13 @@ module HTTPX
 
       attr_reader :status_code, :http_version, :headers
 
-      def initialize(observer)
+      def initialize(observer, max_headers, max_header_value_size)
         @observer = observer
         @state = :idle
         @buffer = "".b
         @headers = {}
+        @max_headers = max_headers
+        @max_header_value_size = max_header_value_size
         @content_length = nil
         @_has_trailers = @upgrade = false
       end
@@ -117,7 +119,11 @@ module HTTPX
           value.strip!
           raise Error, "wrong header format" if value.nil?
 
-          (headers[key.downcase] ||= []) << value
+          values = (headers[key.downcase] ||= []) << value
+
+          raise Error, "maximum header value size exceeded" if @max_header_value_size && (values.sum(&:size) > @max_header_value_size)
+
+          raise Error, "maximum number of response headers exceeded" if headers.size > @max_headers
         end
       end
 

data/lib/httpx/plugins/auth/digest.rb

@@ -24,7 +24,7 @@ module HTTPX
 
         def authenticate(request, authenticate)
           "Digest #{generate_header(request.verb, request.path, authenticate)}"
-        rescue StandardError => e
+        rescue HTTPX::Error => e
           response = ErrorResponse.new(request, e)
           request.response = response
           request.emit_response(response)