httpx 1.6.2 → 1.6.3

data/lib/httpx/loggable.rb

@@ -34,7 +34,7 @@ module HTTPX
         klass = klass.superclass
       end
 
-      message = +"(pid:#{Process.pid}, " \
+      message = +"(time:#{Time.now.utc}, pid:#{Process.pid}, " \
                  "tid:#{Thread.current.object_id}, " \
                  "fid:#{Fiber.current.object_id}, " \
                  "self:#{class_name}##{object_id}) "
@@ -47,7 +47,17 @@ module HTTPX
       log(level: level, color: color, debug_level: debug_level, debug: debug) { ex.full_message }
     end
 
-    def log_redact(text, should_redact = @options.debug_redact)
+    def log_redact_headers(text)
+      log_redact(text, @options.debug_redact == :headers)
+    end
+
+    def log_redact_body(text)
+      log_redact(text, @options.debug_redact == :body)
+    end
+
+    def log_redact(text, should_redact)
+      should_redact ||= @options.debug_redact == true
+
       return text.to_s unless should_redact
 
       "[REDACTED]"

data/lib/httpx/options.rb

@@ -13,6 +13,9 @@ module HTTPX
     CONNECT_TIMEOUT = READ_TIMEOUT = WRITE_TIMEOUT = 60
     REQUEST_TIMEOUT = OPERATION_TIMEOUT = nil
 
+    # default value used for "user-agent" header, when not overridden.
+    USER_AGENT = "httpx.rb/#{VERSION}".freeze # rubocop:disable Style/RedundantFreeze
+
     @options_names = []
 
     class << self
@@ -144,6 +147,7 @@ module HTTPX
         instance_variable_set(:"@#{k}", value)
       end
 
+      do_initialize
       freeze
     end
 
@@ -369,6 +373,8 @@ module HTTPX
     end
 
     def option_headers(value)
+      value = value.dup if value.frozen?
+
       headers_class.new(value)
     end
 
@@ -395,6 +401,20 @@ module HTTPX
       Array(value)
     end
 
+    # called after all options are initialized
+    def do_initialize
+      hs = @headers
+
+      # initialized default request headers
+      hs["user-agent"] = USER_AGENT unless hs.key?("user-agent")
+      hs["accept"] = "*/*" unless hs.key?("accept")
+      if hs.key?("range")
+        hs.delete("accept-encoding")
+      else
+        hs["accept-encoding"] = supported_compression_formats unless hs.key?("accept-encoding")
+      end
+    end
+
     def access_option(obj, k, ivar_map)
       case obj
       when Hash

data/lib/httpx/plugins/callbacks.rb

@@ -64,7 +64,7 @@ module HTTPX
 
             emit_or_callback_error(:connection_opened, connection.origin, connection.io.socket)
           end
-          connection.on(:close) do
+          connection.on(:callback_connection_closed) do
             next unless connection.current_session == self
 
             emit_or_callback_error(:connection_closed, connection.origin) if connection.used?
@@ -121,6 +121,20 @@ module HTTPX
           raise e.cause
         end
       end
+
+      module ConnectionMethods
+        private
+
+        def disconnect
+          return if @exhausted
+
+          return unless @current_session && @current_selector
+
+          emit(:callback_connection_closed)
+
+          super
+        end
+      end
     end
     register_plugin :callbacks, Callbacks
   end

data/lib/httpx/plugins/digest_auth.rb

@@ -48,7 +48,7 @@ module HTTPX
 
             probe_response = wrap { super(request).first }
 
-            return probe_response unless probe_response.is_a?(Response)
+            return ([probe_response] * requests.size) unless probe_response.is_a?(Response)
 
             if probe_response.status == 401 && digest.can_authenticate?(probe_response.headers["www-authenticate"])
               request.transition(:idle)

data/lib/httpx/plugins/proxy/http.rb

@@ -47,6 +47,17 @@ module HTTPX
             super || @state == :connecting || @state == :connected
           end
 
+          def force_close(*)
+            if @state == :connecting
+              # proxy connect related requests should not be reenqueed
+              @parser.reset!
+              @inflight -= @parser.pending.size
+              @parser.pending.clear
+            end
+
+            super
+          end
+
           private
 
           def handle_transition(nextstate)
@@ -64,23 +75,40 @@ module HTTPX
                 parser = @parser
                 parser.extend(ProxyParser)
                 parser.on(:response, &method(:__http_on_connect))
-                parser.on(:close) do |force|
+                parser.on(:close) do
                   next unless @parser
 
-                  if force
-                    reset
-                    emit(:terminate)
-                  end
+                  reset
+                  disconnect
                 end
                 parser.on(:reset) do
                   if parser.empty?
                     reset
                   else
-                    transition(:closing)
-                    transition(:closed)
+                    enqueue_pending_requests_from_parser(parser)
+
+                    initial_state = @state
+
+                    reset
+
+                    if @pending.empty?
+                      @parser = nil
+                      next
+                    end
+                    # keep parser state around due to proxy auth protocol;
+                    # intermediate authenticated request is already inside
+                    # the parser
+                    parser = nil
+
+                    if initial_state == :connecting
+                      parser = @parser
+                      @parser.reset
+                    end
+
+                    idling
+
+                    @parser = parser
 
-                    parser.reset if @parser
-                    transition(:idle)
                     transition(:connecting)
                   end
                 end

data/lib/httpx/plugins/response_cache/file_store.rb

@@ -130,6 +130,7 @@ module HTTPX::Plugins
         response = request.options.response_class.new(request, status, version, response_headers)
         response.original_request = original_request
         response.finish!
+        response.mark_as_cached!
 
         IO.copy_stream(f, response.body)
 

data/lib/httpx/plugins/response_cache.rb

@@ -118,7 +118,10 @@ module HTTPX
 
             response.copy_from_cached!
           elsif request.cacheable_verb? && ResponseCache.cacheable_response?(response)
-            request.options.response_cache_store.set(request, response) unless response.cached?
+            unless response.cached?
+              log { "caching response for #{request.uri}..." }
+              request.options.response_cache_store.set(request, response)
+            end
           end
 
           response
@@ -204,7 +207,7 @@ module HTTPX
         # returns a unique cache key as a String identifying this request
         def response_cache_key
           @response_cache_key ||= begin
-            keys = [@verb, @uri]
+            keys = [@verb, @uri.merge(path)]
 
             @options.supported_vary_headers.each do |field|
               value = @headers[field]
@@ -327,6 +330,14 @@ module HTTPX
           Time.now
         end
       end
+
+      module ResponseBodyMethods
+        def decode_chunk(chunk)
+          return chunk if @response.cached?
+
+          super
+        end
+      end
     end
     register_plugin :response_cache, ResponseCache
   end

data/lib/httpx/plugins/stream_bidi.rb

@@ -16,7 +16,7 @@ module HTTPX
       # The streams keeps send DATA frames while there's data; when they're ain't,
       # the stream is kept open; it must be explicitly closed by the end user.
       #
-      class HTTP2Bidi < Connection::HTTP2
+      module HTTP2Methods
         def initialize(*)
           super
           @lock = Thread::Mutex.new
@@ -117,8 +117,11 @@ module HTTPX
       # which allows it to be registered in the selector alongside actual HTTP-based
       # HTTPX::Connection objects.
       class Signal
+        attr_reader :error
+
         def initialize
           @closed = false
+          @error = nil
           @pipe_read, @pipe_write = IO.pipe
         end
 
@@ -159,6 +162,11 @@ module HTTPX
           @closed = true
         end
 
+        def on_error(error)
+          @error = error
+          terminate
+        end
+
         # noop (the owner connection will take of it)
         def handle_socket_timeout(interval); end
       end
@@ -193,6 +201,7 @@ module HTTPX
 
         def deselect_connection(connection, *)
           super
+
           connection.signal = nil
         end
       end
@@ -294,14 +303,14 @@ module HTTPX
           super
         end
 
-        private
-
-        def parser_type(protocol)
-          return HTTP2Bidi if protocol == "h2"
+        def call
+          return super unless (error = @signal.error)
 
-          super
+          on_error(error)
         end
 
+        private
+
         def set_parser_callbacks(parser)
           super
           parser.on(:flush_buffer) do

data/lib/httpx/pool.rb

@@ -8,7 +8,6 @@ require "httpx/resolver"
 
 module HTTPX
   class Pool
-    using ArrayExtensions::FilterMap
     using URIExtensions
 
     POOL_TIMEOUT = 5
@@ -51,32 +50,53 @@ module HTTPX
         acquire_connection(uri, options) || begin
           if @connections_counter == @max_connections
             # this takes precedence over per-origin
-            @max_connections_cond.wait(@connection_mtx, @pool_timeout)
 
-            if (conn = acquire_connection(uri, options))
-              return conn
-            end
+            expires_at = Utils.now + @pool_timeout
 
-            if @connections_counter == @max_connections
-              # if no matching usable connection was found, the pool will make room and drop a closed connection. if none is found,
-              # this means that all of them are persistent or being used, so raise a timeout error.
-              conn = @connections.find { |c| c.state == :closed }
+            loop do
+              @max_connections_cond.wait(@connection_mtx, @pool_timeout)
 
-              raise PoolTimeoutError.new(@pool_timeout,
-                                         "Timed out after #{@pool_timeout} seconds while waiting for a connection") unless conn
+              if (conn = acquire_connection(uri, options))
+                return conn
+              end
+
+              # if one can afford to create a new connection, do it
+              break unless @connections_counter == @max_connections
+
+              # if no matching usable connection was found, the pool will make room and drop a closed connection.
+              if (conn = @connections.find { |c| c.state == :closed })
+                drop_connection(conn)
+                break
+              end
+
+              # happens when a condition was signalled, but another thread snatched the available connection before
+              # context was passed back here.
+              next if Utils.now < expires_at
 
-              drop_connection(conn)
+              raise PoolTimeoutError.new(@pool_timeout,
+                                         "Timed out after #{@pool_timeout} seconds while waiting for a connection")
             end
 
           end
 
           if @origin_counters[uri.origin] == @max_connections_per_origin
 
-            @origin_conds[uri.origin].wait(@connection_mtx, @pool_timeout)
+            expires_at = Utils.now + @pool_timeout
+
+            loop do
+              @origin_conds[uri.origin].wait(@connection_mtx, @pool_timeout)
+
+              if (conn = acquire_connection(uri, options))
+                return conn
+              end
+
+              # happens when a condition was signalled, but another thread snatched the available connection before
+              # context was passed back here.
+              next if Utils.now < expires_at
 
-            return acquire_connection(uri, options) ||
-                   raise(PoolTimeoutError.new(@pool_timeout,
-                                              "Timed out after #{@pool_timeout} seconds while waiting for a connection to #{uri.origin}"))
+              raise(PoolTimeoutError.new(@pool_timeout,
+                                         "Timed out after #{@pool_timeout} seconds while waiting for a connection to #{uri.origin}"))
+            end
           end
 
           @connections_counter += 1
@@ -91,6 +111,13 @@ module HTTPX
       return if connection.options.io
 
       @connection_mtx.synchronize do
+        if connection.coalesced? || connection.state == :idle
+          # when connections coalesce
+          drop_connection(connection)
+
+          return
+        end
+
         @connections << connection
 
         @max_connections_cond.signal
@@ -128,10 +155,16 @@ module HTTPX
     end
 
     def checkin_resolver(resolver)
-      @resolver_mtx.synchronize do
-        resolvers = @resolvers[resolver.class]
+      resolver_class = resolver.class
+
+      resolver = resolver.multi
 
-        resolver = resolver.multi
+      # a multi requires all sub-resolvers being closed in order to be
+      # correctly checked back in.
+      return unless resolver.closed?
+
+      @resolver_mtx.synchronize do
+        resolvers = @resolvers[resolver_class]
 
         resolvers << resolver unless resolvers.include?(resolver)
       end
@@ -140,6 +173,7 @@ module HTTPX
     # :nocov:
     def inspect
       "#<#{self.class}:#{object_id} " \
+        "@max_connections=#{@max_connections} " \
         "@max_connections_per_origin=#{@max_connections_per_origin} " \
         "@pool_timeout=#{@pool_timeout} " \
         "@connections=#{@connections.size}>"

data/lib/httpx/request.rb

@@ -14,9 +14,6 @@ module HTTPX
 
     ALLOWED_URI_SCHEMES = %w[https http].freeze
 
-    # default value used for "user-agent" header, when not overridden.
-    USER_AGENT = "httpx.rb/#{VERSION}".freeze # rubocop:disable Style/RedundantFreeze
-
     # the upcased string HTTP verb for this request.
     attr_reader :verb
 
@@ -75,16 +72,6 @@ module HTTPX
       @headers = options.headers.dup
       merge_headers(params.delete(:headers)) if params.key?(:headers)
 
-      @headers["user-agent"] ||= USER_AGENT
-      @headers["accept"]     ||= "*/*"
-
-      # forego compression in the Range request case
-      if @headers.key?("range")
-        @headers.delete("accept-encoding")
-      else
-        @headers["accept-encoding"] ||= options.supported_compression_formats
-      end
-
       @query_params = params.delete(:params) if params.key?(:params)
 
       @body = options.request_body_class.new(@headers, options, **params)
@@ -166,6 +153,9 @@ module HTTPX
     # merges +h+ into the instance of HTTPX::Headers of the request.
     def merge_headers(h)
       @headers = @headers.merge(h)
+      return unless @headers.key?("range")
+
+      @headers.delete("accept-encoding")
     end
 
     # the URI scheme of the request +uri+.

data/lib/httpx/resolver/https.rb

@@ -30,7 +30,8 @@ module HTTPX
       use_get: false,
     }.freeze
 
-    def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close, :terminate, :inflight?, :handle_socket_timeout
+    def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close,
+                   :closed?, :deactivate, :terminate, :inflight?, :handle_socket_timeout
 
     def initialize(_, options)
       super
@@ -52,29 +53,24 @@ module HTTPX
       if @uri_addresses.empty?
         ex = ResolveError.new("Can't resolve DNS server #{@uri.host}")
         ex.set_backtrace(caller)
-        connection.force_reset
+        connection.force_close
         throw(:resolve_error, ex)
       end
 
       resolve(connection)
     end
 
-    # This is already indirectly monitored bt the HTTP connection. In order to skip
-    # monitoring, this method returns <tt>true</tt>.
-    def closed?
-      true
-    end
-
-    def empty?
-      true
-    end
-
     def resolver_connection
       # TODO: leaks connection object into the pool
-      @resolver_connection ||= @current_session.find_connection(@uri, @current_selector,
-                                                                @options.merge(ssl: { alpn_protocols: %w[h2] })).tap do |conn|
-        emit_addresses(conn, @family, @uri_addresses) unless conn.addresses
-      end
+      @resolver_connection ||=
+        @current_session.find_connection(
+          @uri,
+          @current_selector,
+          @options.merge(resolver_class: :system, ssl: { alpn_protocols: %w[h2] })
+        ).tap do |conn|
+          emit_addresses(conn, @family, @uri_addresses) unless conn.addresses
+          conn.on(:force_closed, &method(:force_close))
+        end
     end
 
     private
@@ -111,7 +107,9 @@ module HTTPX
         @connections << connection
       rescue ResolveError, Resolv::DNS::EncodeError => e
         reset_hostname(hostname)
+        throw(:resolve_error, e) if connection.pending.empty?
         emit_resolve_error(connection, connection.peer.host, e)
+        close_or_resolve
       end
     end
 
@@ -121,6 +119,7 @@ module HTTPX
       hostname = @requests.delete(request)
       connection = reset_hostname(hostname)
       emit_resolve_error(connection, connection.peer.host, e)
+      close_or_resolve
     else
       # @type var response: HTTPX::Response
       parse(request, response)
@@ -147,6 +146,7 @@ module HTTPX
 
         unless @queries.value?(connection)
           emit_resolve_error(connection)
+          close_or_resolve
           return
         end
 
@@ -156,10 +156,12 @@ module HTTPX
         connection = reset_hostname(host)
 
         emit_resolve_error(connection)
+        close_or_resolve
       when :decode_error
         host = @requests.delete(request)
         connection = reset_hostname(host)
         emit_resolve_error(connection, connection.peer.host, result)
+        close_or_resolve
       end
     end
 
@@ -169,6 +171,7 @@ module HTTPX
         host = @requests.delete(request)
         connection = reset_hostname(host)
         emit_resolve_error(connection)
+        close_or_resolve
         return
 
       else
@@ -207,9 +210,7 @@ module HTTPX
           catch(:coalesced) { emit_addresses(connection, @family, addresses.map { |a| Resolver::Entry.new(a["data"], a["TTL"]) }) }
         end
       end
-      return if @connections.empty?
-
-      resolve
+      close_or_resolve(true)
     end
 
     def build_request(hostname)
@@ -252,5 +253,20 @@ module HTTPX
 
       connection
     end
+
+    def close_or_resolve(should_deactivate = false)
+      # drop already closed connections
+      @connections.shift until @connections.empty? || @connections.first.state != :closed
+
+      if (@connections - @queries.values).empty?
+        if should_deactivate
+          deactivate
+        else
+          disconnect
+        end
+      else
+        resolve
+      end
+    end
   end
 end

data/lib/httpx/resolver/multi.rb

@@ -5,9 +5,6 @@ require "resolv"
 
 module HTTPX
   class Resolver::Multi
-    include Callbacks
-    using ArrayExtensions::FilterMap
-
     attr_reader :resolvers, :options
 
     def initialize(resolver_type, options)
@@ -28,42 +25,22 @@ module HTTPX
 
     def current_selector=(s)
       @current_selector = s
-      @resolvers.each { |r| r.__send__(__method__, s) }
+      @resolvers.each { |r| r.current_selector = s }
     end
 
     def current_session=(s)
       @current_session = s
-      @resolvers.each { |r| r.__send__(__method__, s) }
+      @resolvers.each { |r| r.current_session = s }
     end
 
     def log(*args, **kwargs, &blk)
-      @resolvers.each { |r| r.__send__(__method__, *args, **kwargs, &blk) }
+      @resolvers.each { |r| r.log(*args, **kwargs, &blk) }
     end
 
     def closed?
       @resolvers.all?(&:closed?)
     end
 
-    def empty?
-      @resolvers.all?(&:empty?)
-    end
-
-    def inflight?
-      @resolvers.any(&:inflight?)
-    end
-
-    def timeout
-      @resolvers.filter_map(&:timeout).min
-    end
-
-    def close
-      @resolvers.each(&:close)
-    end
-
-    def connections
-      @resolvers.filter_map { |r| r.resolver_connection if r.respond_to?(:resolver_connection) }
-    end
-
     def early_resolve(connection)
       hostname = connection.peer.host
       addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
@@ -92,9 +69,13 @@ module HTTPX
 
     def lazy_resolve(connection)
       @resolvers.each do |resolver|
-        resolver << @current_session.try_clone_connection(connection, @current_selector, resolver.family)
+        conn_to_resolve = @current_session.try_clone_connection(connection, @current_selector, resolver.family)
+        resolver << conn_to_resolve
+
         next if resolver.empty?
 
+        # both the resolver and the connection it's resolving must be pineed to the session
+        @current_session.pin(conn_to_resolve, @current_selector)
         @current_session.select_resolver(resolver, @current_selector)
       end
     end