httpx 1.7.2 → 1.7.6

data/lib/httpx/plugins/proxy/http.rb

@@ -35,7 +35,10 @@ module HTTPX
               request.headers["proxy-authorization"] =
                 options.proxy.authenticate(request, response.headers["proxy-authenticate"])
               send_request(request, selector, options)
-              return
+
+              # recalling itself, in case an error was triggered by the above, and we can
+              # verify retriability again.
+              return fetch_response(request, selector, options)
             end
 
             response
@@ -46,7 +49,7 @@ module HTTPX
           def force_close(*)
             if @state == :connecting
               # proxy connect related requests should not be reenqueed
-              @parser.reset!
+              @parser.reset
               @inflight -= @parser.pending.size
               @parser.pending.clear
             end
@@ -67,18 +70,16 @@ module HTTPX
               return unless @io.connected?
 
               @parser || begin
-                @parser = parser_type(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
-                parser = @parser
+                @parser = parser = parser_type(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
                 parser.extend(ProxyParser)
                 parser.on(:response, &method(:__http_on_connect))
                 parser.on(:close) do
                   next unless @parser
 
                   reset
-                  disconnect
                 end
                 parser.on(:reset) do
-                  if parser.empty?
+                  if parser.pending.empty? && parser.empty?
                     reset
                   else
                     enqueue_pending_requests_from_parser(parser)
@@ -94,17 +95,23 @@ module HTTPX
                     # keep parser state around due to proxy auth protocol;
                     # intermediate authenticated request is already inside
                     # the parser
-                    parser = nil
+                    connect_request = parser = nil
 
                     if initial_state == :connecting
                       parser = @parser
                       @parser.reset
+                      if @pending.first.is_a?(ConnectRequest)
+                        connect_request = @pending.shift # this happened when reenqueing
+                      end
                     end
 
                     idling
 
                     @parser = parser
-
+                    if connect_request
+                      @inflight += 1
+                      parser.send(connect_request)
+                    end
                     transition(:connecting)
                   end
                 end

data/lib/httpx/plugins/proxy.rb

@@ -202,7 +202,10 @@ module HTTPX
               log { "failed connecting to proxy, trying next..." }
               request.transition(:idle)
               send_request(request, selector, options)
-              return
+
+              # recalling itself, in case an error was triggered by the above, and we can
+              # verify retriability again.
+              return fetch_response(request, selector, options)
             end
             response
           rescue ProxyError
@@ -320,7 +323,12 @@ module HTTPX
 
         def purge_after_closed
           super
-          @io = @io.proxy_io if @io.respond_to?(:proxy_io)
+
+          while @io.respond_to?(:proxy_io)
+            @io = @io.proxy_io
+
+            super
+          end
         end
       end
 

data/lib/httpx/plugins/rate_limiter.rb

@@ -16,25 +16,7 @@ module HTTPX
 
       class << self
         def load_dependencies(klass)
-          klass.plugin(:retries, retry_after: method(:retry_after_rate_limit))
-        end
-
-        # Servers send the "Retry-After" header field to indicate how long the
-        # user agent ought to wait before making a follow-up request.  When
-        # sent with a 503 (Service Unavailable) response, Retry-After indicates
-        # how long the service is expected to be unavailable to the client.
-        # When sent with any 3xx (Redirection) response, Retry-After indicates
-        # the minimum time that the user agent is asked to wait before issuing
-        # the redirected request.
-        #
-        def retry_after_rate_limit(_, response)
-          return unless response.is_a?(Response)
-
-          retry_after = response.headers["retry-after"]
-
-          return unless retry_after
-
-          Utils.parse_retry_after(retry_after)
+          klass.plugin(:retries)
         end
       end
 
@@ -52,6 +34,24 @@ module HTTPX
         def rate_limit_error?(response)
           response.is_a?(Response) && RATE_LIMIT_CODES.include?(response.status)
         end
+
+        # Servers send the "Retry-After" header field to indicate how long the
+        # user agent ought to wait before making a follow-up request.  When
+        # sent with a 503 (Service Unavailable) response, Retry-After indicates
+        # how long the service is expected to be unavailable to the client.
+        # When sent with any 3xx (Redirection) response, Retry-After indicates
+        # the minimum time that the user agent is asked to wait before issuing
+        # the redirected request.
+        #
+        def when_to_retry(_, response, options)
+          return super unless response.is_a?(Response)
+
+          retry_after = response.headers["retry-after"]
+
+          return super unless retry_after
+
+          Utils.parse_retry_after(retry_after)
+        end
       end
     end
 

data/lib/httpx/plugins/retries.rb

@@ -148,10 +148,7 @@ module HTTPX
             log { "failed to get response, #{request.retries} tries to go..." }
             prepare_to_retry(request, response)
 
-            retry_after = options.retry_after
-            retry_after = retry_after.call(request, response) if retry_after.respond_to?(:call)
-
-            if retry_after
+            if (retry_after = when_to_retry(request, response, options)) && retry_after.positive?
               # apply jitter
               if (jitter = request.options.retry_jitter)
                 retry_after = jitter.call(retry_after)
@@ -169,11 +166,15 @@ module HTTPX
                   send_request(request, selector, options)
                 end
               end
+
+              return
             else
               send_request(request, selector, options)
-            end
 
-            return
+              # recalling itself, in case an error was triggered by the above, and we can
+              # verify retriability again.
+              return fetch_response(request, selector, options)
+            end
           end
           response
         end
@@ -201,6 +202,12 @@ module HTTPX
           request.transition(:idle)
         end
 
+        def when_to_retry(request, response, options)
+          retry_after = options.retry_after
+          retry_after = retry_after.call(request, response) if retry_after.respond_to?(:call)
+          retry_after
+        end
+
         #
         # Attempt to set the request to perform a partial range request.
         # This happens if the peer server accepts byte-range requests, and
@@ -237,14 +244,15 @@ module HTTPX
         def initialize(*args)
           super
           @retries = @options.max_retries
+          @partial_response = nil
         end
 
         def response=(response)
-          if @partial_response
+          if (partial_response = @partial_response)
             if response.is_a?(Response) && response.status == 206
-              response.from_partial_response(@partial_response)
+              response.from_partial_response(partial_response)
             else
-              @partial_response.close
+              partial_response.close
             end
             @partial_response = nil
           end

data/lib/httpx/plugins/ssrf_filter.rb

@@ -106,6 +106,7 @@ module HTTPX
             error = ServerSideRequestForgeryError.new("#{request.uri} URI scheme not allowed")
             error.set_backtrace(caller)
             response = ErrorResponse.new(request, error)
+            request.response = response
             request.emit(:response, response)
             response
           end

data/lib/httpx/plugins/stream_bidi.rb

@@ -189,6 +189,10 @@ module HTTPX
           !@closed
         end
 
+        def force_close(*)
+          terminate
+        end
+
         def terminate
           return if @closed
 
@@ -202,6 +206,8 @@ module HTTPX
           terminate
         end
 
+        alias_method :on_io_error, :on_error
+
         # noop (the owner connection will take of it)
         def handle_socket_timeout(interval); end
       end

data/lib/httpx/plugins/tracing.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module HTTPX::Plugins
+  #
+  # This plugin adds a simple interface to integrate request tracing SDKs.
+  #
+  # An example of such an integration is the datadog adapter.
+  #
+  # https://gitlab.com/os85/httpx/wikis/Tracing
+  #
+  module Tracing
+    class Wrapper
+      attr_reader :tracers
+      protected :tracers
+
+      def initialize(*tracers)
+        @tracers = tracers.flat_map do |tracer|
+          case tracer
+          when Wrapper
+            tracer.tracers
+          else
+            tracer
+          end
+        end.uniq
+      end
+
+      def merge(tracer)
+        case tracer
+        when Wrapper
+          Wrapper.new(*@tracers, *tracer.tracers)
+        else
+          Wrapper.new(*@tracers, tracer)
+        end
+      end
+
+      def freeze
+        @tracers.each(&:freeze).freeze
+        super
+      end
+
+      %i[start finish reset enabled?].each do |callback|
+        class_eval(<<-OUT, __FILE__, __LINE__ + 1)
+          # proxies ##{callback} calls to wrapper tracers.
+          def #{callback}(*args)                        # def start(*args)
+            @tracers.each { |t| t.#{callback}(*args) }  # @tracers.each { |t| t.start(*args) }
+          end                                           # end
+        OUT
+      end
+    end
+
+    # adds support for the following options:
+    #
+    # :tracer :: object which responds to #start, #finish and #reset.
+    module OptionsMethods
+      private
+
+      def option_tracer(tracer)
+        unless tracer.respond_to?(:start) &&
+               tracer.respond_to?(:finish) &&
+               tracer.respond_to?(:reset) &&
+               tracer.respond_to?(:enabled?)
+          raise TypeError, "#{tracer} must to respond to `#start(r)`, `#finish` and `#reset` and `#enabled?"
+        end
+
+        tracer = Wrapper.new(@tracer, tracer) if @tracer
+        tracer
+      end
+    end
+
+    module RequestMethods
+      attr_accessor :init_time
+
+      # intercepts request initialization to inject the tracing logic.
+      def initialize(*)
+        super
+
+        @init_time = nil
+
+        tracer = @options.tracer
+
+        return unless tracer && tracer.enabled?(self)
+
+        on(:idle) do
+          tracer.reset(self)
+
+          # request is reset when it's retried.
+          @init_time = nil
+        end
+        on(:headers) do
+          # the usual request init time (when not including the connection handshake)
+          # should be the time the request is buffered the first time.
+          @init_time ||= ::Time.now.utc
+
+          tracer.start(self)
+        end
+        on(:response) { |response| tracer.finish(self, response) }
+      end
+
+      def response=(*)
+        # init_time should be set when it's send to a connection.
+        # However, there are situations where connection initialization fails.
+        # Example is the :ssrf_filter plugin, which raises an error on
+        # initialize if the host is an IP which matches against the known set.
+        # in such cases, we'll just set here right here.
+        @init_time ||= ::Time.now.utc
+
+        super
+      end
+    end
+
+    # Connection mixin
+    module ConnectionMethods
+      def initialize(*)
+        super
+
+        @init_time = ::Time.now.utc
+      end
+
+      def send(request)
+        # request init time is only the same as the connection init time
+        # if the connection is going through the connection handshake.
+        request.init_time ||= @init_time unless open?
+
+        super
+      end
+
+      def idling
+        super
+
+        # time of initial request(s) is accounted from the moment
+        # the connection is back to :idle, and ready to connect again.
+        @init_time = ::Time.now.utc
+      end
+    end
+  end
+  register_plugin :tracing, Tracing
+end

data/lib/httpx/pool.rb

@@ -122,12 +122,6 @@ module HTTPX
 
         @max_connections_cond.signal
         @origin_conds[connection.origin.to_s].signal
-
-        # Observed situations where a session handling multiple requests in a loop
-        # across multiple threads checks the same connection in and out, while another
-        # thread which is waiting on the same connection never gets the chance to pick
-        # it up, because ruby's thread scheduler never switched on to it in the process.
-        Thread.pass
       end
     end
 
@@ -153,16 +147,20 @@ module HTTPX
         resolvers = @resolvers[resolver_type]
 
         idx = resolvers.find_index do |res|
-          res.options == options
+          res.options.resolver_options_match?(options)
         end
         resolvers.delete_at(idx) if idx
       end || checkout_new_resolver(resolver_type, options)
     end
 
     def checkin_resolver(resolver)
-      resolver_class = resolver.class
+      if resolver.is_a?(Resolver::Multi)
+        resolver_class = resolver.resolvers.first.class
+      else
+        resolver_class = resolver.class
 
-      resolver = resolver.multi
+        resolver = resolver.multi
+      end
 
       # a multi requires all sub-resolvers being closed in order to be
       # correctly checked back in.

data/lib/httpx/request.rb

@@ -42,6 +42,9 @@ module HTTPX
     # The IP address from the peer server.
     attr_accessor :peer_address
 
+    # the connection the request is currently being sent to (none if before or after transaction)
+    attr_writer :connection
+
     attr_writer :persistent
 
     attr_reader :active_timeouts
@@ -91,7 +94,7 @@ module HTTPX
       raise UnsupportedSchemeError, "#{@uri}: #{@uri.scheme}: unsupported URI scheme" unless ALLOWED_URI_SCHEMES.include?(@uri.scheme)
 
       @state = :idle
-      @response = @peer_address = @informational_status = nil
+      @connection = @response = @drainer = @peer_address = @informational_status = nil
       @ping = false
       @persistent = @options.persistent
       @active_timeouts = []
@@ -274,8 +277,7 @@ module HTTPX
       when :idle
         @body.rewind
         @ping = false
-        @response = nil
-        @drainer = nil
+        @response = @drainer = nil
         @active_timeouts.clear
       when :headers
         return unless @state == :idle
@@ -321,6 +323,16 @@ module HTTPX
         callbacks(event).delete(clb)
       end
     end
+
+    def handle_error(error)
+      if (connection = @connection)
+        connection.on_error(error, self)
+      else
+        response = ErrorResponse.new(self, error)
+        self.response = response
+        emit(:response, response)
+      end
+    end
   end
 end
 

data/lib/httpx/resolver/multi.rb

@@ -71,14 +71,7 @@ module HTTPX
 
     def lazy_resolve(connection)
       @resolvers.each do |resolver|
-        conn_to_resolve = @current_session.try_clone_connection(connection, @current_selector, resolver.family)
-        resolver << conn_to_resolve
-
-        next if resolver.empty?
-
-        # both the resolver and the connection it's resolving must be pineed to the session
-        @current_session.pin(conn_to_resolve, @current_selector)
-        @current_session.select_resolver(resolver, @current_selector)
+        resolver.lazy_resolve(connection)
       end
     end
 

data/lib/httpx/resolver/native.rb

@@ -66,7 +66,7 @@ module HTTPX
     end
 
     def closed?
-      @state == :closed
+      @state == :idle || @state == :closed
     end
 
     def to_io
@@ -529,7 +529,7 @@ module HTTPX
 
         resolve if @queries.empty? && !@connections.empty?
       when :closed
-        return unless @state == :open
+        return if @state == :closed
 
         @io.close if @io
         @start_timeout = nil

data/lib/httpx/resolver/resolver.rb

@@ -119,6 +119,11 @@ module HTTPX
       end
     end
 
+    def on_io_error(e)
+      on_error(e)
+      force_close(true)
+    end
+
     def on_error(error)
       handle_error(error)
       disconnect
@@ -138,6 +143,19 @@ module HTTPX
       true
     end
 
+    def lazy_resolve(connection)
+      return unless @current_session && @current_selector
+
+      conn_to_resolve = @current_session.try_clone_connection(connection, @current_selector, @family)
+      self << conn_to_resolve
+
+      return if empty?
+
+      # both the resolver and the connection it's resolving must be pinned to the session
+      @current_session.pin(conn_to_resolve, @current_selector)
+      @current_session.select_resolver(self, @current_selector)
+    end
+
     private
 
     def emit_resolved_connection(connection, addresses, early_resolve)
@@ -181,9 +199,10 @@ module HTTPX
     end
 
     def disconnect
-      return if closed?
-
       close
+
+      return unless closed?
+
       @current_session.deselect_resolver(self, @current_selector)
     end
   end

data/lib/httpx/resolver/system.rb

@@ -116,7 +116,9 @@ module HTTPX
       @current_session.select_resolver(self, @current_selector)
     end
 
-    def early_resolve(connection, **); end
+    def early_resolve(_, **) # rubocop:disable Naming/PredicateMethod
+      false
+    end
 
     def handle_socket_timeout(interval)
       error = HTTPX::ResolveTimeoutError.new(interval, "timed out while waiting on select")

data/lib/httpx/response.rb

@@ -112,6 +112,7 @@ module HTTPX
     def finish!
       @finished = true
       @headers.freeze
+      @request.connection = nil
     end
 
     # returns whether the response contains body payload.
@@ -282,6 +283,7 @@ module HTTPX
       @error = error
       @options = request.options
       log_exception(@error)
+      finish!
     end
 
     # returns the exception full message.
@@ -299,7 +301,9 @@ module HTTPX
       true
     end
 
-    def finish!; end
+    def finish!
+      @request.connection = nil
+    end
 
     # raises the wrapped exception.
     def raise_for_status

data/lib/httpx/selector.rb

@@ -29,7 +29,7 @@ module HTTPX
 
     def_delegator :@timers, :after
 
-    def_delegator :@selectables, :empty?
+    def_delegator :@selectables, :each
 
     def initialize
       @timers = Timers.new
@@ -37,8 +37,8 @@ module HTTPX
       @is_timer_interval = false
     end
 
-    def each(&blk)
-      @selectables.each(&blk)
+    def empty?
+      @selectables.empty? && @timers.empty?
     end
 
     def next_tick
@@ -81,7 +81,8 @@ module HTTPX
 
     def find_resolver(options)
       res = @selectables.find do |c|
-        c.is_a?(Resolver::Resolver) && options == c.options
+        c.is_a?(Resolver::Resolver) &&
+          options.resolver_options_match?(c.options)
       end
 
       res.multi if res
@@ -148,16 +149,18 @@ module HTTPX
 
         next(is_closed) if is_closed
 
-        io.log(level: 2) do
-          "[#{io.state}] registering in selector##{object_id} for select (#{interests})#{" for #{interval} seconds" unless interval.nil?}"
-        end
+        if interests
+          io.log(level: 2) do
+            "[#{io.state}] registering in selector##{object_id} for select (#{interests})#{" for #{interval} seconds" unless interval.nil?}"
+          end
 
-        if READABLE.include?(interests)
-          r = r.nil? ? io : (Array(r) << io)
-        end
+          if READABLE.include?(interests)
+            r = r.nil? ? io : (Array(r) << io)
+          end
 
-        if WRITABLE.include?(interests)
-          w = w.nil? ? io : (Array(w) << io)
+          if WRITABLE.include?(interests)
+            w = w.nil? ? io : (Array(w) << io)
+          end
         end
 
         is_closed
@@ -204,8 +207,7 @@ module HTTPX
       rescue IOError => e
         (Array(r) + Array(w)).each do |sel|
           # TODO: is there a way to cheaply find the IO associated with the error?
-          sel.on_error(e)
-          sel.force_close(true)
+          sel.on_io_error(e)
         end
       rescue StandardError => e
         (Array(r) + Array(w)).each do |sel|
@@ -249,8 +251,9 @@ module HTTPX
           when :rw then rw_wait(io, interval)
           end
       rescue IOError => e
-        io.on_error(e)
-        io.force_close(true)
+        io.on_io_error(e)
+
+        return
       rescue StandardError => e
         io.on_error(e)