httpx 1.8.1 → 1.8.4

data/lib/httpx/plugins/follow_redirects.rb

@@ -75,9 +75,12 @@ module HTTPX
 
           max_redirects = redirect_request.max_redirects
 
-          return response unless response.is_a?(Response)
-          return response unless REDIRECT_STATUS.include?(response.status) && response.headers.key?("location")
-          return response unless max_redirects.positive?
+          return response unless response.is_a?(Response) &&
+                                 # it's a redirect response with location
+                                 REDIRECT_STATUS.include?(response.status) &&
+                                 response.headers.key?("location") &&
+                                 # maximum number of redirects hasn't been reached yet
+                                 max_redirects.positive?
 
           redirect_uri = __get_location_from_response(response)
 
@@ -135,9 +138,9 @@ module HTTPX
             return ErrorResponse.new(request, error)
           end
 
-          retry_request = build_request(redirect_method, redirect_uri, redirect_params, options)
+          redirect_request = build_request(redirect_method, redirect_uri, redirect_params, options)
 
-          request.redirect_request = retry_request
+          request.redirect_request = redirect_request
 
           redirect_after = response.headers["retry-after"]
 
@@ -151,24 +154,24 @@ module HTTPX
             redirect_after = Utils.parse_retry_after(redirect_after)
 
             retry_start = Utils.now
-            log { "redirecting after #{redirect_after} secs..." }
+            redirect_request.log { "redirecting after #{redirect_after} secs..." }
             selector.after(redirect_after) do
               if (response = request.response)
                 response.finish!
-                retry_request.response = response
+                redirect_request.response = response
                 # request has terminated abruptly meanwhile
-                retry_request.emit_response(response)
+                redirect_request.emit_response(response)
               else
-                log { "redirecting (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
-                send_request(retry_request, selector, options)
+                redirect_request.log { "redirecting (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
+                send_request(redirect_request, selector, options)
               end
             end
           else
-            send_request(retry_request, selector, options)
+            send_request(redirect_request, selector, options)
 
             # recalling itself, in case an error was triggered by the above, and we can
             # verify retriability again.
-            return fetch_response(request, selector, options)
+            return fetch_response(redirect_request, selector, options)
           end
           nil
         end
@@ -190,8 +193,7 @@ module HTTPX
 
         # :nodoc:
         def __get_location_from_response(response)
-          # @type var location_uri: http_uri
-          location_uri = URI(response.headers["location"])
+          location_uri = URI(response.headers["location"]) #: http_uri
           location_uri = response.uri.merge(location_uri) if location_uri.relative?
           location_uri
         end
@@ -231,6 +233,18 @@ module HTTPX
           @redirect_request.response
         end
 
+        def response=(response)
+          return super unless @redirect_request && @response.nil? # rubocop:disable Lint/ReturnInVoidContext
+
+          @redirect_request.response = response
+        end
+
+        def emit_response(response)
+          return super unless @redirect_request && @response.nil?
+
+          @redirect_request.emit_response(response)
+        end
+
         def max_redirects
           @options.max_redirects || MAX_REDIRECTS
         end

data/lib/httpx/plugins/grpc/call.rb

@@ -48,12 +48,12 @@ module HTTPX
           end
         end
 
-        def respond_to_missing?(meth, *args, &blk)
-          grpc_response.respond_to?(meth, *args) || super
+        def respond_to_missing?(meth, ...)
+          grpc_response.respond_to?(meth, ...) || super
         end
 
-        def method_missing(meth, *args, &blk)
-          return grpc_response.__send__(meth, *args, &blk) if grpc_response.respond_to?(meth)
+        def method_missing(meth, ...)
+          return grpc_response.__send__(meth, ...) if grpc_response.respond_to?(meth)
 
           super
         end

data/lib/httpx/plugins/grpc/grpc_encoding.rb

@@ -78,8 +78,8 @@ module HTTPX
         end
       end
 
-      def self.encode(*args, **kwargs)
-        Deflater.new(*args, **kwargs)
+      def self.encode(...)
+        Deflater.new(...)
       end
 
       def self.decode(response)

data/lib/httpx/plugins/oauth.rb

@@ -296,9 +296,11 @@ module HTTPX
 
         private
 
-        def generate_auth_token
-          return unless @oauth_session
+        def generate_auth_token(*)
+          return super unless @oauth_session
 
+          # should_regenerate arg ignored, as there's no way to check
+          # for token expiration yet.
           @oauth_session.fetch_access_token(self)
         end
 

data/lib/httpx/plugins/persistent.rb

@@ -18,6 +18,12 @@ module HTTPX
     # https://gitlab.com/os85/httpx/wikis/Persistent
     #
     module Persistent
+      SAFE_REONNECTABLE_ERRORS = [
+        PingTimeoutError,
+        Connection::HTTP2::GoawayError,
+        Connection::HTTP2::PingError,
+      ].freeze
+
       class << self
         def load_dependencies(klass)
           klass.plugin(:fiber_concurrency)
@@ -59,6 +65,15 @@ module HTTPX
           Retries::RECONNECTABLE_ERRORS.any? { |klass| error.is_a?(klass) }
         end
 
+        # whether the error can be safely retried without booking threshold attempts.
+        def safe_reconnectable_error?(error)
+          SAFE_REONNECTABLE_ERRORS.any? { |klass| error.is_a?(klass) }
+        end
+
+        def can_reconnect?(_, response)
+          super || (response.is_a?(ErrorResponse) && safe_reconnectable_error?(response.error))
+        end
+
         def when_to_retry(request, response, *)
           return super unless response.is_a?(ErrorResponse)
 

data/lib/httpx/plugins/proxy/http.rb

@@ -46,6 +46,11 @@ module HTTPX
         end
 
         module ConnectionMethods
+          def initialize(*)
+            @tunnel_parser = nil
+            super
+          end
+
           def force_close(*)
             if @state == :connecting
               # proxy connect related requests should not be reenqueed
@@ -59,6 +64,12 @@ module HTTPX
 
           private
 
+          def purge_after_closed
+            super
+
+            @tunnel_parser = nil
+          end
+
           def handle_transition(nextstate)
             return super unless @options.proxy && @options.proxy.uri.scheme == "http"
 
@@ -69,8 +80,15 @@ module HTTPX
               @io.connect
               return unless @io.connected?
 
-              @parser || begin
-                @parser = parser = parser_type(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
+              @tunnel_parser || begin
+                if @parser
+                  # in case this happens, requests from @parser need to be reenqueued for delivery.
+                  enqueue_pending_requests_from_parser(@parser)
+                  @parser = nil
+                end
+
+                @tunnel_parser = @parser = parser =
+                                   parser_type(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
                 parser.extend(ProxyParser)
                 parser.on(:response, &method(:__http_on_connect))
                 parser.on(:close) do
@@ -86,6 +104,9 @@ module HTTPX
 
                     initial_state = @state
 
+                    # because #reset will nillify @parser...
+                    tunnel_parser = @parser
+
                     reset
 
                     if @pending.empty?
@@ -98,8 +119,8 @@ module HTTPX
                     connect_request = parser = nil
 
                     if initial_state == :connecting
-                      parser = @parser
-                      @parser.reset
+                      parser = tunnel_parser
+                      parser.reset
                       if @pending.first.is_a?(ConnectRequest)
                         connect_request = @pending.shift # this happened when reenqueing
                       end
@@ -107,7 +128,7 @@ module HTTPX
 
                     idling
 
-                    @parser = parser
+                    @tunnel_parser = @parser = parser
                     if connect_request
                       @inflight += 1
                       parser.send(connect_request)
@@ -124,7 +145,7 @@ module HTTPX
               case @state
               when :connecting
                 parser = @parser
-                @parser = nil
+                @tunnel_parser = @parser = nil
                 parser.close
               when :idle
                 @parser.callbacks.clear

data/lib/httpx/plugins/proxy.rb

@@ -92,16 +92,16 @@ module HTTPX
           @authenticator = load_authenticator(scheme, @username, @password)
         end
 
-        def can_authenticate?(*args)
+        def can_authenticate?(...)
           return false unless @authenticator && @authenticator.respond_to?(:can_authenticate?)
 
-          @authenticator.can_authenticate?(*args)
+          @authenticator.can_authenticate?(...)
         end
 
-        def authenticate(*args)
+        def authenticate(...)
           return unless @authenticator
 
-          @authenticator.authenticate(*args)
+          @authenticator.authenticate(...)
         end
 
         def ==(other)
@@ -182,9 +182,11 @@ module HTTPX
           if (no_proxy = proxy.no_proxy)
             no_proxy = no_proxy.join(",") if no_proxy.is_a?(Array)
 
-            # TODO: setting proxy to nil leaks the connection object in the pool
-            return super(request_uri, selector, options.merge(proxy: nil)) unless URI::Generic.use_proxy?(request_uri.host, next_proxy.host,
-                                                                                                          next_proxy.port, no_proxy)
+            unless no_proxy != "*" && # NO_PROXY=* bypasses proxy use
+                   URI::Generic.use_proxy?(request_uri.host, next_proxy.host, next_proxy.port, no_proxy)
+              # TODO: setting proxy to nil leaks the connection object in the pool
+              return super(request_uri, selector, options.merge(proxy: nil))
+            end
           end
 
           super(request_uri, selector, options.merge(proxy: proxy))
@@ -341,6 +343,10 @@ module HTTPX
         module InstanceMethods
           private
 
+          def proxy_error?(request, response, _)
+            super && !request.retries.positive?
+          end
+
           def retryable_error?(ex, *)
             super || ex.is_a?(ProxyConnectionError)
           end

data/lib/httpx/plugins/retries.rb

@@ -29,10 +29,23 @@ module HTTPX
         Errno::ETIMEDOUT,
         ConnectionError,
         TLSError,
-        Connection::HTTP2::Error,
+        Zlib::BufError,
+        PingTimeoutError,
+        # HTTP/2 GOAWAY or RST_STREAM errors guaranteed to be retriable for all
+        # types of requests.
+        Connection::HTTP2::RefusedStreamError,
+        Connection::HTTP2::GoawayError,
+        Connection::HTTP2::PingError,
+
+        # happens if connections were somehow sent to a closed or closing
+        # HTTP2::Connection, request guaranteed to not have been processed.
+        ::HTTP2::Error::ConnectionClosed,
       ].freeze
 
       RETRYABLE_ERRORS = (RECONNECTABLE_ERRORS + [
+        # most RST_STREAM errors aren't safely retriable, as request may have
+        # been partially processed.
+        Connection::HTTP2::RstStreamError,
         Parser::Error,
         TimeoutError,
       ]).freeze
@@ -148,10 +161,13 @@ module HTTPX
              retryable_response?(response, options)
             try_partial_retry(request, response)
             log { "failed to get response, #{request.retries} tries to go..." }
-            prepare_to_retry(request, response)
 
-            if (retry_after = when_to_retry(request, response, options)) && retry_after.positive?
+            # retry-after must be calculated before prepare_to_retry, as it relies on
+            # state changed byit.
+            retry_after = when_to_retry(request, response, options)
+            prepare_to_retry(request, response)
 
+            if retry_after&.positive?
               retry_start = Utils.now
               log { "retrying after #{retry_after} secs..." }
               selector.after(retry_after) do
@@ -191,13 +207,14 @@ module HTTPX
           RETRYABLE_ERRORS.any? { |klass| ex.is_a?(klass) } && !ex.is_a?(TotalRequestTimeoutError)
         end
 
-        def proxy_error?(request, response, _)
-          super && !request.retries.positive?
+        def prepare_to_retry(request, response)
+          request.retries -= 1 unless can_reconnect?(request, response)
+          request.transition(:idle)
         end
 
-        def prepare_to_retry(request, _response)
-          request.retries -= 1 unless request.ping? # do not exhaust retries on connection liveness probes
-          request.transition(:idle)
+        # do not exhaust retries on connection liveness probes
+        def can_reconnect?(request, _)
+          request.ping?
         end
 
         def when_to_retry(request, response, options)
@@ -249,7 +266,7 @@ module HTTPX
         attr_writer :partial_response
 
         # initializes the request instance, sets the number of retries for the request.
-        def initialize(*args)
+        def initialize(*)
           super
           @retries = @options.max_retries
           @partial_response = nil

data/lib/httpx/plugins/server_sent_events.rb

@@ -40,7 +40,7 @@ module HTTPX
       end
 
       module InstanceMethods
-        def request(*args, **options)
+        def request(*, **options)
           options[:stream] = true if options[:event_stream]
 
           super
@@ -146,7 +146,7 @@ module HTTPX
           def when_to_retry(request, *)
             retry_after = request.last_server_sent_message&.retry_after
 
-            retry_after / 1_000.0 if retry_after # original in milliseconds
+            retry_after.to_i / 1_000.0 if retry_after # original in milliseconds
 
             request.last_server_sent_message&.retry_after && super
           end

data/lib/httpx/plugins/stream.rb

@@ -111,10 +111,10 @@ module HTTPX
       end || super
     end
 
-    def method_missing(meth, *args, **kwargs, &block)
+    def method_missing(meth, ...)
       return super unless response.respond_to?(meth)
 
-      response.__send__(meth, *args, **kwargs, &block)
+      response.__send__(meth, ...)
     end
   end
 

data/lib/httpx/plugins/tracing.rb

@@ -36,9 +36,9 @@ module HTTPX::Plugins
       %i[start finish reset enabled?].each do |callback|
         class_eval(<<-OUT, __FILE__, __LINE__ + 1)
           # proxies ##{callback} calls to wrapper tracers.
-          def #{callback}(*args)                        # def start(*args)
-            @tracers.each { |t| t.#{callback}(*args) }  # @tracers.each { |t| t.start(*args) }
-          end                                           # end
+          def #{callback}(...)                        # def start(...)
+            @tracers.each { |t| t.#{callback}(...) }  # @tracers.each { |t| t.start(....) }
+          end                                         # end
         OUT
       end
     end
@@ -108,11 +108,13 @@ module HTTPX::Plugins
       def initialize(*)
         super
 
-        @init_time = ::Time.now.utc
+        @init_time = nil
       end
 
       def send_request_to_parser(request)
         if connecting?
+          @init_time ||= ::Time.now.utc
+
           # request span timeframe should include the time it took to connect.
           request.init_time ||= @init_time
         end
@@ -123,13 +125,24 @@ module HTTPX::Plugins
       def idling
         super
 
-        # time of initial request(s) is accounted from the moment
-        # the connection is back to :idle, and ready to connect again.
-        @init_time = ::Time.now.utc
+        @init_time = nil
+      end
+
+      def terminate
+        super
+
+        # ensure that connections which go back to the pool reset their init time.
+        @init_time = nil
       end
 
       private
 
+      def connect
+        @init_time ||= ::Time.now.utc
+
+        super
+      end
+
       def ping(request)
         # if a connection is probed for liveness, the request timeframe should include
         # it too.

data/lib/httpx/request/body.rb

@@ -4,14 +4,17 @@ module HTTPX
   # Implementation of the HTTP Request body as a delegator which iterates (responds to +each+) payload chunks.
   class Request::Body < SimpleDelegator
     class << self
-      def new(_, options, body: nil, **params)
-        if body.is_a?(self)
+      def new(h, options, body: nil, **params)
+        case body
+        when self
           # request derives its options from body
           body.options = options.merge(params)
-          return body
+          body
+        when nil
+          super(h, options, **params)
+        else
+          super
         end
-
-        super
       end
     end
 

data/lib/httpx/request.rb

@@ -105,9 +105,9 @@ module HTTPX
 
       @state = :idle
       @connection = @response =
-        @drainer = @peer_address =
-          @informational_status = @on_response_arrived = nil
-      @ping = @started = false
+        @drainer = @peer_address = @callbacks =
+                     @informational_status = @on_response_arrived = nil
+      @ping = @started = @complete = false
       @persistent = @options.persistent
       @active_timeouts = []
     end
@@ -115,14 +115,17 @@ module HTTPX
     # dupped initialization
     def initialize_dup(orig)
       super
-      @uri = orig.instance_variable_get(:@uri).dup
       @headers = orig.instance_variable_get(:@headers).dup
       @body = orig.instance_variable_get(:@body).dup
+      @active_timeouts = orig.instance_variable_get(:@active_timeouts).dup
     end
 
     def complete!(response = @response)
+      return false if @complete
+
       emit(:complete, response)
       reset_timers(true)
+      @complete = true
     end
 
     # whether request has been buffered with a ping
@@ -296,7 +299,7 @@ module HTTPX
       case nextstate
       when :idle
         @body.rewind
-        @ping = false
+        @ping = @complete = false
         @response = @drainer = nil
 
         # request may be sent to a different connection and will be

data/lib/httpx/resolver/multi.rb

@@ -35,8 +35,8 @@ module HTTPX
       @resolvers.each { |r| r.current_session = s }
     end
 
-    def log(*args, **kwargs, &blk)
-      @resolvers.each { |r| r.log(*args, **kwargs, &blk) }
+    def log(...)
+      @resolvers.each { |r| r.log(...) }
     end
 
     def closed?

data/lib/httpx/resolver/native.rb

@@ -539,6 +539,10 @@ module HTTPX
         return unless @io.connected?
 
         resolve if @queries.empty? && !@connections.empty?
+
+        # #resolve may have closed the resolver already as part of error handling.
+        # @fiber-switch-guard
+        return unless @io
       when :closed
         return if @state == :closed
 

data/lib/httpx/resolver/resolver.rb

@@ -56,9 +56,9 @@ module HTTPX
       call
     end
 
-    def force_close(*args)
+    def force_close(...)
       while (connection = @connections.shift)
-        connection.force_close(*args)
+        connection.force_close(...)
       end
     end
 

data/lib/httpx/response/body.rb

@@ -69,7 +69,7 @@ module HTTPX
     end
 
     # reads a chunk from the payload (implementation of the IO reader protocol).
-    def read(*args)
+    def read(...)
       return unless @buffer
 
       unless @reader
@@ -77,7 +77,7 @@ module HTTPX
         @reader = @buffer
       end
 
-      @reader.read(*args)
+      @reader.read(...)
     end
 
     # size of the decoded response payload. May differ from "content-length" header if

data/lib/httpx/response.rb

@@ -36,10 +36,10 @@ module HTTPX
     # * HTTPX::Response::Body#read
     # * HTTPX::Response::Body#copy_to
     # * HTTPX::Response::Body#close
-    attr_reader  :body
+    attr_reader :body
 
     # The HTTP protocol version used to fetch the response.
-    attr_reader  :version
+    attr_reader :version
 
     # returns the response body buffered in a string.
     def_delegator :@body, :to_s
@@ -68,7 +68,7 @@ module HTTPX
       @headers = @options.headers_class.new(headers)
       @body = @options.response_body_class.new(self, @options)
       @finished = complete?
-      @content_type = @content_length = nil
+      @callbacks = @content_type = @content_length = nil
     end
 
     # dupped initialization
@@ -176,8 +176,8 @@ module HTTPX
     #
     #   response.json #≈> { "foo" => "bar" } for "{\"foo\":\"bar\"}" payload
     #   response.json(symbolize_names: true) #≈> { foo: "bar" } for "{\"foo\":\"bar\"}" payload
-    def json(*args)
-      decode(Transcoder::JSON, *args)
+    def json(...)
+      decode(Transcoder::JSON, ...)
     end
 
     # decodes the response payload into a ruby object **if** the payload is valid
@@ -200,7 +200,7 @@ module HTTPX
     #
     # +transcoder+ must implement the internal transcoder API, i.e. respond to <tt>decode(HTTPX::Response response)</tt>,
     # which returns a decoder which responds to <tt>call(HTTPX::Response response, **kwargs)</tt>
-    def decode(transcoder, *args)
+    def decode(transcoder, ...)
       # TODO: check if content-type is a valid format, i.e. "application/json" for json parsing
 
       decoder = transcoder.decode(self)
@@ -209,7 +209,7 @@ module HTTPX
 
       @body.rewind
 
-      decoder.call(self, *args)
+      decoder.call(self, ...)
     end
   end
 

data/lib/httpx/selector.rb

@@ -56,7 +56,7 @@ module HTTPX
 
         begin
           select(timeout) do |c|
-            c.log(level: 2) { "[#{c.state}] selected from selector##{object_id} #{" after #{timeout} secs" unless timeout.nil?}..." }
+            c.log(level: 2) { "[#{c.state}] selected from selector##{object_id}#{" after #{timeout} secs" unless timeout.nil?}..." }
 
             c.call
           end

data/lib/httpx/session.rb

@@ -237,7 +237,7 @@ module HTTPX
 
       return unless response && response.finished?
 
-      log(level: 2) { "response##{response.object_id} fetched" }
+      request.log(level: 2) { "response##{response.object_id} fetched" }
 
       response
     end
@@ -337,6 +337,8 @@ module HTTPX
         end
       end
 
+      log(level: 2) { "waiting to receive #{pending} pending requests..." }
+
       until pending.zero? || selector.empty?
         # loop on selector until at least one response has been received.
         waiting = true
@@ -353,14 +355,16 @@ module HTTPX
           # making the next loop cheaper (because we're dropping).
           next unless response
 
-          request.complete!(response)
+          next unless request.complete!(response)
+
           responses[idx] = response
           request.on_response_arrived = nil
           pending -= 1
         end
       end
 
-      raise Error, "something went wrong, responses not found and requests not resent" unless pending.zero?
+      raise Error, "something went wrong, #{pending} responses not found " \
+                   "and requests not resent" unless pending.zero?
 
       responses
     end