httpx 1.8.3 → 1.9.0

data/lib/httpx/plugins/proxy.rb

@@ -11,7 +11,7 @@ module HTTPX
     #
     # * HTTP proxies
     # * HTTPS proxies
-    # * Socks4/4a proxies
+    # * Socks4 /Socks4a proxies
     # * Socks5 proxies
     #
     # https://gitlab.com/os85/httpx/wikis/Proxy
@@ -19,6 +19,7 @@ module HTTPX
     module Proxy
       class ProxyConnectionError < ProxyError; end
 
+      # :nodoc:
       PROXY_ERRORS = [TimeoutError, IOError, SystemCallError, Error].freeze
 
       class << self
@@ -39,21 +40,24 @@ module HTTPX
         end
       end
 
+      # Manages the proxy parameters and the proxy authentication flow.
       class Parameters
         attr_reader :uri, :username, :password, :scheme, :no_proxy
 
-        def initialize(uri: nil, scheme: nil, username: nil, password: nil, no_proxy: nil, **extra)
+        def initialize(uri: nil, ns: 0, scheme: nil, username: nil, password: nil, no_proxy: nil, **extra)
           @no_proxy = Array(no_proxy) if no_proxy
           @uris = Array(uri)
-          uri = @uris.first
+          uri = @uris[ns]
 
           @username = username
           @password = password
 
-          @ns = 0
+          @initial_credentials = @username && @password
+
+          @ns = ns
 
           if uri
-            @uri = uri.is_a?(URI::Generic) ? uri : URI(uri)
+            @uri = URI(uri)
             @username ||= @uri.user
             @password ||= @uri.password
           end
@@ -74,34 +78,27 @@ module HTTPX
         end
 
         def shift
-          # TODO: this operation must be synchronized
-          @ns += 1
-          @uri = @uris[@ns]
-
-          return unless @uri
-
-          @uri = URI(@uri) unless @uri.is_a?(URI::Generic)
-
-          scheme = infer_default_auth_scheme(@uri)
-
-          return unless scheme != @scheme
-
-          @scheme = scheme
-          @username = username || @uri.user
-          @password = password || @uri.password
-          @authenticator = load_authenticator(scheme, @username, @password)
+          return if @ns >= @uris.size - 1
+
+          Parameters.new(
+            uri: @uris,
+            ns: @ns + 1,
+            username: (@username if @initial_credentials),
+            password: (@password if @initial_credentials),
+            no_proxy: @no_proxy
+          )
         end
 
-        def can_authenticate?(*args)
+        def can_authenticate?(...)
           return false unless @authenticator && @authenticator.respond_to?(:can_authenticate?)
 
-          @authenticator.can_authenticate?(*args)
+          @authenticator.can_authenticate?(...)
         end
 
-        def authenticate(*args)
+        def authenticate(...)
           return unless @authenticator
 
-          @authenticator.authenticate(*args)
+          @authenticator.authenticate(...)
         end
 
         def ==(other)
@@ -144,8 +141,26 @@ module HTTPX
 
       # adds support for the following options:
       #
-      # :proxy :: proxy options defining *:uri*, *:username*, *:password* or
-      #           *:scheme* (i.e. <tt>{ uri: "http://proxy" }</tt>)
+      # :proxy :: hash of proxy options, where a URI an be passed containing most of the requred information,
+      #           or instead the different proxy options can be passed independently.
+      #
+      # The following proxy options are supported:
+      #
+      # :uri :: proxy origin URI, or list of URIs; in the case of the latter, any failed attempt to connect to a list item
+      #         will retry for the next item of the list, until all elements are exhausted or it times out meanwhile.
+      #         <tt>:username</tt> and <tt>:password</tt> will be inferred from it when available. The URI scheme will identify
+      #         the proxy type.  (ex: <tt>"http://proxi-uri:8980"</tt>, <tt>"http://user:pass@proxy-uri:8181"</tt>,
+      #         <tt>"socks5://user:pass@socks-proxy-uri:10300"</tt>, <tt>["http://proxi-uri:8980", "http://user:pass@proxy-uri:8181"]</tt>).
+      # :scheme :: identifies the authentication scheme to use with the proxy, if necessary; defaults to <tt>"basic"</tt>, for basic auth,
+      #            can also be <tt>"digest"</tt> or <tt>"ntml"</tt>). Your recommended to use session methods like HTTP::InstanceMethods#with_proxy_basic_auth
+      #            to set these instead.
+      # :username :: the username to use when authenticating with the proxy
+      # :password :: the password to use when authenticating with the proxy
+      # :no_proxy :: A list of hostnames (or regex patterns) for which the proxy should be bypassed
+      #              (ex: <tt>["example.com", "localhost", /.*\.google\.com/]</tt>).
+      #
+      # if you systems setting environment variables like `HTTP_PROXY` or `NO_PROXY` though, you won't need to set
+      # most of the above.
       module OptionsMethods
         private
 
@@ -161,25 +176,31 @@ module HTTPX
       end
 
       module InstanceMethods
-        def find_connection(request_uri, selector, options)
-          return super unless options.respond_to?(:proxy)
+        def initialize(*)
+          super
+          @proxy_parameters = @options.proxy
+        end
+
+        def initialize_dup(other)
+          @proxy_parameters = other.instance_variable_get(:@proxy_parameters)
+          super
+        end
 
+        def find_connection(request_uri, selector, options)
           if (next_proxy = request_uri.find_proxy)
             return super(request_uri, selector, options.merge(proxy: Parameters.new(uri: next_proxy)))
           end
 
-          proxy = options.proxy
-
-          return super unless proxy
+          return super unless @proxy_parameters
 
-          next_proxy = proxy.uri
+          next_proxy = @proxy_parameters.uri
 
           raise ProxyError, "Failed to connect to proxy" unless next_proxy
 
           raise ProxyError,
                 "#{next_proxy.scheme}: unsupported proxy protocol" unless options.supported_proxy_protocols.include?(next_proxy.scheme)
 
-          if (no_proxy = proxy.no_proxy)
+          if (no_proxy = @proxy_parameters.no_proxy)
             no_proxy = no_proxy.join(",") if no_proxy.is_a?(Array)
 
             unless no_proxy != "*" && # NO_PROXY=* bypasses proxy use
@@ -189,7 +210,7 @@ module HTTPX
             end
           end
 
-          super(request_uri, selector, options.merge(proxy: proxy))
+          super(request_uri, selector, options.merge(proxy: @proxy_parameters))
         end
 
         private
@@ -201,10 +222,10 @@ module HTTPX
             response = super
 
             if response.is_a?(ErrorResponse) && proxy_error?(request, response, options)
-              options.proxy.shift
+              @proxy_parameters = options.proxy.shift
 
               # return last error response if no more proxies to try
-              return response if options.proxy.uri.nil?
+              return response unless @proxy_parameters
 
               log { "failed connecting to proxy, trying next..." }
               request.transition(:idle)
@@ -222,22 +243,22 @@ module HTTPX
           end
         end
 
-        def proxy_error?(_request, response, options)
-          return false unless options.proxy
+        def proxy_error?(_request, response, _options)
+          proxy_parameters = @proxy_parameters
+
+          return false unless proxy_parameters
 
           error = response.error
           case error
           when NativeResolveError
-            proxy_uri = URI(options.proxy.uri)
+            proxy_uri = proxy_parameters.uri
 
             unresolved_host = error.host
 
             # failed resolving proxy domain
             unresolved_host == proxy_uri.host
           when ResolveError
-            proxy_uri = URI(options.proxy.uri)
-
-            error.message.end_with?(proxy_uri.to_s)
+            error.message.end_with?(proxy_parameters.uri.to_s)
           when ProxyConnectionError
             # timeout errors connecting to proxy
             true
@@ -256,7 +277,7 @@ module HTTPX
 
           # redefining the connection origin as the proxy's URI,
           # as this will be used as the tcp peer ip.
-          @proxy_uri = URI(@options.proxy.uri)
+          @proxy_uri = @options.proxy.uri
         end
 
         def peer
@@ -312,7 +333,7 @@ module HTTPX
           when :idle
             transition(:connecting)
           when :connected
-            transition(:open)
+            super
           end
         end
 

data/lib/httpx/plugins/push_promise.rb

@@ -12,8 +12,10 @@ module HTTPX
     #
     module PushPromise
       def self.extra_options(options)
-        options.merge(http2_settings: { settings_enable_push: 1 },
-                      max_concurrent_requests: 1)
+        options.merge(
+          http2_settings: { settings_enable_push: 1 },
+          max_concurrent_requests: 1
+        )
       end
 
       module ResponseMethods

data/lib/httpx/plugins/retries.rb

@@ -31,11 +31,21 @@ module HTTPX
         TLSError,
         Zlib::BufError,
         PingTimeoutError,
+        # HTTP/2 GOAWAY or RST_STREAM errors guaranteed to be retriable for all
+        # types of requests.
+        Connection::HTTP2::RefusedStreamError,
         Connection::HTTP2::GoawayError,
         Connection::HTTP2::PingError,
+
+        # happens if connections were somehow sent to a closed or closing
+        # HTTP2::Connection, request guaranteed to not have been processed.
+        ::HTTP2::Error::ConnectionClosed,
       ].freeze
 
       RETRYABLE_ERRORS = (RECONNECTABLE_ERRORS + [
+        # most RST_STREAM errors aren't safely retriable, as request may have
+        # been partially processed.
+        Connection::HTTP2::RstStreamError,
         Parser::Error,
         TimeoutError,
       ]).freeze
@@ -155,7 +165,7 @@ module HTTPX
             # retry-after must be calculated before prepare_to_retry, as it relies on
             # state changed byit.
             retry_after = when_to_retry(request, response, options)
-            prepare_to_retry(request, response)
+            prepare_to_retry(request, response, retry_after)
 
             if retry_after&.positive?
               retry_start = Utils.now
@@ -197,7 +207,7 @@ module HTTPX
           RETRYABLE_ERRORS.any? { |klass| ex.is_a?(klass) } && !ex.is_a?(TotalRequestTimeoutError)
         end
 
-        def prepare_to_retry(request, response)
+        def prepare_to_retry(request, response, _retry_after)
           request.retries -= 1 unless can_reconnect?(request, response)
           request.transition(:idle)
         end
@@ -256,7 +266,7 @@ module HTTPX
         attr_writer :partial_response
 
         # initializes the request instance, sets the number of retries for the request.
-        def initialize(*args)
+        def initialize(*)
           super
           @retries = @options.max_retries
           @partial_response = nil

data/lib/httpx/plugins/server_sent_events.rb

@@ -40,7 +40,8 @@ module HTTPX
       end
 
       module InstanceMethods
-        def request(*args, **options)
+        # sends a stream request.
+        def request(*, **options)
           options[:stream] = true if options[:event_stream]
 
           super

data/lib/httpx/plugins/ssrf_filter.rb

@@ -142,7 +142,7 @@ module HTTPX
         end
 
         def addresses=(addrs)
-          addrs.reject! do |ipaddr|
+          addrs = addrs.reject do |ipaddr|
             ipaddr = ipaddr.address
             next false if @options.safe_private_ranges&.any? { |r| r.include?(ipaddr) }
 

data/lib/httpx/plugins/stream.rb

@@ -111,10 +111,10 @@ module HTTPX
       end || super
     end
 
-    def method_missing(meth, *args, **kwargs, &block)
+    def method_missing(meth, ...)
       return super unless response.respond_to?(meth)
 
-      response.__send__(meth, *args, **kwargs, &block)
+      response.__send__(meth, ...)
     end
   end
 

data/lib/httpx/plugins/tracing.rb

@@ -36,9 +36,9 @@ module HTTPX::Plugins
       %i[start finish reset enabled?].each do |callback|
         class_eval(<<-OUT, __FILE__, __LINE__ + 1)
           # proxies ##{callback} calls to wrapper tracers.
-          def #{callback}(*args)                        # def start(*args)
-            @tracers.each { |t| t.#{callback}(*args) }  # @tracers.each { |t| t.start(*args) }
-          end                                           # end
+          def #{callback}(...)                        # def start(...)
+            @tracers.each { |t| t.#{callback}(...) }  # @tracers.each { |t| t.start(....) }
+          end                                         # end
         OUT
       end
     end
@@ -63,13 +63,18 @@ module HTTPX::Plugins
     end
 
     module RequestMethods
+      # time at which a request started being buffered for sending.
       attr_accessor :init_time
 
+      # when not nil, it means this request was sent to a non-open connection, and contains how much time it took
+      # the connection to connect.
+      attr_accessor :handshake_time
+
       # intercepts request initialization to inject the tracing logic.
       def initialize(*)
         super
 
-        @init_time = nil
+        @init_time = @handshake_time = nil
 
         tracer = @options.tracer
 
@@ -79,12 +84,13 @@ module HTTPX::Plugins
           tracer.reset(self)
 
           # request is reset when it's retried.
-          @init_time = nil
+          @init_time = @handshake_time = nil
         end
         on(:headers) do
           # the usual request init time (when not including the connection handshake)
           # should be the time the request is buffered the first time.
-          @init_time ||= ::Time.now.utc
+          @init_time = ::Time.now.utc
+          @init_time -= @handshake_time if @handshake_time
 
           tracer.start(self)
         end
@@ -92,12 +98,15 @@ module HTTPX::Plugins
       end
 
       def response=(*)
-        # init_time should be set when it's send to a connection.
-        # However, there are situations where connection initialization fails.
+        # There are situations where connection initialization fails.
         # Example is the :ssrf_filter plugin, which raises an error on
         # initialize if the host is an IP which matches against the known set.
         # in such cases, we'll just set here right here.
-        @init_time ||= ::Time.now.utc
+        unless @init_time
+          @init_time = ::Time.now.utc
+
+          @init_time -= @handshake_time if @handshake_time
+        end
 
         super
       end
@@ -112,11 +121,11 @@ module HTTPX::Plugins
       end
 
       def send_request_to_parser(request)
-        if connecting?
-          @init_time ||= ::Time.now.utc
-
-          # request span timeframe should include the time it took to connect.
-          request.init_time ||= @init_time
+        if connecting? && @init_time
+          # at this point, the connection has connected, but hasn't yet transitioned to :open,
+          # so it's the time the requests which caused this connection to connect, are finally being
+          # sent to the parser, so handshake time should factor into the span time.
+          request.handshake_time ||= ::Time.now.utc - @init_time
         end
 
         super
@@ -137,6 +146,17 @@ module HTTPX::Plugins
 
       private
 
+      def handle_error(er, request = nil)
+        if connecting? && @init_time
+          # bookkeep handshake time back into pending requests, which were never initialized with it.
+          handshake_time = ::Time.now.utc - @init_time
+          request.handshake_time ||= handshake_time if request
+          @pending.each { |req| req.handshake_time ||= handshake_time }
+        end
+
+        super
+      end
+
       def connect
         @init_time ||= ::Time.now.utc
 
@@ -151,6 +171,33 @@ module HTTPX::Plugins
         super
       end
     end
+
+    module HTTP2Methods
+      def initialize(*)
+        super
+
+        @handshake_time = nil
+        @handshake_init_time = ::Time.now.utc
+      end
+
+      def send(request, head = false)
+        if head
+          # only true for pending requests waiting on the handshake, so handshake time accrues.
+          request.handshake_time ||= 0.0
+          request.handshake_time += @handshake_time
+        end
+
+        super
+      end
+
+      private
+
+      def on_settings(*)
+        @handshake_time = ::Time.now.utc - @handshake_init_time unless @handshake_completed
+
+        super
+      end
+    end
   end
   register_plugin :tracing, Tracing
 end

data/lib/httpx/plugins/upgrade/h2.rb

@@ -25,7 +25,7 @@ module HTTPX
         def interests
           return super unless connecting? && @parser
 
-          connect
+          try_connect
 
           return @io.interests if connecting?
 

data/lib/httpx/plugins/upgrade.rb

@@ -30,6 +30,8 @@ module HTTPX
       end
 
       module InstanceMethods
+        private
+
         def fetch_response(request, selector, options)
           response = super
 

data/lib/httpx/plugins/webdav.rb

@@ -13,14 +13,18 @@ module HTTPX
       end
 
       module InstanceMethods
+        # sends a COPY WebDav request for +src+ and +dest+.
         def copy(src, dest)
           request("COPY", src, headers: { "destination" => @options.origin.merge(dest) })
         end
 
+        # sends a MOVE WebDav request for +src+ and +dest+.
         def move(src, dest)
           request("MOVE", src, headers: { "destination" => @options.origin.merge(dest) })
         end
 
+        # sends a LOCK WebDav request for +path+ (for the time set by +timeout+, when passed).
+        # it calls #unlock after the block is executed.
         def lock(path, timeout: nil, &blk)
           headers = {}
           headers["timeout"] = if timeout && timeout.positive?
@@ -51,14 +55,17 @@ module HTTPX
           response
         end
 
+        # sends a UNLOCK WebDav request for +path+, using the given +lock_token+.
         def unlock(path, lock_token)
           request("UNLOCK", path, headers: { "lock-token" => lock_token })
         end
 
+        # sends a MKCOL WebDav request for +dir+.
         def mkcol(dir)
           request("MKCOL", dir)
         end
 
+        # sends a PROPFIND WebDav request for +path+.
         def propfind(path, xml = nil)
           body = case xml
                  when :acl
@@ -73,6 +80,7 @@ module HTTPX
           request("PROPFIND", path, headers: { "depth" => "1" }, xml: body)
         end
 
+        # sends a PROPPATCH WebDav request for +path+.
         def proppatch(path, xml)
           body = "<?xml version=\"1.0\"?>" \
                  "<D:propertyupdate xmlns:D=\"DAV:\" xmlns:Z=\"http://ns.example.com/standards/z39.50/\">#{xml}</D:propertyupdate>"

data/lib/httpx/request/body.rb

@@ -32,6 +32,7 @@ module HTTPX
       @headers = h
       @body = self.class.initialize_body(params)
       @options = options.merge(params)
+      @unbounded_body = nil
 
       if @body
         if @options.compress_request_body && @headers.key?("content-encoding")
@@ -102,7 +103,7 @@ module HTTPX
 
     # returns whether the body yields infinitely.
     def unbounded_body?
-      return @unbounded_body if defined?(@unbounded_body)
+      return @unbounded_body unless @unbounded_body.nil?
 
       @unbounded_body = !@body.nil? && (chunked? || @body.bytesize == Float::INFINITY)
     end

data/lib/httpx/request.rb

@@ -107,7 +107,7 @@ module HTTPX
       @connection = @response =
         @drainer = @peer_address = @callbacks =
                      @informational_status = @on_response_arrived = nil
-      @ping = @started = false
+      @ping = @started = @complete = false
       @persistent = @options.persistent
       @active_timeouts = []
     end
@@ -121,8 +121,11 @@ module HTTPX
     end
 
     def complete!(response = @response)
+      return false if @complete
+
       emit(:complete, response)
       reset_timers(true)
+      @complete = true
     end
 
     # whether request has been buffered with a ping
@@ -137,22 +140,22 @@ module HTTPX
 
     # the read timeout defined for this request.
     def read_timeout
-      @options.timeout[:read_timeout]
+      @options.timeout.read_timeout
     end
 
     # the write timeout defined for this request.
     def write_timeout
-      @options.timeout[:write_timeout]
+      @options.timeout.write_timeout
     end
 
     # the request timeout defined for this request.
     def request_timeout
-      @options.timeout[:request_timeout]
+      @options.timeout.request_timeout
     end
 
     # the total request timeout defined for this request.
     def total_request_timeout
-      @options.timeout[:total_request_timeout]
+      @options.timeout.total_request_timeout
     end
 
     def persistent?
@@ -296,7 +299,7 @@ module HTTPX
       case nextstate
       when :idle
         @body.rewind
-        @ping = false
+        @ping = @complete = false
         @response = @drainer = nil
 
         # request may be sent to a different connection and will be
@@ -336,7 +339,7 @@ module HTTPX
 
     # whether the request supports the 100-continue handshake and already processed the 100 response.
     def expects?
-      @headers["expect"] == "100-continue" && @informational_status == 100 && !@response
+      @headers.key?("expect") && @headers.get("expect").include?("100-continue") && @informational_status == 100 && !@response
     end
 
     def set_timeout_callback(event, &callback)

data/lib/httpx/resolver/multi.rb

@@ -35,8 +35,8 @@ module HTTPX
       @resolvers.each { |r| r.current_session = s }
     end
 
-    def log(*args, **kwargs, &blk)
-      @resolvers.each { |r| r.log(*args, **kwargs, &blk) }
+    def log(...)
+      @resolvers.each { |r| r.log(...) }
     end
 
     def closed?

data/lib/httpx/resolver/resolver.rb

@@ -56,9 +56,9 @@ module HTTPX
       call
     end
 
-    def force_close(*args)
+    def force_close(...)
       while (connection = @connections.shift)
-        connection.force_close(*args)
+        connection.force_close(...)
       end
     end
 

data/lib/httpx/resolver.rb

@@ -57,7 +57,11 @@ module HTTPX
 
       if message.rcode != Resolv::DNS::RCode::NoError
         case message.rcode
-        when Resolv::DNS::RCode::ServFail
+        when Resolv::DNS::RCode::ServFail,
+             # getaddrinfo also retries these errors, as they're not about the hostname being resolved,
+             # but the DNS resolver state.
+             Resolv::DNS::RCode::NotImp,
+             Resolv::DNS::RCode::Refused
           return :retriable_error, message.rcode
         else
           return :dns_error, message.rcode

data/lib/httpx/response/body.rb

@@ -69,7 +69,7 @@ module HTTPX
     end
 
     # reads a chunk from the payload (implementation of the IO reader protocol).
-    def read(*args)
+    def read(...)
       return unless @buffer
 
       unless @reader
@@ -77,7 +77,7 @@ module HTTPX
         @reader = @buffer
       end
 
-      @reader.read(*args)
+      @reader.read(...)
     end
 
     # size of the decoded response payload. May differ from "content-length" header if