httpx 1.8.0 → 1.8.2

data/lib/httpx/io/ssl.rb

@@ -6,22 +6,22 @@ module HTTPX
   TLSError = OpenSSL::SSL::SSLError
 
   class SSL < TCP
-    # rubocop:disable Style/MutableConstant
-    TLS_OPTIONS = { alpn_protocols: %w[h2 http/1.1].freeze }
-    # https://github.com/jruby/jruby-openssl/issues/284
+    tls_options = { alpn_protocols: %w[h2 http/1.1].freeze }
     # TODO: remove when dropping support for jruby-openssl < 0.15.4
-    TLS_OPTIONS[:verify_hostname] = true if RUBY_ENGINE == "jruby" && JOpenSSL::VERSION < "0.15.4"
-    # rubocop:enable Style/MutableConstant
-    TLS_OPTIONS.freeze
+    # https://github.com/jruby/jruby-openssl/issues/284
+    tls_options[:verify_hostname] = true if RUBY_ENGINE == "jruby" && JOpenSSL::VERSION < "0.15.4"
+    TLS_OPTIONS = tls_options.freeze
 
     attr_writer :ssl_session
 
     def initialize(_, _, options)
       super
 
-      @ssl_session = nil
-      ctx_options = TLS_OPTIONS.merge(options.ssl)
-      @sni_hostname = ctx_options.delete(:hostname) || @hostname
+      @ssl_session = @session_new_cb = nil
+
+      ctx_options = TLS_OPTIONS
+      ctx_options = ctx_options.merge(options.ssl) if options.ssl && !options.ssl.empty?
+      @sni_hostname = (ctx_options.delete(:hostname) if ctx_options.key?(:hostname)) || @hostname
 
       if @keep_open && @io.is_a?(OpenSSL::SSL::SSLSocket)
         # externally initiated ssl socket
@@ -29,11 +29,12 @@ module HTTPX
         @state = :negotiated
       else
         @ctx = OpenSSL::SSL::SSLContext.new
-        @ctx.set_params(ctx_options) unless ctx_options.empty?
+        @ctx.set_params(ctx_options)
         unless @ctx.session_cache_mode.nil? # a dummy method on JRuby
           @ctx.session_cache_mode =
             OpenSSL::SSL::SSLContext::SESSION_CACHE_CLIENT | OpenSSL::SSL::SSLContext::SESSION_CACHE_NO_INTERNAL_STORE
         end
+        init_session_new_cb
 
         yield(self) if block_given?
       end
@@ -42,33 +43,41 @@ module HTTPX
     end
 
     if OpenSSL::SSL::SSLContext.method_defined?(:session_new_cb=)
+      # sets the ssl session callback to be picked up by the ssl context.
       def session_new_cb(&pr)
-        @ctx.session_new_cb = proc { |_, sess| pr.call(sess) }
+        @session_new_cb = pr
+      end
+
+      # sets the ssl context's new session callback, which points at @session_new_cb when available.
+      def init_session_new_cb
+        @ctx.session_new_cb = proc { |_, sess| @session_new_cb&.call(sess) }
       end
     else
       # session_new_cb not implemented under JRuby
       def session_new_cb; end
+
+      def init_session_new_cb; end
     end
 
+    private :init_session_new_cb
+
     def protocol
-      # @type ivar @io: OpenSSL::SSL::SSLSocket
+      return super unless @io.is_a?(OpenSSL::SSL::SSLSocket)
+
       @io.alpn_protocol || super
-    rescue StandardError
-      super
     end
 
     if RUBY_ENGINE == "jruby"
       # in jruby, alpn_protocol may return ""
       # https://github.com/jruby/jruby-openssl/issues/287
       def protocol
-        # @type ivar @io: OpenSSL::SSL::SSLSocket
+        return super unless @io.is_a?(OpenSSL::SSL::SSLSocket)
+
         proto = @io.alpn_protocol
 
         return super if proto.nil? || proto.empty?
 
         proto
-      rescue StandardError
-        super
       end
     end
 

data/lib/httpx/io/tcp.rb

@@ -182,11 +182,18 @@ module HTTPX
     end
 
     def close
-      return if @keep_open || closed?
+      return if @keep_open
+
+      # mark tcp as closed, so that it can be disconnected.
+      # this bypasses the state machine API as not not allow the transition
+      # from idle to closed in normal circumstances.
+      @state = :closed if @state == :idle
+
+      return if closed?
 
       begin
         @io.close
-      rescue StandardError => e
+      rescue IOError => e
         log { "error closing socket" }
         log { e.full_message(highlight: false) }
       ensure
@@ -212,7 +219,7 @@ module HTTPX
       @state == :idle || @state == :closed
     end
 
-    # :nocov:
+    # simplecov:disable
     def inspect
       "#<#{self.class}:#{object_id} " \
         "#{@ip}:#{@port} " \
@@ -221,7 +228,7 @@ module HTTPX
         "@addresses=#{@addresses} " \
         "@state=#{@state}>"
     end
-    # :nocov:
+    # simplecov:enable
 
     private
 

data/lib/httpx/io/unix.rb

@@ -60,11 +60,11 @@ module HTTPX
       true
     end
 
-    # :nocov:
+    # simplecov:disable
     def inspect
       "#<#{self.class}:#{object_id} @path=#{@path}) @state=#{@state})>"
     end
-    # :nocov:
+    # simplecov:enable
 
     private
 

data/lib/httpx/loggable.rb

@@ -17,6 +17,12 @@ module HTTPX
 
     USE_DEBUG_LOG = ENV.key?("HTTPX_DEBUG")
 
+    def self.log_identifiers
+      "pid=#{Process.pid} " \
+        "tid=#{Thread.current.object_id} " \
+        "fid=#{Fiber.current.object_id}"
+    end
+
     def log(
       level: @options.debug_level,
       color: nil,
@@ -36,10 +42,9 @@ module HTTPX
         klass = klass.superclass
       end
 
-      message = +"(time:#{Time.now.utc}, pid:#{Process.pid}, " \
-                 "tid:#{Thread.current.object_id}, " \
-                 "fid:#{Fiber.current.object_id}, " \
-                 "self:#{class_name}##{object_id}) "
+      message = +"(time=#{Time.now.utc} " \
+                 "#{Loggable.log_identifiers} " \
+                 "self=#{class_name}##{object_id}) "
       message << msg.call << "\n"
       message = "\e[#{COLORS[color]}m#{message}\e[0m" if color && debug_stream.respond_to?(:isatty) && debug_stream.isatty
       debug_stream << message
@@ -49,6 +54,8 @@ module HTTPX
       log(level: level, color: color, debug_level: debug_level, debug: debug) { ex.full_message }
     end
 
+    private
+
     def log_redact_headers(text)
       log_redact(text, @options.debug_redact == :headers)
     end

data/lib/httpx/options.rb

@@ -257,12 +257,16 @@ module HTTPX
 
         other_opts = opts_names
       else
-        other_opts = other # : Hash[Symbol, untyped]
+        other_opts = other #: Hash[Symbol, untyped]
         other_opts = Hash[other] unless other.is_a?(Hash)
 
         return self if other_opts.empty?
 
-        return self if other_opts.all? { |opt, v| !respond_to?(opt) || public_send(opt) == v }
+        return self if other_opts.all? do |opt, v|
+          raise Error, "unknown option: #{opt}" unless respond_to?(opt)
+
+          public_send(opt) == v
+        end
       end
 
       opts = dup

data/lib/httpx/plugins/auth/digest.rb

@@ -24,7 +24,7 @@ module HTTPX
 
         def authenticate(request, authenticate)
           "Digest #{generate_header(request.verb, request.path, authenticate)}"
-        rescue StandardError => e
+        rescue HTTPX::Error => e
           response = ErrorResponse.new(request, e)
           request.response = response
           request.emit_response(response)

data/lib/httpx/plugins/auth.rb

@@ -19,6 +19,8 @@ module HTTPX
       # adds support for the following options:
       #
       # :auth_header_value :: the token to use as a string, or a callable which returns a string when called.
+      #                       the callable is called a request, and a boolean: when true, the user must regenerate a token, otherwise it
+      #                       may probe for token freshness and return the same token.
       # :auth_header_type :: the authentication type to use in the "authorization" header value (i.e. "Bearer", "Digest"...)
       # :auth_header_expires_at :: timestamp at which the auth header will be discarded. should be a callable (like a proc)
       #                            receiving the request as an argument, and should return either a Time object, or an integer (UNIX time).
@@ -96,9 +98,8 @@ module HTTPX
           auth_header_value = @auth_header_value_mtx.synchronize do
             try_invalidate_auth_header_value
 
-            @auth_header_value ||= begin
+            @auth_header_value ||= generate_auth_token(request, false).tap do
               set_auth_header_expires_at(request)
-              generate_auth_token
             end
           end
 
@@ -115,10 +116,10 @@ module HTTPX
           @auth_header_value = @auth_header_expires_at = nil
         end
 
-        def generate_auth_token
+        def generate_auth_token(request, should_regenerate)
           return unless (auth_value = @options.auth_header_value)
 
-          auth_value = auth_value.call(self) if dynamic_auth_token?(auth_value)
+          auth_value = auth_value.call(request, should_regenerate) if dynamic_auth_token?(auth_value)
 
           auth_value
         end
@@ -127,8 +128,8 @@ module HTTPX
           @auth_header_expires_at = if (expires_in = request.options.auth_header_expires_in)
             Time.now.to_i + expires_in
           elsif (expires_at = request.options.auth_header_expires_at)
-            if expires_at.respond_to?(:call)
-              expires_at = expires_at.call(request).to_f
+            if expires_at.respond_to?(:call) && (expires_at = expires_at.call(request))
+              expires_at = expires_at.to_f
               raise Error, "`:auth_header_expires_at` must be positive" unless expires_at.positive?
 
               expires_at
@@ -196,7 +197,7 @@ module HTTPX
             # use whatever was generated for it.
             @auth_header_value_mtx.synchronize do
               if request.auth_token_value == @auth_header_value
-                @auth_header_value = generate_auth_token
+                @auth_header_value = generate_auth_token(request, true)
                 set_auth_header_expires_at(request)
               end
             end
@@ -204,6 +205,15 @@ module HTTPX
             request.unauthorize!
           end
 
+          def when_to_retry(request, response, *)
+            return super unless response.is_a?(Response)
+
+            # retry immediately if authentication no longer valid.
+            return if response.status == 401
+
+            super
+          end
+
           def auth_error?(response, options)
             response.is_a?(Response) && response.status == 401 && dynamic_auth_token?(options.auth_header_value)
           end

data/lib/httpx/plugins/callbacks.rb

@@ -122,6 +122,18 @@ module HTTPX
         end
       end
 
+      module RequestMethods
+        def drain_body
+          super.tap do |chunk|
+            emit(:body_chunk, chunk) if chunk
+          rescue StandardError => e
+            # in case an error occurs in callback code
+            @drain_error = e
+            nil
+          end
+        end
+      end
+
       module ConnectionMethods
         private
 

data/lib/httpx/plugins/digest_auth.rb

@@ -51,9 +51,13 @@ module HTTPX
 
             if probe_response.status == 401 && digest.can_authenticate?(probe_response.headers["www-authenticate"])
               request.transition(:idle)
+
               if (auth_header = digest.authenticate(request, probe_response.headers["www-authenticate"]))
                 request.authorize(auth_header)
               end
+
+              next(request.response) if request.response
+
               super(request)
             else
               probe_response

data/lib/httpx/plugins/fiber_concurrency.rb

@@ -70,6 +70,12 @@ module HTTPX
           )
         end
 
+        def initial_call
+          return unless current_context?
+
+          super
+        end
+
         def interests
           return if connecting? && @pending.none?(&:current_context?)
 
@@ -193,6 +199,12 @@ module HTTPX
       end
 
       module ResolverNativeMethods
+        def initial_call
+          return unless @queries.values.any?(&:current_context?) || @connections.any?(&:current_context?)
+
+          super
+        end
+
         def calculate_interests
           return if @queries.empty?
 
@@ -218,11 +230,23 @@ module HTTPX
       end
 
       module ResolverSystemMethods
+        def initial_call
+          return unless current_context?
+
+          super
+        end
+
         def interests
-          return unless @queries.any? { |_, conn| conn.current_context? }
+          return unless current_context?
 
           super
         end
+
+        private
+
+        def current_context?
+          @queries.any? { |_, conn| conn.current_context? }
+        end
       end
 
       module FiberConcurrencyH2C

data/lib/httpx/plugins/follow_redirects.rb

@@ -135,9 +135,9 @@ module HTTPX
             return ErrorResponse.new(request, error)
           end
 
-          retry_request = build_request(redirect_method, redirect_uri, redirect_params, options)
+          redirect_request = build_request(redirect_method, redirect_uri, redirect_params, options)
 
-          request.redirect_request = retry_request
+          request.redirect_request = redirect_request
 
           redirect_after = response.headers["retry-after"]
 
@@ -151,24 +151,24 @@ module HTTPX
             redirect_after = Utils.parse_retry_after(redirect_after)
 
             retry_start = Utils.now
-            log { "redirecting after #{redirect_after} secs..." }
+            redirect_request.log { "redirecting after #{redirect_after} secs..." }
             selector.after(redirect_after) do
               if (response = request.response)
                 response.finish!
-                retry_request.response = response
+                redirect_request.response = response
                 # request has terminated abruptly meanwhile
-                retry_request.emit_response(response)
+                redirect_request.emit_response(response)
               else
-                log { "redirecting (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
-                send_request(retry_request, selector, options)
+                redirect_request.log { "redirecting (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
+                send_request(redirect_request, selector, options)
               end
             end
           else
-            send_request(retry_request, selector, options)
+            send_request(redirect_request, selector, options)
 
             # recalling itself, in case an error was triggered by the above, and we can
             # verify retriability again.
-            return fetch_response(request, selector, options)
+            return fetch_response(redirect_request, selector, options)
           end
           nil
         end
@@ -231,6 +231,18 @@ module HTTPX
           @redirect_request.response
         end
 
+        def response=(response)
+          return super unless @redirect_request && @response.nil? # rubocop:disable Lint/ReturnInVoidContext
+
+          @redirect_request.response = response
+        end
+
+        def emit_response(response)
+          return super unless @redirect_request && @response.nil?
+
+          @redirect_request.emit_response(response)
+        end
+
         def max_redirects
           @options.max_redirects || MAX_REDIRECTS
         end

data/lib/httpx/plugins/oauth.rb

@@ -296,9 +296,11 @@ module HTTPX
 
         private
 
-        def generate_auth_token
-          return unless @oauth_session
+        def generate_auth_token(*)
+          return super unless @oauth_session
 
+          # should_regenerate arg ignored, as there's no way to check
+          # for token expiration yet.
           @oauth_session.fetch_access_token(self)
         end
 

data/lib/httpx/plugins/persistent.rb

@@ -18,6 +18,12 @@ module HTTPX
     # https://gitlab.com/os85/httpx/wikis/Persistent
     #
     module Persistent
+      SAFE_REONNECTABLE_ERRORS = [
+        PingTimeoutError,
+        Connection::HTTP2::GoawayError,
+        Connection::HTTP2::PingError,
+      ].freeze
+
       class << self
         def load_dependencies(klass)
           klass.plugin(:fiber_concurrency)
@@ -55,13 +61,38 @@ module HTTPX
 
         private
 
+        def reconnectable_error?(error)
+          Retries::RECONNECTABLE_ERRORS.any? { |klass| error.is_a?(klass) }
+        end
+
+        # whether the error can be safely retried without booking threshold attempts.
+        def safe_reconnectable_error?(error)
+          SAFE_REONNECTABLE_ERRORS.any? { |klass| error.is_a?(klass) }
+        end
+
+        def can_reconnect?(_, response)
+          super || (response.is_a?(ErrorResponse) && safe_reconnectable_error?(response.error))
+        end
+
+        def when_to_retry(request, response, *)
+          return super unless response.is_a?(ErrorResponse)
+
+          error = response.error
+          # allow request to be retried immediately if the request failed right after the keep alive timeout.
+          # the chances are, the request failed because the connect has been dropped by the peer server, so it's
+          # fine to reopen.
+          return if request.ping? && reconnectable_error?(error)
+
+          super
+        end
+
         def retryable_request?(request, response, *)
           super || begin
-            return false unless response && response.is_a?(ErrorResponse)
+            return false unless response.is_a?(ErrorResponse)
 
             error = response.error
 
-            Retries::RECONNECTABLE_ERRORS.any? { |klass| error.is_a?(klass) }
+            reconnectable_error?(error)
           end
         end
 

data/lib/httpx/plugins/proxy.rb

@@ -182,9 +182,11 @@ module HTTPX
           if (no_proxy = proxy.no_proxy)
             no_proxy = no_proxy.join(",") if no_proxy.is_a?(Array)
 
-            # TODO: setting proxy to nil leaks the connection object in the pool
-            return super(request_uri, selector, options.merge(proxy: nil)) unless URI::Generic.use_proxy?(request_uri.host, next_proxy.host,
-                                                                                                          next_proxy.port, no_proxy)
+            unless no_proxy != "*" && # NO_PROXY=* bypasses proxy use
+                   URI::Generic.use_proxy?(request_uri.host, next_proxy.host, next_proxy.port, no_proxy)
+              # TODO: setting proxy to nil leaks the connection object in the pool
+              return super(request_uri, selector, options.merge(proxy: nil))
+            end
           end
 
           super(request_uri, selector, options.merge(proxy: proxy))
@@ -341,6 +343,10 @@ module HTTPX
         module InstanceMethods
           private
 
+          def proxy_error?(request, response, _)
+            super && !request.retries.positive?
+          end
+
           def retryable_error?(ex, *)
             super || ex.is_a?(ProxyConnectionError)
           end

data/lib/httpx/plugins/push_promise.rb

@@ -44,9 +44,9 @@ module HTTPX
 
         def __on_promise_request(parser, stream, h)
           log(level: 1, color: :yellow) do
-            # :nocov:
+            # simplecov:disable
             h.map { |k, v| "#{stream.id}: -> PROMISE HEADER: #{k}: #{v}" }.join("\n")
-            # :nocov:
+            # simplecov:enable
           end
           headers = @options.headers_class.new(h)
           path = headers[":path"]

data/lib/httpx/plugins/retries.rb

@@ -29,7 +29,10 @@ module HTTPX
         Errno::ETIMEDOUT,
         ConnectionError,
         TLSError,
-        Connection::HTTP2::Error,
+        Zlib::BufError,
+        PingTimeoutError,
+        Connection::HTTP2::GoawayError,
+        Connection::HTTP2::PingError,
       ].freeze
 
       RETRYABLE_ERRORS = (RECONNECTABLE_ERRORS + [
@@ -148,10 +151,13 @@ module HTTPX
              retryable_response?(response, options)
             try_partial_retry(request, response)
             log { "failed to get response, #{request.retries} tries to go..." }
-            prepare_to_retry(request, response)
 
-            if (retry_after = when_to_retry(request, response, options)) && retry_after.positive?
+            # retry-after must be calculated before prepare_to_retry, as it relies on
+            # state changed byit.
+            retry_after = when_to_retry(request, response, options)
+            prepare_to_retry(request, response)
 
+            if retry_after&.positive?
               retry_start = Utils.now
               log { "retrying after #{retry_after} secs..." }
               selector.after(retry_after) do
@@ -191,13 +197,14 @@ module HTTPX
           RETRYABLE_ERRORS.any? { |klass| ex.is_a?(klass) } && !ex.is_a?(TotalRequestTimeoutError)
         end
 
-        def proxy_error?(request, response, _)
-          super && !request.retries.positive?
+        def prepare_to_retry(request, response)
+          request.retries -= 1 unless can_reconnect?(request, response)
+          request.transition(:idle)
         end
 
-        def prepare_to_retry(request, _response)
-          request.retries -= 1 unless request.ping? # do not exhaust retries on connection liveness probes
-          request.transition(:idle)
+        # do not exhaust retries on connection liveness probes
+        def can_reconnect?(request, _)
+          request.ping?
         end
 
         def when_to_retry(request, response, options)

data/lib/httpx/plugins/server_sent_events.rb

@@ -146,7 +146,7 @@ module HTTPX
           def when_to_retry(request, *)
             retry_after = request.last_server_sent_message&.retry_after
 
-            retry_after / 1_000.0 if retry_after # original in milliseconds
+            retry_after.to_i / 1_000.0 if retry_after # original in milliseconds
 
             request.last_server_sent_message&.retry_after && super
           end

data/lib/httpx/plugins/ssrf_filter.rb

@@ -149,7 +149,7 @@ module HTTPX
             SsrfFilter.unsafe_ip_address?(ipaddr) || @options.extra_unsafe_ranges&.any? { |r| r.include?(ipaddr) }
           end
 
-          raise ServerSideRequestForgeryError, "#{@origin.host} has no public IP addresses" if addrs.empty?
+          raise ServerSideRequestForgeryError, "#{@origin.host} has no allowed IP addresses" if addrs.empty?
 
           super
         end

data/lib/httpx/plugins/stream.rb

@@ -75,11 +75,11 @@ module HTTPX
       @on_chunk.call(chunk)
     end
 
-    # :nocov:
+    # simplecov:disable
     def inspect
       "#<#{self.class}:#{object_id}>"
     end
-    # :nocov:
+    # simplecov:enable
 
     def to_s
       if @request.response

data/lib/httpx/plugins/stream_bidi.rb

@@ -177,6 +177,8 @@ module HTTPX
           @pipe_read.readpartial(1)
         end
 
+        def initial_call; end
+
         def interests
           return if @closed