httpx 1.7.1 → 1.7.3

data/lib/httpx/plugins/follow_redirects.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
 module HTTPX
-  InsecureRedirectError = Class.new(Error)
+  class InsecureRedirectError < Error
+  end
+
   module Plugins
     #
     # This plugin adds support for automatically following redirect (status 30X) responses.

data/lib/httpx/plugins/ntlm_auth.rb

@@ -46,7 +46,7 @@ module HTTPX
 
               if probe_response.status == 401 && ntlm.can_authenticate?(probe_response.headers["www-authenticate"])
                 request.transition(:idle)
-                request.headers.get("authorization").pop
+                request.unauthorize!
                 request.authorize(ntlm.authenticate(request, probe_response.headers["www-authenticate"]).encode("utf-8"))
                 super(request)
               else

data/lib/httpx/plugins/rate_limiter.rb

@@ -16,25 +16,7 @@ module HTTPX
 
       class << self
         def load_dependencies(klass)
-          klass.plugin(:retries, retry_after: method(:retry_after_rate_limit))
-        end
-
-        # Servers send the "Retry-After" header field to indicate how long the
-        # user agent ought to wait before making a follow-up request.  When
-        # sent with a 503 (Service Unavailable) response, Retry-After indicates
-        # how long the service is expected to be unavailable to the client.
-        # When sent with any 3xx (Redirection) response, Retry-After indicates
-        # the minimum time that the user agent is asked to wait before issuing
-        # the redirected request.
-        #
-        def retry_after_rate_limit(_, response)
-          return unless response.is_a?(Response)
-
-          retry_after = response.headers["retry-after"]
-
-          return unless retry_after
-
-          Utils.parse_retry_after(retry_after)
+          klass.plugin(:retries)
         end
       end
 
@@ -52,6 +34,24 @@ module HTTPX
         def rate_limit_error?(response)
           response.is_a?(Response) && RATE_LIMIT_CODES.include?(response.status)
         end
+
+        # Servers send the "Retry-After" header field to indicate how long the
+        # user agent ought to wait before making a follow-up request.  When
+        # sent with a 503 (Service Unavailable) response, Retry-After indicates
+        # how long the service is expected to be unavailable to the client.
+        # When sent with any 3xx (Redirection) response, Retry-After indicates
+        # the minimum time that the user agent is asked to wait before issuing
+        # the redirected request.
+        #
+        def when_to_retry(_, response, options)
+          return super unless response.is_a?(Response)
+
+          retry_after = response.headers["retry-after"]
+
+          return super unless retry_after
+
+          Utils.parse_retry_after(retry_after)
+        end
       end
     end
 

data/lib/httpx/plugins/retries.rb

@@ -148,10 +148,7 @@ module HTTPX
             log { "failed to get response, #{request.retries} tries to go..." }
             prepare_to_retry(request, response)
 
-            retry_after = options.retry_after
-            retry_after = retry_after.call(request, response) if retry_after.respond_to?(:call)
-
-            if retry_after
+            if (retry_after = when_to_retry(request, response, options))
               # apply jitter
               if (jitter = request.options.retry_jitter)
                 retry_after = jitter.call(retry_after)
@@ -201,6 +198,12 @@ module HTTPX
           request.transition(:idle)
         end
 
+        def when_to_retry(request, response, options)
+          retry_after = options.retry_after
+          retry_after = retry_after.call(request, response) if retry_after.respond_to?(:call)
+          retry_after
+        end
+
         #
         # Attempt to set the request to perform a partial range request.
         # This happens if the peer server accepts byte-range requests, and
@@ -237,14 +240,15 @@ module HTTPX
         def initialize(*args)
           super
           @retries = @options.max_retries
+          @partial_response = nil
         end
 
         def response=(response)
-          if @partial_response
+          if (partial_response = @partial_response)
             if response.is_a?(Response) && response.status == 206
-              response.from_partial_response(@partial_response)
+              response.from_partial_response(partial_response)
             else
-              @partial_response.close
+              partial_response.close
             end
             @partial_response = nil
           end

data/lib/httpx/plugins/ssrf_filter.rb

@@ -106,6 +106,7 @@ module HTTPX
             error = ServerSideRequestForgeryError.new("#{request.uri} URI scheme not allowed")
             error.set_backtrace(caller)
             response = ErrorResponse.new(request, error)
+            request.response = response
             request.emit(:response, response)
             response
           end

data/lib/httpx/plugins/stream_bidi.rb

@@ -61,7 +61,7 @@ module HTTPX
         def handle_stream(stream, request)
           return super unless @options.stream
 
-          request.on(:body) do
+          request.flush_buffer_on_body do
             next unless request.headers_sent
 
             handle(request, stream)
@@ -189,6 +189,10 @@ module HTTPX
           !@closed
         end
 
+        def force_close(*)
+          terminate
+        end
+
         def terminate
           return if @closed
 
@@ -202,6 +206,8 @@ module HTTPX
           terminate
         end
 
+        alias_method :on_io_error, :on_error
+
         # noop (the owner connection will take of it)
         def handle_socket_timeout(interval); end
       end
@@ -254,9 +260,14 @@ module HTTPX
           super
           @headers_sent = false
           @closed = false
+          @flush_buffer_on_body_cb = nil
           @mutex = Thread::Mutex.new
         end
 
+        def flush_buffer_on_body(&cb)
+          @flush_buffer_on_body_cb = on(:body, &cb)
+        end
+
         def closed?
           return super unless @options.stream
 
@@ -280,6 +291,11 @@ module HTTPX
           case nextstate
           when :idle
             headers_sent = false
+
+            if @flush_buffer_on_body_cb
+              callbacks(:body).delete(@flush_buffer_on_body_cb)
+              @flush_buffer_on_body_cb = nil
+            end
           when :waiting_for_chunk
             return unless @state == :body
           when :body

data/lib/httpx/request.rb

@@ -91,7 +91,7 @@ module HTTPX
       raise UnsupportedSchemeError, "#{@uri}: #{@uri.scheme}: unsupported URI scheme" unless ALLOWED_URI_SCHEMES.include?(@uri.scheme)
 
       @state = :idle
-      @response = @peer_address = @informational_status = nil
+      @response = @drainer = @peer_address = @informational_status = nil
       @ping = false
       @persistent = @options.persistent
       @active_timeouts = []

data/lib/httpx/resolver/resolver.rb

@@ -119,6 +119,11 @@ module HTTPX
       end
     end
 
+    def on_io_error(e)
+      on_error(e)
+      force_close(true)
+    end
+
     def on_error(error)
       handle_error(error)
       disconnect

data/lib/httpx/selector.rb

@@ -204,8 +204,7 @@ module HTTPX
       rescue IOError => e
         (Array(r) + Array(w)).each do |sel|
           # TODO: is there a way to cheaply find the IO associated with the error?
-          sel.on_error(e)
-          sel.force_close(true)
+          sel.on_io_error(e)
         end
       rescue StandardError => e
         (Array(r) + Array(w)).each do |sel|
@@ -249,8 +248,9 @@ module HTTPX
           when :rw then rw_wait(io, interval)
           end
       rescue IOError => e
-        io.on_error(e)
-        io.force_close(true)
+        io.on_io_error(e)
+
+        return
       rescue StandardError => e
         io.on_error(e)
 

data/lib/httpx/session.rb

@@ -140,9 +140,10 @@ module HTTPX
       end
       selector.deregister(connection)
 
+      # do not check-in connections only created for Happy Eyeballs
       return if cloned
 
-      return if @closing && connection.state == :closed
+      return if @closing && connection.state == :closed && !connection.used?
 
       connection.log(level: 2) { "check-in connection##{connection.object_id}(#{connection.state}) in pool##{@pool.object_id}" }
       @pool.checkin_connection(connection)
@@ -177,16 +178,15 @@ module HTTPX
 
     # returns the HTTPX::Connection through which the +request+ should be sent through.
     def find_connection(request_uri, selector, options)
-      log(level: 2) { "finding connection for #{request_uri}..." }
       if (connection = selector.find_connection(request_uri, options))
         connection.idling if connection.state == :closed
-        connection.log(level: 2) { "found connection##{connection.object_id}(#{connection.state}) in selector##{selector.object_id}" }
+        log(level: 2) { "found connection##{connection.object_id}(#{connection.state}) in selector##{selector.object_id}" }
         return connection
       end
 
       connection = @pool.checkout_connection(request_uri, options)
 
-      connection.log(level: 2) { "found connection##{connection.object_id}(#{connection.state}) in pool##{@pool.object_id}" }
+      log(level: 2) { "found connection##{connection.object_id}(#{connection.state}) in pool##{@pool.object_id}" }
 
       case connection.state
       when :idle
@@ -240,7 +240,7 @@ module HTTPX
 
       return unless response && response.finished?
 
-      log(level: 2) { "response fetched" }
+      log(level: 2) { "response##{response.object_id} fetched" }
 
       response
     end
@@ -249,6 +249,7 @@ module HTTPX
     def send_request(request, selector, options = request.options)
       error = begin
         catch(:resolve_error) do
+          log(level: 2) { "finding connection for request##{request.object_id}..." }
           connection = find_connection(request.uri, selector, options)
           connection.send(request)
         end

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "1.7.1"
+  VERSION = "1.7.3"
 end

data/sig/chainable.rbs

@@ -29,7 +29,7 @@ module HTTPX
               | (:proxy, ?options) -> (Plugins::sessionProxy & Plugins::httpProxy)
               | (:push_promise, ?options) -> Plugins::sessionPushPromise
               | (:retries, ?options) -> Plugins::sessionRetries
-              | (:rate_limiter, ?options) -> Session
+              | (:rate_limiter, ?options) -> Plugins::sessionRateLimiter
               | (:stream, ?options) -> Plugins::sessionStream
               | (:stream_bidi, ?options) -> Plugins::sessionStreamBidi
               | (:aws_sigv4, ?options) -> Plugins::awsSigV4Session

data/sig/connection/http2.rbs

@@ -79,6 +79,8 @@ module HTTPX
 
     def on_stream_refuse: (::HTTP2::Stream stream, Request request, StandardError error) -> void
 
+    def on_stream_half_close: (::HTTP2::Stream stream, Request request) -> void
+
     def on_stream_close: (::HTTP2::Stream stream, Request request, (Symbol | StandardError)? error) -> void
 
     def on_frame: (string bytes) -> void
@@ -87,13 +89,15 @@ module HTTPX
 
     def on_close: (Integer last_frame, Symbol? error, String? payload) -> void
 
-    def on_frame_sent: (::HTTP2::frame) -> void
+    def on_frame_sent: (::HTTP2::frame frame) -> void
+
+    def frame_with_extra_info: (::HTTP2::frame frame) -> Hash[Symbol, untyped]
 
-    def on_frame_received: (::HTTP2::frame) -> void
+    def on_frame_received: (::HTTP2::frame frame) -> void
 
-    def on_altsvc: (String origin, ::HTTP2::frame) -> void
+    def on_altsvc: (String origin, ::HTTP2::frame frame) -> void
 
-    def on_promise: (::HTTP2::Stream) -> void
+    def on_promise: (::HTTP2::Stream stream) -> void
 
     def on_origin: (String) -> void
 

data/sig/connection.rbs

@@ -37,6 +37,7 @@ module HTTPX
     @read_buffer: Buffer
     @write_buffer: Buffer
     @inflight: Integer
+    @max_concurrent_requests: Integer?
     @keep_alive_timeout: Numeric?
     @timeout: Numeric?
     @current_timeout: Numeric?
@@ -119,6 +120,8 @@ module HTTPX
 
     def on_error: (HTTPX::TimeoutError | Error | StandardError error, ?Request? request) -> void
 
+    def on_io_error: (IOError error) -> void
+
     private
 
     def initialize: (http_uri uri, Options options) -> void

data/sig/plugins/auth.rbs

@@ -29,6 +29,12 @@ module HTTPX
       end
 
       module RequestMethods
+        @auth_token_value: String?
+
+        def authorized?: () -> bool
+
+        def unauthorize!: () -> void
+
         def authorize: (String auth_value) -> void
       end
 

data/sig/plugins/cookies/cookie.rbs

@@ -31,6 +31,8 @@ module HTTPX
       def cookie_value: () -> String
       alias to_s cookie_value
 
+      def match?: (String | cookie_attributes name_or_options) -> bool
+
       def valid_for_uri?: (http_uri uri) -> bool
 
       def self.new: (Cookie) -> instance
@@ -41,8 +43,7 @@ module HTTPX
 
       private
 
-      def initialize: (cookie_attributes) -> untyped
-                    | (_ToS, _ToS, ?cookie_attributes) -> untyped
+      def initialize: (_ToS name, _ToS value, ?cookie_attributes) -> void
 
       def acceptable_from_uri?: (uri) -> bool
 

data/sig/plugins/cookies/jar.rbs

@@ -6,11 +6,20 @@ module HTTPX
       include Enumerable[Cookie]
 
       @cookies: Array[Cookie]
+      @mtx: Thread::Mutex
 
       def parse: (String set_cookie) -> void
 
+      def get: (String | cookie_attributes name_or_options) -> Cookie?
+
+      def get_all: (String | cookie_attributes name_or_options) -> Array[Cookie]
+
+      def set: (_ToS | Cookie name, ?(cookie_attributes | _ToS) value_or_options) -> void
+
       def add: (Cookie name, ?String path) -> void
 
+      def delete: (String | Cookie | cookie_attributes name_or_options) -> void
+
       def []: (http_uri) -> Array[Cookie]
 
       def each: (?http_uri?) { (Cookie) -> void } -> void
@@ -21,6 +30,8 @@ module HTTPX
       private
 
       def initialize: (?_Each[cookie] cookies) -> untyped
+
+      def synchronize: [T] { () -> T } -> T
     end
   end
 end

data/sig/plugins/cookies.rbs

@@ -15,6 +15,8 @@ module HTTPX
 
       module InstanceMethods
         def cookies: () -> Jar
+
+        def make_jar: (*untyped) -> Jar
       end
 
       module HeadersMethods

data/sig/plugins/expect.rbs

@@ -2,14 +2,29 @@ module HTTPX
   module Plugins
     module Expect
       EXPECT_TIMEOUT: Integer
+      NOEXPECT_STORE_MUTEX: Thread::Mutex
+
+      self.@no_expect_store: Store
+      def self.no_expect_store: () -> Store
+
+      def self.extra_options: (Options) -> (Options & _ExpectOptions)
+
+      class Store
+        @store: Array[String]
+        @mutex: Thread::Mutex
+
+        def include?: (String host) -> bool
+
+        def add: (String host) -> void
+
+        def delete: (String host) -> void
+      end
 
       interface _ExpectOptions
         def expect_timeout: () -> Integer?
 
         def expect_threshold_size: () -> Integer?
       end
-
-      def self.extra_options: (Options) -> (Options & _ExpectOptions)
     end
   end
 end

data/sig/plugins/proxy/socks4.rbs

@@ -5,6 +5,10 @@ module HTTPX
   module Plugins
     module Proxy
       module Socks4
+        VERSION: Integer
+        CONNECT: Integer
+        GRANTED: Integer
+        PROTOCOLS: Array[String]
 
         module ConnectionMethods
           def __socks4_proxy_connect: () ->  void

data/sig/plugins/rate_limiter.rbs

@@ -5,11 +5,11 @@ module HTTPX
 
       def self.load_dependencies: (singleton(Session)) -> void
 
-      def self.retry_after_rate_limit: (untyped, response) -> Numeric?
-
       module InstanceMethods
         def rate_limit_error?: (response response) -> bool
       end
     end
+
+    type sessionRateLimiter = Session & RateLimiter::InstanceMethods
   end
 end

data/sig/plugins/response_cache.rbs

@@ -51,7 +51,7 @@ module HTTPX
       module ResponseMethods
         attr_writer original_request: cacheRequest
 
-        @cache: bool
+        @cached: bool
         @cache_control: Array[String]?
         @vary: Array[String]?
         @date: Time?
@@ -66,9 +66,9 @@ module HTTPX
 
         def fresh?: () -> bool
 
-        def cache_control: () -> Array[String]?
+         %a{pure} def cache_control: () -> Array[String]?
 
-        def vary: () -> Array[String]?
+         %a{pure} def vary: () -> Array[String]?
 
         private
 

data/sig/plugins/retries.rbs

@@ -8,16 +8,16 @@ module HTTPX
       DEFAULT_JITTER: ^(Numeric) -> Numeric
       BACKOFF_ALGORITHMS: Array[Symbol]
 
-      def self?.retry_after_polynomial_backoff: (retriesRequest request, response response) -> Numeric
+      def self?.retry_after_polynomial_backoff: (retriesRequest request, retriesResponse response) -> Numeric
 
-      def self?.retry_after_exponential_backoff: (retriesRequest request, response response) -> Numeric
+      def self?.retry_after_exponential_backoff: (retriesRequest request, retriesResponse response) -> Numeric
 
       interface _RetryCallback
-        def call: (response response) -> bool?
+        def call: (retriesResponse response) -> bool?
       end
 
       interface _RetriesOptions
-        def retry_after: () -> (^(retriesRequest request, response response) -> Numeric | Numeric)?
+        def retry_after: () -> (^(retriesRequest request, retriesResponse response) -> Numeric | Numeric)?
 
         def retry_jitter: () -> ^(Numeric jitter) -> Numeric
 
@@ -35,17 +35,19 @@ module HTTPX
 
         private
 
-        def fetch_response: (retriesRequest request, Selector selector, retriesOptions options) -> (retriesResponse | ErrorResponse)?
+        def fetch_response: (retriesRequest request, Selector selector, retriesOptions options) -> retriesResponse?
 
-        def retryable_request?: (retriesRequest request, response response, retriesOptions options) -> boolish
+        def retryable_request?: (retriesRequest request, retriesResponse response, retriesOptions options) -> boolish
 
-        def retryable_response?: (response response, retriesOptions options) -> boolish
+        def retryable_response?: (retriesResponse response, retriesOptions options) -> boolish
 
         def retryable_error?: (_Exception error, Options options) -> boolish
 
-        def try_partial_retry: (retriesRequest request, (retriesResponse | ErrorResponse) response) -> void
+        def try_partial_retry: (retriesRequest request, retriesResponse response) -> void
 
-        def prepare_to_retry: (Request & RequestMethods request, response response) -> void
+        def prepare_to_retry: (Request & RequestMethods request, retriesResponse response) -> void
+
+        def when_to_retry: (Request & RequestMethods request, retriesResponse response, retriesOptions options) -> void
       end
 
       module RequestMethods
@@ -53,20 +55,22 @@ module HTTPX
 
         attr_accessor retries: Integer
 
-        attr_writer partial_response: Response?
+        attr_writer partial_response: retriesResponse?
 
-        def response=: (retriesResponse | ErrorResponse response) -> void
+        def response=: (retriesResponse response) -> void
       end
 
       module ResponseMethods
-        def from_partial_response: (Response response) -> void
+        def from_partial_response: (retriesHTTPResponse response) -> void
       end
 
       type retriesOptions = Options & _RetriesOptions
 
       type retriesRequest = Request & RequestMethods
 
-      type retriesResponse = Response & ResponseMethods
+      type retriesHTTPResponse = Response & ResponseMethods
+
+      type retriesResponse = retriesHTTPResponse | ErrorResponse
     end
 
     type sessionRetries = Session & Retries::InstanceMethods

data/sig/plugins/stream_bidi.rbs

@@ -47,8 +47,11 @@ module HTTPX
         attr_accessor headers_sent: bool
 
         @closed: bool
+        @flush_buffer_on_body_cb: ^() -> void | nil
         @mutex: Thread::Mutex
 
+        def flush_buffer_on_body: { () -> void } -> void
+
         def closed?: () -> bool
       end
 

data/sig/request.rbs

@@ -6,6 +6,7 @@ module HTTPX
 
     METHODS: Array[Symbol]
     USER_AGENT: String
+    ALLOWED_URI_SCHEMES: Array[String]
 
     attr_reader verb: verb
     attr_reader uri: http_uri

data/sig/resolver/native.rbs

@@ -37,6 +37,8 @@ module HTTPX
 
       def handle_socket_timeout: (Numeric interval) -> void
 
+      def on_io_error: (IOError error) -> void
+
       private
 
       def initialize: (ip_family family, options options) -> void

data/sig/resolver/resolver.rbs

@@ -27,8 +27,6 @@ module HTTPX
 
       def terminate: () -> void
 
-      def force_close: (*untyped args) -> void
-
       def closed?: () -> bool
 
       def empty?: () -> bool
@@ -60,6 +58,8 @@ module HTTPX
       def resolve_error: (String hostname, ?StandardError?) -> (ResolveError | ResolveTimeoutError)
 
       def close_or_resolve: () -> void
+
+      def disconnect: () -> void
     end
   end
 end

data/sig/selector.rbs

@@ -13,6 +13,10 @@ module HTTPX
     def handle_socket_timeout: (Numeric interval) -> void
 
     def on_error: (StandardError) -> void
+
+    def on_io_error: (IOError error) -> void
+
+    def force_close: (?bool delete_pending) -> void
   end
 
   class Selector