httpx 1.7.0 → 1.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2af63a63fe08211db58764570618b2e7c234d3473a28fc1ad6f5a31c3d0fb13d
-  data.tar.gz: 6b2671b85ac69e4817b8b764dfdabf638dacf94cfdcab6c44627ad64a57bcb50
+  metadata.gz: a9c30e22a2d406a61ef87a58fddd607bb2b1fad7b50d837d8db062d4a538a2d2
+  data.tar.gz: 9c5e1997b9c03434071c59b2deb2b004f7b1a1077c2ccaed03b9e3a1db7aafe5
 SHA512:
-  metadata.gz: 50a2d2d3c0cb27f3bf84cc34553b06bd9fadf46ad789f9cfa08091cc458ac07751cb5593e01d242d88e444f10ac2b3014ea3e4e56f7c616ebc860517d08ef90d
-  data.tar.gz: 6b15f21262e85639c6d32b9851926d7e717b6f9bcbc226b9af3872d67461799fd24847a2d563dfbad1967d5349d371c18a1a6e1a28c721c5f33387f2fc1446a7
+  metadata.gz: 87d99c4971b99f086f7811be81a8453e71bd2535120b5e560a2fc837d50e8a040e70ab1b2b191beee4b481fcea5063576d8895d318a0089183c21cb59fdd0f24
+  data.tar.gz: 85469cf5b5822990367f1e5591798a1512de90a60226853da4b4e6b00dde8816d6218dd2aacc9b5d85f84f6cb1637dd4f7b37836863f44144f05329e4fbdf7f5

data/doc/release_notes/1_7_1.md

@@ -0,0 +1,21 @@
+# 1.7.1
+
+## Improvements
+
+* fixed timers handling in the selector loop which caused them to be traversed-for-drop twice on each tick.
+* connection: take proxy connecting states when transitioning to `:closing` state.
+* refactored name resolution cache to a cache adapter API with a default memory cache which keeps the current behaviour and will allow to add others.
+  * a new option, `:resolver_cache`, was added, which is `:memory` by default.
+  * this fixes an issue introduced for multi-ractor support where the cache store would not be thread-safe when used in a non main ractor.
+* `:auth` plugin: when loaded with the `:retries` plugin, and the auth value method is dynamic/callable, will recover out-of-the-box from 401 HTTP responses by retrying the request with a newly-generated token.
+  * the `:rate_limiter` plugin will use this work to retry rate-limited responses without having to set setting `:retry_change_requests` to true, thereby eliminating a potential issue with non-idempotent requests.
+
+## Bugfixes
+
+* HTTP1 parser: clear buffer on reset.
+* http1 fix: handle the case in `#handle_error` where the response is an error response
+* `:stream_bidi` plugin: fix internal state preventing bidi requests from being retried.
+* `:stream_bidi` plugin: will only allow initial request body being passed using `:body` param (others, like `:json`, will raise an exception)
+* https resolver: return `:idle` on `#state` calls when no connection is available (sometimes called in internal log messages).
+* selector loop fix: when there are no selectables and an interval is passed, sleep instead of returning (thereby avoiding potential busy loop).
+*

data/lib/httpx/adapters/webmock.rb

@@ -38,12 +38,15 @@ module WebMock
 
           return build_error_response(request, webmock_response.exception) if webmock_response.exception
 
-          request.options.response_class.new(request,
-                                             webmock_response.status[0],
-                                             "2.0",
-                                             webmock_response.headers).tap do |res|
-            res.mocked = true
-          end
+          request
+            .options
+            .response_class
+            .new(
+              request,
+              webmock_response.status[0],
+              "2.0",
+              webmock_response.headers
+            ).tap(&:mock!)
         end
 
         def build_error_response(request, exception)
@@ -72,17 +75,23 @@ module WebMock
       end
 
       module ResponseMethods
-        attr_accessor :mocked
-
         def initialize(*)
           super
           @mocked = false
         end
+
+        def mock!
+          @mocked = true
+        end
+
+        def mocked?
+          @mocked
+        end
       end
 
       module ResponseBodyMethods
         def decode_chunk(chunk)
-          return chunk if @response.mocked
+          return chunk if @response.mocked?
 
           super
         end

data/lib/httpx/altsvc.rb

@@ -43,7 +43,7 @@ module HTTPX
       end
 
       def altsvc_match?(uri, other_uri)
-        other_uri = URI(other_uri)
+        other_uri = URI(other_uri) #: http_uri
 
         uri.origin == other_uri.origin || begin
           case uri.scheme

data/lib/httpx/connection/http1.rb

@@ -197,9 +197,10 @@ module HTTPX
     end
 
     def handle_error(ex, request = nil)
-      if (ex.is_a?(EOFError) || ex.is_a?(TimeoutError)) && @request && @request.response &&
-         !@request.response.headers.key?("content-length") &&
-         !@request.response.headers.key?("transfer-encoding")
+      if (ex.is_a?(EOFError) || ex.is_a?(TimeoutError)) && @request &&
+         (response = @request.response) && response.is_a?(Response) &&
+         !response.headers.key?("content-length") &&
+         !response.headers.key?("transfer-encoding")
         # if the response does not contain a content-length header, the server closing the
         # connnection is the indicator of response consumed.
         # https://greenbytes.de/tech/webdav/rfc2616.html#rfc.section.4.4

data/lib/httpx/connection.rb

@@ -724,7 +724,7 @@ module HTTPX
 
         disconnect
       when :closing
-        return unless @state == :idle || @state == :open
+        return unless connecting? || @state == :open
 
         unless @write_buffer.empty?
           # preset state before handshake, as error callbacks
@@ -850,6 +850,9 @@ module HTTPX
       end
     end
 
+    # recover internal state and emit all relevant error responses when +error+ was raised.
+    # this takes an optiona +request+ which may have already been handled and can be opted out
+    # in the state recovery process.
     def handle_error(error, request = nil)
       parser.handle_error(error, request) if @parser && @parser.respond_to?(:handle_error)
       while (req = @pending.shift)

data/lib/httpx/io/tcp.rb

@@ -111,7 +111,7 @@ module HTTPX
       raise e if @ip_index.negative?
 
       log { "failed connecting to #{@ip} (#{e.message}), evict from cache and trying next..." }
-      Resolver.cached_lookup_evict(@hostname, @ip)
+      @options.resolver_cache.evict(@hostname, @ip)
 
       @io = build_socket
       retry

data/lib/httpx/options.rb

@@ -12,6 +12,7 @@ module HTTPX
     CLOSE_HANDSHAKE_TIMEOUT = 10
     CONNECT_TIMEOUT = READ_TIMEOUT = WRITE_TIMEOUT = 60
     REQUEST_TIMEOUT = OPERATION_TIMEOUT = nil
+    RESOLVER_TYPES = %i[memory file].freeze
 
     # default value used for "user-agent" header, when not overridden.
     USER_AGENT = "httpx.rb/#{VERSION}".freeze # rubocop:disable Style/RedundantFreeze
@@ -44,7 +45,7 @@ module HTTPX
 
         return unless meth =~ /^option_(.+)$/
 
-        optname = Regexp.last_match(1)
+        optname = Regexp.last_match(1) #: String
 
         if optname =~ /^(.+[^_])_+with/
           # ignore alias method chain generated methods.
@@ -52,14 +53,14 @@ module HTTPX
           # it relies on the "_with/_without" separator, which is the most used convention,
           # however it shouldn't be used in practice in httpx given the plugin architecture
           # as the main extension API.
-          orig_name = Regexp.last_match(1)
+          orig_name = Regexp.last_match(1) #: String
 
           return if @options_names.include?(orig_name.to_sym)
         end
 
         optname = optname.to_sym
 
-        attr_reader(optname)
+        attr_reader(optname) unless method_defined?(optname)
 
         @options_names << optname unless @options_names.include?(optname)
       end
@@ -103,7 +104,10 @@ module HTTPX
     # :io :: open socket, or domain/ip-to-socket hash, which requests should be sent to
     # :persistent :: whether to persist connections in between requests (defaults to <tt>true</tt>)
     # :resolver_class :: which resolver to use (defaults to <tt>:native</tt>, can also be <tt>:system<tt> for
-    #                    using getaddrinfo or <tt>:https</tt> for DoH resolver, or a custom class)
+    #                    using getaddrinfo or <tt>:https</tt> for DoH resolver, or a custom class inheriting
+    #                    from HTTPX::Resolver::Resolver)
+    # :resolver_cache :: strategy to cache DNS results, ignored by the <tt>:system</tt> resolver, can be set to <tt>:memory<tt>
+    #                    or an instance of a custom class inheriting from HTTPX::Resolver::Cache::Base
     # :resolver_options :: hash of options passed to the resolver. Accepted keys depend on the resolver type.
     # :pool_options :: hash of options passed to the connection pool (See Pool#initialize).
     # :ip_families :: which socket families are supported (system-dependent)
@@ -151,6 +155,36 @@ module HTTPX
       freeze
     end
 
+    # returns the class with which to instantiate the DNS resolver.
+    def resolver_class
+      case @resolver_class
+      when Symbol
+        public_send(:"resolver_#{@resolver_class}_class")
+      else
+        @resolver_class
+      end
+    end
+
+    def resolver_cache
+      cache_type = @resolver_cache
+
+      case cache_type
+      when :memory
+        Resolver::Cache::Memory.cache(cache_type)
+      when :file
+        Resolver::Cache::File.cache(cache_type)
+      else
+        unless cache_type.respond_to?(:resolve) &&
+               cache_type.respond_to?(:get) &&
+               cache_type.respond_to?(:set) &&
+               cache_type.respond_to?(:evict)
+          raise TypeError, ":resolver_cache must be a compatible resolver cache and implement #get, #set and #evict"
+        end
+
+        cache_type #: Object & Resolver::_Cache
+      end
+    end
+
     def freeze
       self.class.options_names.each do |ivar|
         # avoid freezing debug option, as when it's set, it's usually an
@@ -172,6 +206,7 @@ module HTTPX
       super || options_equals?(other)
     end
 
+    # checks whether +other+ is equal by comparing the session options
     def options_equals?(other, ignore_ivars = REQUEST_BODY_IVARS)
       # headers and other request options do not play a role, as they are
       # relevant only for the request.
@@ -187,6 +222,8 @@ module HTTPX
       end
     end
 
+    # returns a HTTPX::Options instance resulting of the merging of +other+ with self.
+    # it may return self if +other+ is self or equal to self.
     def merge(other)
       if (is_options = other.is_a?(Options))
 
@@ -361,7 +398,7 @@ module HTTPX
       request_class response_class headers_class request_body_class
       response_body_class connection_class http1_class http2_class
       resolver_native_class resolver_system_class resolver_https_class options_class pool_class
-      io fallback_protocol debug debug_redact resolver_class
+      io fallback_protocol debug debug_redact
       compress_request_body decompress_response_body
       persistent close_on_fork
     ].each do |method_name|
@@ -421,6 +458,41 @@ module HTTPX
       Array(value)
     end
 
+    def option_resolver_class(resolver_type)
+      case resolver_type
+      when Symbol
+        meth = :"resolver_#{resolver_type}_class"
+
+        raise TypeError, ":resolver_class must be a supported type" unless respond_to?(meth)
+
+        resolver_type
+      when Class
+        raise TypeError, ":resolver_class must be a subclass of `#{Resolver::Resolver}`" unless resolver_type < Resolver::Resolver
+
+        resolver_type
+      else
+        raise TypeError, ":resolver_class must be a supported type"
+      end
+    end
+
+    def option_resolver_cache(cache_type)
+      if cache_type.is_a?(Symbol)
+        raise TypeError, ":resolver_cache: #{cache_type} is invalid" unless RESOLVER_TYPES.include?(cache_type)
+
+        require "httpx/resolver/cache/file" if cache_type == :file
+
+      else
+        unless cache_type.respond_to?(:resolve) &&
+               cache_type.respond_to?(:get) &&
+               cache_type.respond_to?(:set) &&
+               cache_type.respond_to?(:evict)
+          raise TypeError, ":resolver_cache must be a compatible resolver cache and implement #resolve, #get, #set and #evict"
+        end
+      end
+
+      cache_type
+    end
+
     # called after all options are initialized
     def do_initialize
       hs = @headers
@@ -496,6 +568,7 @@ module HTTPX
       :addresses => nil,
       :persistent => false,
       :resolver_class => (ENV["HTTPX_RESOLVER"] || :native).to_sym,
+      :resolver_cache => (ENV["HTTPX_RESOLVER_CACHE"] || :memory).to_sym,
       :resolver_options => { cache: true }.freeze,
       :pool_options => EMPTY_HASH,
       :ip_families => nil,

data/lib/httpx/parser/http1.rb

@@ -26,6 +26,7 @@ module HTTPX
         @headers = {}
         @content_length = nil
         @_has_trailers = nil
+        @buffer.clear
       end
 
       def upgrade?

data/lib/httpx/plugins/auth.rb

@@ -16,6 +16,13 @@ module HTTPX
         }
       end
 
+      # adds support for the following options:
+      #
+      # :auth_header_value :: the token to use as a string, or a callable which returns a string when called.
+      # :auth_header_type :: the authentication type to use in the "authorization" header value (i.e. "Bearer", "Digest"...)
+      # :generate_auth_value_on_retry :: callable which returns whether the request should regenerate the auth_header_value
+      #                                  when the request is retried (this option will only work if the session also loads the
+      #                                  <tt>:retries</tt> plugin).
       module OptionsMethods
         def option_auth_header_value(value)
           value
@@ -74,10 +81,14 @@ module HTTPX
         def generate_auth_token
           return unless (auth_value = @options.auth_header_value)
 
-          auth_value = auth_value.call(self) if auth_value.respond_to?(:call)
+          auth_value = auth_value.call(self) if dynamic_auth_token?(auth_value)
 
           auth_value
         end
+
+        def dynamic_auth_token?(auth_header_value)
+          auth_header_value&.respond_to?(:call)
+        end
       end
 
       module RequestMethods
@@ -92,14 +103,29 @@ module HTTPX
 
       module AuthRetries
         module InstanceMethods
+          private
+
+          def retryable_request?(request, response, options)
+            super || auth_error?(response, options)
+          end
+
+          def retryable_response?(response, options)
+            auth_error?(response, options) || super
+          end
+
           def prepare_to_retry(request, response)
             super
 
-            return unless @options.generate_auth_value_on_retry && @options.generate_auth_value_on_retry.call(response)
+            return unless auth_error?(response, request.options) ||
+                          (@options.generate_auth_value_on_retry && @options.generate_auth_value_on_retry.call(response))
 
             request.headers.get("authorization").pop
             @auth_header_value = generate_auth_token
           end
+
+          def auth_error?(response, options)
+            response.is_a?(Response) && response.status == 401 && dynamic_auth_token?(options.auth_header_value)
+          end
         end
       end
     end

data/lib/httpx/plugins/oauth.rb

@@ -180,6 +180,10 @@ module HTTPX
         end
       end
 
+      # adds support for the following options:
+      #
+      # :oauth_options :: an hash of options to be used during session management.
+      #                   check the parameters to initialize the OAuthSession class.
       module OptionsMethods
         private
 
@@ -255,29 +259,18 @@ module HTTPX
 
           @oauth_session.fetch_access_token(self)
         end
-      end
 
-      module OAuthRetries
-        class << self
-          def extra_options(options)
-            options.merge(
-              retry_on: method(:response_oauth_error?),
-              generate_auth_value_on_retry: method(:response_oauth_error?)
-            )
-          end
-
-          def response_oauth_error?(res)
-            res.is_a?(Response) && res.status == 401
-          end
+        def dynamic_auth_token?(_)
+          @oauth_session
         end
+      end
 
+      module OAuthRetries
         module InstanceMethods
-          def prepare_to_retry(_request, response)
-            unless @oauth_session && @options.generate_auth_value_on_retry && @options.generate_auth_value_on_retry.call(response)
-              return super
-            end
+          private
 
-            @oauth_session.reset!
+          def prepare_to_retry(_request, response)
+            @oauth_session.reset! if @oauth_session
 
             super
           end

data/lib/httpx/plugins/persistent.rb

@@ -55,10 +55,8 @@ module HTTPX
 
         private
 
-        def repeatable_request?(request, _)
+        def retryable_request?(request, response, *)
           super || begin
-            response = request.response
-
             return false unless response && response.is_a?(ErrorResponse)
 
             error = response.error
@@ -67,13 +65,13 @@ module HTTPX
           end
         end
 
-        def retryable_error?(ex)
+        def retryable_error?(ex, options)
           super &&
             # under the persistent plugin rules, requests are only retried for connection related errors,
             # which do not include request timeout related errors. This only gets overriden if the end user
             # manually changed +:max_retries+ to something else, which means it is aware of the
             # consequences.
-            (!ex.is_a?(RequestTimeoutError) || @options.max_retries != 1)
+            (!ex.is_a?(RequestTimeoutError) || options.max_retries != 1)
         end
       end
     end

data/lib/httpx/plugins/proxy/http.rb

@@ -43,10 +43,6 @@ module HTTPX
         end
 
         module ConnectionMethods
-          def connecting?
-            super || @state == :connecting || @state == :connected
-          end
-
           def force_close(*)
             if @state == :connecting
               # proxy connect related requests should not be reenqueed

data/lib/httpx/plugins/proxy.rb

@@ -326,7 +326,9 @@ module HTTPX
 
       module ProxyRetries
         module InstanceMethods
-          def retryable_error?(ex)
+          private
+
+          def retryable_error?(ex, *)
             super || ex.is_a?(ProxyConnectionError)
           end
         end

data/lib/httpx/plugins/query.rb

@@ -23,7 +23,7 @@ module HTTPX
         module InstanceMethods
           private
 
-          def repeatable_request?(request, options)
+          def retryable_request?(request, *)
             super || request.verb == "QUERY"
           end
         end

data/lib/httpx/plugins/rate_limiter.rb

@@ -12,22 +12,11 @@ module HTTPX
     # https://gitlab.com/os85/httpx/wikis/Rate-Limiter
     #
     module RateLimiter
-      class << self
-        RATE_LIMIT_CODES = [429, 503].freeze
-
-        def configure(klass)
-          klass.plugin(:retries,
-                       retry_change_requests: true,
-                       retry_on: method(:retry_on_rate_limited_response?),
-                       retry_after: method(:retry_after_rate_limit))
-        end
-
-        def retry_on_rate_limited_response?(response)
-          return false unless response.is_a?(Response)
+      RATE_LIMIT_CODES = [429, 503].freeze
 
-          status = response.status
-
-          RATE_LIMIT_CODES.include?(status)
+      class << self
+        def load_dependencies(klass)
+          klass.plugin(:retries, retry_after: method(:retry_after_rate_limit))
         end
 
         # Servers send the "Retry-After" header field to indicate how long the
@@ -48,6 +37,22 @@ module HTTPX
           Utils.parse_retry_after(retry_after)
         end
       end
+
+      module InstanceMethods
+        private
+
+        def retryable_request?(request, response, options)
+          super || rate_limit_error?(response)
+        end
+
+        def retryable_response?(response, options)
+          rate_limit_error?(response) || super
+        end
+
+        def rate_limit_error?(response)
+          response.is_a?(Response) && RATE_LIMIT_CODES.include?(response.status)
+        end
+      end
     end
 
     register_plugin :rate_limiter, RateLimiter

data/lib/httpx/plugins/retries.rb

@@ -142,15 +142,8 @@ module HTTPX
 
           if response &&
              request.retries.positive? &&
-             repeatable_request?(request, options) &&
-             (
-               (
-                 response.is_a?(ErrorResponse) && retryable_error?(response.error)
-               ) ||
-
-                 options.retry_on&.call(response)
-
-             )
+             retryable_request?(request, response, options) &&
+             retryable_response?(response, options)
             try_partial_retry(request, response)
             log { "failed to get response, #{request.retries} tries to go..." }
             prepare_to_retry(request, response)
@@ -186,12 +179,16 @@ module HTTPX
         end
 
         # returns whether +request+ can be retried.
-        def repeatable_request?(request, options)
+        def retryable_request?(request, _, options)
           IDEMPOTENT_METHODS.include?(request.verb) || options.retry_change_requests
         end
 
+        def retryable_response?(response, options)
+          (response.is_a?(ErrorResponse) && retryable_error?(response.error, options)) || options.retry_on&.call(response)
+        end
+
         # returns whether the +ex+ exception happend for a retriable request.
-        def retryable_error?(ex)
+        def retryable_error?(ex, _)
           RETRYABLE_ERRORS.any? { |klass| ex.is_a?(klass) }
         end
 

data/lib/httpx/plugins/stream.rb

@@ -164,7 +164,7 @@ module HTTPX
 
             unless request.options.stream && !request.stream
               if options[:stream]
-                warn "passing `stream: true` with a request obkect is not supported anymore. " \
+                warn "passing `stream: true` with a request object is not supported anymore. " \
                      "You can instead build the request object with `stream :true`"
               end
               return super
@@ -217,7 +217,7 @@ module HTTPX
 
           @stream.on_chunk(chunk.dup)
 
-          chunk.size
+          chunk.bytesize
         end
 
         private

data/lib/httpx/plugins/stream_bidi.rb

@@ -278,6 +278,8 @@ module HTTPX
           headers_sent = @headers_sent
 
           case nextstate
+          when :idle
+            headers_sent = false
           when :waiting_for_chunk
             return unless @state == :body
           when :body
@@ -325,7 +327,17 @@ module HTTPX
       module RequestBodyMethods
         def initialize(*, **)
           super
-          @headers.delete("content-length") if @options.stream
+
+          return unless @options.stream
+
+          @headers.delete("content-length")
+
+          return unless @body
+
+          return if @body.is_a?(Transcoder::Body::Encoder)
+
+          raise Error, "bidirectional streams only allow the usage of the `:body` param to set request bodies." \
+                       "You must encode it yourself if you wish to do so."
         end
 
         def empty?

data/lib/httpx/pool.rb

@@ -148,7 +148,6 @@ module HTTPX
 
     def checkout_resolver(options)
       resolver_type = options.resolver_class
-      resolver_type = Resolver.resolver_for(resolver_type, options)
 
       @resolver_mtx.synchronize do
         resolvers = @resolvers[resolver_type]

data/lib/httpx/request/body.rb

@@ -128,7 +128,7 @@ module HTTPX
           Transcoder::Body.encode(body)
         elsif (form = params.delete(:form))
           if Transcoder::Multipart.multipart?(form)
-            # @type var form: Transcoder::multipart_input
+            # @type var form: Transcoder::Multipart::multipart_input
             Transcoder::Multipart.encode(form)
           else
             # @type var form: Transcoder::urlencoded_input