httpx 1.6.3 → 1.7.0

data/lib/httpx/resolver.rb

@@ -5,7 +5,10 @@ require "resolv"
 
 module HTTPX
   module Resolver
+    extend self
+
     RESOLVE_TIMEOUT = [2, 3].freeze
+    MAX_CACHE_SIZE = 512
 
     require "httpx/resolver/entry"
     require "httpx/resolver/resolver"
@@ -15,26 +18,19 @@ module HTTPX
     require "httpx/resolver/multi"
 
     @lookup_mutex = Thread::Mutex.new
+    @hostnames = []
     @lookups = Hash.new { |h, k| h[k] = [] }
 
     @identifier_mutex = Thread::Mutex.new
     @identifier = 1
     @hosts_resolver = Resolv::Hosts.new
 
-    module_function
-
     def supported_ip_families
-      @supported_ip_families ||= begin
-        # https://github.com/ruby/resolv/blob/095f1c003f6073730500f02acbdbc55f83d70987/lib/resolv.rb#L408
-        list = Socket.ip_address_list
-        if list.any? { |a| a.ipv6? && !a.ipv6_loopback? && !a.ipv6_linklocal? }
-          [Socket::AF_INET6, Socket::AF_INET]
-        else
-          [Socket::AF_INET]
-        end
-      rescue NotImplementedError
-        [Socket::AF_INET]
-      end.freeze
+      if in_ractor?
+        Ractor.store_if_absent(:httpx_supported_ip_families) { find_supported_ip_families }
+      else
+        @supported_ip_families ||= find_supported_ip_families
+      end
     end
 
     def resolver_for(resolver_type, options)
@@ -63,7 +59,12 @@ module HTTPX
     # matches +hostname+ to entries in the hosts file, returns <tt>nil</nil> if none is
     # found, or there is no hosts file.
     def hosts_resolve(hostname)
-      ips = @hosts_resolver.getaddresses(hostname)
+      ips = if in_ractor?
+        Ractor.store_if_absent(:httpx_hosts_resolver) { Resolv::Hosts.new }
+      else
+        @hosts_resolver
+      end.getaddresses(hostname)
+
       return if ips.empty?
 
       ips.map { |ip| Entry.new(ip) }
@@ -72,13 +73,20 @@ module HTTPX
 
     def cached_lookup(hostname)
       now = Utils.now
-      lookup_synchronize do |lookups|
-        lookup(hostname, lookups, now)
+      lookup_synchronize do |lookups, hostnames|
+        lookup(hostname, lookups, hostnames, now)
       end
     end
 
     def cached_lookup_set(hostname, family, entries)
-      lookup_synchronize do |lookups|
+      lookup_synchronize do |lookups, hostnames|
+        # lru cleanup
+        while lookups.size >= MAX_CACHE_SIZE
+          hs = hostnames.shift
+          lookups.delete(hs)
+        end
+        hostnames << hostname
+
         case family
         when Socket::AF_INET6
           lookups[hostname].concat(entries)
@@ -86,13 +94,14 @@ module HTTPX
           lookups[hostname].unshift(*entries)
         end
         entries.each do |entry|
-          next unless entry["name"] != hostname
+          name = entry["name"]
+          next unless name != hostname
 
           case family
           when Socket::AF_INET6
-            lookups[entry["name"]] << entry
+            lookups[name] << entry
           when Socket::AF_INET
-            lookups[entry["name"]].unshift(entry)
+            lookups[name].unshift(entry)
           end
         end
       end
@@ -101,26 +110,38 @@ module HTTPX
     def cached_lookup_evict(hostname, ip)
       ip = ip.to_s
 
-      lookup_synchronize do |lookups|
+      lookup_synchronize do |lookups, hostnames|
         entries = lookups[hostname]
 
         return unless entries
 
-        lookups.delete_if { |entry| entry["data"] == ip }
+        entries.delete_if { |entry| entry["data"] == ip }
+
+        if entries.empty?
+          lookups.delete(hostname)
+          hostnames.delete(hostname)
+        end
       end
     end
 
     # do not use directly!
-    def lookup(hostname, lookups, ttl)
+    def lookup(hostname, lookups, hostnames, ttl)
       return unless lookups.key?(hostname)
 
-      entries = lookups[hostname] = lookups[hostname].select do |address|
-        address["TTL"] > ttl
+      entries = lookups[hostname]
+
+      entries.delete_if do |address|
+        address["TTL"] < ttl
+      end
+
+      if entries.empty?
+        lookups.delete(hostname)
+        hostnames.delete(hostname)
       end
 
       ips = entries.flat_map do |address|
         if (als = address["alias"])
-          lookup(als, lookups, ttl)
+          lookup(als, lookups, hostnames, ttl)
         else
           Entry.new(address["data"], address["TTL"])
         end
@@ -130,7 +151,12 @@ module HTTPX
     end
 
     def generate_id
-      id_synchronize { @identifier = (@identifier + 1) & 0xFFFF }
+      if in_ractor?
+        identifier = Ractor.store_if_absent(:httpx_resolver_identifier) { -1 }
+        Ractor.current[:httpx_resolver_identifier] = (identifier + 1) & 0xFFFF
+      else
+        id_synchronize { @identifier = (@identifier + 1) & 0xFFFF }
+      end
     end
 
     def encode_dns_query(hostname, type: Resolv::DNS::Resource::IN::A, message_id: generate_id)
@@ -152,7 +178,14 @@ module HTTPX
 
       return :message_truncated if message.tc == 1
 
-      return :dns_error, message.rcode if message.rcode != Resolv::DNS::RCode::NoError
+      if message.rcode != Resolv::DNS::RCode::NoError
+        case message.rcode
+        when Resolv::DNS::RCode::ServFail
+          return :retriable_error, message.rcode
+        else
+          return :dns_error, message.rcode
+        end
+      end
 
       addresses = []
 
@@ -178,12 +211,47 @@ module HTTPX
       [:ok, addresses]
     end
 
+    private
+
     def lookup_synchronize
-      @lookup_mutex.synchronize { yield(@lookups) }
+      if in_ractor?
+        lookups = Ractor.store_if_absent(:httpx_resolver_lookups) { Hash.new { |h, k| h[k] = [] } }
+        hostnames = Ractor.store_if_absent(:httpx_resolver_hostnames) { [] }
+        return yield(lookups, hostnames)
+      end
+
+      @lookup_mutex.synchronize { yield(@lookups, @hostnames) }
     end
 
     def id_synchronize(&block)
       @identifier_mutex.synchronize(&block)
     end
+
+    def find_supported_ip_families
+      list = Socket.ip_address_list
+
+      begin
+        if list.any? { |a| a.ipv6? && !a.ipv6_loopback? && !a.ipv6_linklocal? }
+          [Socket::AF_INET6, Socket::AF_INET]
+        else
+          [Socket::AF_INET]
+        end
+      rescue NotImplementedError
+        [Socket::AF_INET]
+      end.freeze
+    end
+
+    if defined?(Ractor) &&
+       # no ractor support for 3.0
+       RUBY_VERSION >= "3.1.0"
+
+      def in_ractor?
+        Ractor.main != Ractor.current
+      end
+    else
+      def in_ractor?
+        false
+      end
+    end
   end
 end

data/lib/httpx/response/body.rb

@@ -138,6 +138,8 @@ module HTTPX
       else
         IO.copy_stream(@buffer, dest)
       end
+    ensure
+      close
     end
 
     # closes/cleans the buffer, resets everything

data/lib/httpx/response.rb

@@ -211,16 +211,19 @@ module HTTPX
 
     def initialize(header_value)
       @header_value = header_value
+      @mime_type = @charset = nil
+      @initialized = false
     end
 
     # returns the mime type declared in the header.
     #
     #   ContentType.new("application/json; charset=utf-8").mime_type #=> "application/json"
     def mime_type
-      return @mime_type if defined?(@mime_type)
+      return @mime_type if @initialized
 
-      m = @header_value.to_s[MIME_TYPE_RE, 1]
-      m && @mime_type = m.strip.downcase
+      load
+
+      @mime_type
     end
 
     # returns the charset declared in the header.
@@ -228,10 +231,23 @@ module HTTPX
     #   ContentType.new("application/json; charset=utf-8").charset #=> "utf-8"
     #   ContentType.new("text/plain").charset #=> nil
     def charset
-      return @charset if defined?(@charset)
+      return @charset if @initialized
+
+      load
+
+      @charset
+    end
+
+    private
+
+    def load
+      m = @header_value.to_s[MIME_TYPE_RE, 1]
+      m && @mime_type = m.strip.downcase
+
+      c = @header_value.to_s[CHARSET_RE, 1]
+      c && @charset = c.strip.delete('"')
 
-      m = @header_value.to_s[CHARSET_RE, 1]
-      m && @charset = m.strip.delete('"')
+      @initialized = true
     end
   end
 

data/lib/httpx/selector.rb

@@ -199,6 +199,12 @@ module HTTPX
     def select_many(r, w, interval, &block)
       begin
         readers, writers = ::IO.select(r, w, nil, interval)
+      rescue IOError => e
+        (Array(r) + Array(w)).each do |sel|
+          # TODO: is there a way to cheaply find the IO associated with the error?
+          sel.on_error(e)
+          sel.force_close(true)
+        end
       rescue StandardError => e
         (Array(r) + Array(w)).each do |sel|
           sel.on_error(e)
@@ -240,6 +246,9 @@ module HTTPX
           when :w then io.to_io.wait_writable(interval)
           when :rw then rw_wait(io, interval)
           end
+      rescue IOError => e
+        io.on_error(e)
+        io.force_close(true)
       rescue StandardError => e
         io.on_error(e)
 

data/lib/httpx/session.rb

@@ -177,6 +177,7 @@ module HTTPX
 
     # returns the HTTPX::Connection through which the +request+ should be sent through.
     def find_connection(request_uri, selector, options)
+      log(level: 2) { "finding connection for ##{request_uri}..." }
       if (connection = selector.find_connection(request_uri, options))
         connection.idling if connection.state == :closed
         connection.log(level: 2) { "found connection##{connection.object_id}(#{connection.state}) in selector##{selector.object_id}" }
@@ -224,7 +225,7 @@ module HTTPX
 
     # tries deactivating connections in the +selector+, deregistering the ones that have been deactivated.
     def deactivate(selector)
-      selector.each_connection.each(&:deactivate)
+      selector.each_connection.to_a.each(&:deactivate)
     end
 
     # callback executed when an HTTP/2 promise frame has been received.
@@ -415,12 +416,12 @@ module HTTPX
 
     def find_resolver_for(connection, selector)
       if (resolver = selector.find_resolver(connection.options))
-        resolver.log(level: 2) { "found resolver##{connection.object_id}(#{connection.state}) in selector##{selector.object_id}" }
+        resolver.log(level: 2) { "found resolver##{resolver.object_id}(#{resolver.state}) in selector##{selector.object_id}" }
         return resolver
       end
 
       resolver = @pool.checkout_resolver(connection.options)
-      resolver.log(level: 2) { "found resolver##{connection.object_id}(#{connection.state}) in pool##{@pool.object_id}" }
+      resolver.log(level: 2) { "found resolver##{resolver.object_id}(#{resolver.state}) in pool##{@pool.object_id}" }
       pin(resolver, selector)
 
       resolver
@@ -436,14 +437,13 @@ module HTTPX
           conn1.log(level: 2) { "check-in connection##{conn1.object_id}(#{conn1.state}) in pool##{@pool.object_id}" }
           @pool.checkin_connection(conn1)
         end
-        return false
+        return
       end
 
       conn2.log(level: 2) { "coalescing with connection##{conn1.object_id}[#{conn1.origin}])" }
       select_connection(conn1, selector) if from_pool
       conn2.coalesce!(conn1)
       conn2.disconnect
-      true
     end
 
     def get_current_selector
@@ -472,6 +472,7 @@ module HTTPX
       th.thread_variable_get(:httpx_persistent_selector_store)
     end
 
+    Options.freeze
     @default_options = Options.new
     @default_options.freeze
     @plugins = []
@@ -493,7 +494,6 @@ module HTTPX
       #
       def plugin(pl, options = nil, &block)
         label = pl
-        # raise Error, "Cannot add a plugin to a frozen config" if frozen?
         pl = Plugins.load_plugin(pl) if pl.is_a?(Symbol)
         raise ArgumentError, "Invalid plugin type: #{pl.class.inspect}" unless pl.is_a?(Module)
 

data/lib/httpx/transcoder/body.rb

@@ -12,7 +12,7 @@ module HTTPX::Transcoder
       def initialize(body)
         body = body.open(File::RDONLY, encoding: Encoding::BINARY) if Object.const_defined?(:Pathname) && body.is_a?(Pathname)
         @body = body
-        super(body)
+        super
       end
 
       def bytesize

data/lib/httpx/transcoder/json.rb

@@ -64,7 +64,7 @@ module HTTPX::Transcoder
     else
       require "json"
       def json_load(*args); ::JSON.parse(*args); end
-      def json_dump(*args); ::JSON.dump(*args); end
+      def json_dump(*args); ::JSON.generate(*args); end
     end
     # rubocop:enable Style/SingleLineMethods
   end

data/lib/httpx/transcoder/multipart/decoder.rb

@@ -60,7 +60,7 @@ module HTTPX
           when :idle
             raise Error, "payload does not start with boundary" unless @buffer.start_with?("#{@intermediate_boundary}#{CRLF}")
 
-            @buffer = @buffer.byteslice(@intermediate_boundary.bytesize + 2..-1)
+            @buffer = @buffer.byteslice((@intermediate_boundary.bytesize + 2)..-1)
 
             @state = :part_header
           when :part_header
@@ -70,7 +70,7 @@ module HTTPX
             return unless idx
 
             # @type var head: String
-            head = @buffer.byteslice(0..idx + 4 - 1)
+            head = @buffer.byteslice(0..(idx + 4 - 1))
 
             @buffer = @buffer.byteslice(head.bytesize..-1)
 
@@ -107,8 +107,8 @@ module HTTPX
             idx = @buffer.index(body_separator)
 
             if idx
-              payload = @buffer.byteslice(0..idx - 1)
-              @buffer = @buffer.byteslice(idx + body_separator.bytesize..-1)
+              payload = @buffer.byteslice(0..(idx - 1))
+              @buffer = @buffer.byteslice((idx + body_separator.bytesize)..-1)
               part << payload
               part.rewind if part.respond_to?(:rewind)
               @state = :parse_boundary

data/lib/httpx/transcoder/multipart/encoder.rb

@@ -53,7 +53,7 @@ module HTTPX
 
       def to_parts(form)
         params = form.each_with_object([]) do |(key, val), aux|
-          Transcoder.normalize_keys(key, val, MULTIPART_VALUE_COND) do |k, v|
+          Transcoder::Multipart.normalize_keys(key, val) do |k, v|
             next if v.nil?
 
             value, content_type, filename = Part.call(v)

data/lib/httpx/transcoder/multipart.rb

@@ -7,20 +7,28 @@ require_relative "multipart/mime_type_detector"
 
 module HTTPX::Transcoder
   module Multipart
-    MULTIPART_VALUE_COND = lambda do |value|
+    module_function
+
+    def multipart?(form_data)
+      form_data.any? do |_, v|
+        multipart_value?(v) ||
+          (v.respond_to?(:to_ary) && v.to_ary.any? { |av| multipart_value?(av) }) ||
+          (v.respond_to?(:to_hash) && v.to_hash.any? { |_, e| multipart_value?(e) })
+      end
+    end
+
+    def multipart_value?(value)
       value.respond_to?(:read) ||
         (value.respond_to?(:to_hash) &&
           value.key?(:body) &&
           (value.key?(:filename) || value.key?(:content_type)))
     end
 
-    module_function
-
-    def multipart?(form_data)
-      form_data.any? do |_, v|
-        Multipart::MULTIPART_VALUE_COND.call(v) ||
-          (v.respond_to?(:to_ary) && v.to_ary.any?(&Multipart::MULTIPART_VALUE_COND)) ||
-          (v.respond_to?(:to_hash) && v.to_hash.any? { |_, e| Multipart::MULTIPART_VALUE_COND.call(e) })
+    def normalize_keys(key, value, transcoder = self, &block)
+      if multipart_value?(value)
+        block.call(key.to_s, value)
+      else
+        HTTPX::Transcoder.normalize_keys(key, value, transcoder, &block)
       end
     end
 

data/lib/httpx/transcoder.rb

@@ -4,20 +4,18 @@ module HTTPX
   module Transcoder
     module_function
 
-    def normalize_keys(key, value, cond = nil, &block)
-      if cond && cond.call(value)
-        block.call(key.to_s, value)
-      elsif value.respond_to?(:to_ary)
+    def normalize_keys(key, value, transcoder = self, &block)
+      if value.respond_to?(:to_ary)
         if value.empty?
           block.call("#{key}[]")
         else
           value.to_ary.each do |element|
-            normalize_keys("#{key}[]", element, cond, &block)
+            transcoder.normalize_keys("#{key}[]", element, transcoder, &block)
           end
         end
       elsif value.respond_to?(:to_hash)
         value.to_hash.each do |child_key, child_value|
-          normalize_keys("#{key}[#{child_key}]", child_value, cond, &block)
+          transcoder.normalize_keys("#{key}[#{child_key}]", child_value, transcoder, &block)
         end
       else
         block.call(key.to_s, value)

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "1.6.3"
+  VERSION = "1.7.0"
 end

data/sig/altsvc.rbs

@@ -1,6 +1,7 @@
 module HTTPX
   module AltSvc
     module ConnectionMixin
+      H2_ALTSVC_SCHEMES: Array[String]
 
       def send: (Request request) -> void
 
@@ -11,6 +12,8 @@ module HTTPX
       def match_altsvcs?: (URI::Generic uri) -> bool
 
       def match_altsvc_options?: (URI::Generic uri, Options options) -> bool
+
+      def altsvc_match?: (uri uri, uri other_uri) -> bool
     end
 
     type altsvc_params = Hash[String, untyped]

data/sig/chainable.rbs

@@ -1,7 +1,7 @@
 module HTTPX
   module Chainable
-    def request: (*Request, **untyped) -> Array[response]
-               | (Request, **untyped) -> response
+    def request: (Request, **untyped) -> response
+               | (*Request, **untyped) -> Array[response]
                | (verb, uri | [uri], **untyped) -> response
                | (Array[[verb, uri] | [verb, uri, request_params]], **untyped) -> Array[response]
                | (verb, _Each[uri | [uri, request_params]], **untyped) -> Array[response]
@@ -12,7 +12,7 @@ module HTTPX
     def with: (options) -> Session
             | (options) { (Session) -> void } -> void
 
-    def plugin: (:auth, ?options) -> Plugins::sessionAuthorization
+    def plugin: (:auth, ?options) -> Plugins::sessionAuth
               | (:basic_auth, ?options) -> Plugins::sessionBasicAuth
               | (:digest_auth, ?options) -> Plugins::sessionDigestAuth
               | (:ntlm_auth, ?options) -> Plugins::sessionNTLMAuth

data/sig/connection.rbs

@@ -61,7 +61,7 @@ module HTTPX
 
     def addresses?: () -> boolish
 
-    def match?: (URI::Generic uri, Options options) -> bool
+    def match?: (http_uri uri, Options options) -> bool
 
     def mergeable?: (Connection connection) -> bool
 
@@ -71,8 +71,6 @@ module HTTPX
 
     def coalescable?: (Connection connection) -> bool
 
-    def create_idle: (?Hash[Symbol, untyped] options) -> instance
-
     def merge: (Connection connection) -> void
 
     def purge_pending: () { (Request request) -> void } -> void

data/sig/options.rbs

@@ -138,7 +138,7 @@ module HTTPX
 
     def options_equals?: (Options other, ?Array[Symbol] ignore_ivars) -> bool
 
-    def merge: (_ToHash[Symbol, untyped] other) -> instance
+    def merge: (Object & _ToHash[Symbol, untyped] other) -> (instance | self)
 
     def to_hash: () -> Hash[Symbol, untyped]
 

data/sig/plugins/auth/digest.rbs

@@ -2,9 +2,13 @@ module HTTPX
   module Plugins
     module Authentication
       class Digest
+        Error: singleton(HTTPX::Error)
+
+
         @user: String
         @password: String
         @hashed: bool
+        @nonce: Integer
 
         def can_authenticate?: (String? authenticate) -> boolish
 
@@ -19,6 +23,8 @@ module HTTPX
         def make_cnonce: () -> String
 
         def next_nonce: () -> Integer
+
+        def raise_format_error: () -> void
       end
     end
   end

data/sig/plugins/auth.rbs

@@ -1,13 +1,37 @@
 module HTTPX
   module Plugins
-    module Authorization
+    module Auth
+      interface _AuthOptions
+        def auth_header_value: () -> (String | ^(Request request) -> string)?
+
+        def auth_header_type: () -> String?
+      end
+
       module InstanceMethods
-        def authorization: (string token) -> instance
+        @auth_header_value: String
+        @skip_auth_header_value: bool
+
+        def authorization: (?string token, ?auth_header_type: string) ?{ (Request) -> string } -> instance
+
+        def bearer_auth: (?string token) ?{ (Request) -> string } -> instance
+
+        def skip_auth_header: [T] { () -> T } -> T
+
+        def reset_auth_header_value!: () -> void
+
+        private
+
+        def generate_auth_token: () -> String?
+      end
+
+      module RequestMethods
+        def authorize: (String auth_value) -> void
+      end
 
-        def bearer_auth: (string token) -> instance
+      module AuthRetries
       end
     end
 
-    type sessionAuthorization = Session & Authorization::InstanceMethods
+    type sessionAuth = Session & Auth::InstanceMethods
   end
 end

data/sig/plugins/basic_auth.rbs

@@ -1,15 +1,15 @@
 module HTTPX
   module Plugins
     module BasicAuth
-      def self.load_dependencies: (singleton(Session)) -> void
+      def self.load_dependencies: (singleton(Session) klass) -> void
 
-      def self.configure: (singleton(Session)) -> void
+      def self.configure: (singleton(Session) klass) -> void
 
       module InstanceMethods
         def basic_auth: (string user, string password) -> instance
       end
     end
 
-    type sessionBasicAuth = sessionAuthorization & BasicAuth::InstanceMethods
+    type sessionBasicAuth = sessionAuth & BasicAuth::InstanceMethods
   end
 end

data/sig/plugins/digest_auth.rbs

@@ -1,21 +1,19 @@
 module HTTPX
   module Plugins
     module DigestAuth
-      DigestError: singleton(Error)
-
       interface _DigestOptions
         def digest: () -> Authentication::Digest?
       end
 
       def self.extra_options: (Options) -> (Options & _DigestOptions)
 
-      def self.load_dependencies: (*untyped) -> void
+      def self.load_dependencies: (singleton(Session) klass) -> void
 
       module InstanceMethods
         def digest_auth: (string user, string password, ?hashed: bool) -> instance
       end
     end
 
-    type sessionDigestAuth = sessionAuthorization & DigestAuth::InstanceMethods
+    type sessionDigestAuth = sessionAuth & DigestAuth::InstanceMethods
   end
 end