httpx 1.7.0 → 1.7.1

data/lib/httpx/resolver/cache/base.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+require "resolv"
+
+module HTTPX
+  module Resolver::Cache
+    # Base class of the Resolver Cache adapter implementations.
+    #
+    # While resolver caches are not required to inherit from this class, it nevertheless provides
+    # common useful functions for desired functionality, such as singleton object ractor-safe access,
+    # or a default #resolve implementation which deals with IPs and the system hosts file.
+    #
+    class Base
+      MAX_CACHE_SIZE = 512
+      CACHE_MUTEX = Thread::Mutex.new
+      HOSTS = Resolv::Hosts.new
+      @cache = nil
+
+      class << self
+        attr_reader :hosts_resolver
+
+        # returns the singleton instance to be used within the current ractor.
+        def cache(label)
+          return Ractor.store_if_absent(:"httpx_resolver_cache_#{label}") { new } if Utils.in_ractor?
+
+          @cache ||= CACHE_MUTEX.synchronize do
+            @cache || new
+          end
+        end
+      end
+
+      # resolves +hostname+ into an instance of HTTPX::Resolver::Entry if +hostname+ is an IP,
+      # or can be found in the cache, or can be found in the system hosts file.
+      def resolve(hostname)
+        ip_resolve(hostname) || get(hostname) || hosts_resolve(hostname)
+      end
+
+      private
+
+      # tries to convert +hostname+ into an IPAddr, returns <tt>nil</tt> otherwise.
+      def ip_resolve(hostname)
+        [Resolver::Entry.new(hostname)]
+      rescue ArgumentError
+      end
+
+      # matches +hostname+ to entries in the hosts file, returns <tt>nil</nil> if none is
+      # found, or there is no hosts file.
+      def hosts_resolve(hostname)
+        ips = if Utils.in_ractor?
+          Ractor.store_if_absent(:httpx_hosts_resolver) { Resolv::Hosts.new }
+        else
+          HOSTS
+        end.getaddresses(hostname)
+
+        return if ips.empty?
+
+        ips.map { |ip| Resolver::Entry.new(ip) }
+      rescue IOError
+      end
+
+      # not to be used directly!
+      def _get(hostname, lookups, hostnames, ttl)
+        return unless lookups.key?(hostname)
+
+        entries = lookups[hostname]
+
+        return unless entries
+
+        entries.delete_if do |address|
+          address["TTL"] < ttl
+        end
+
+        if entries.empty?
+          lookups.delete(hostname)
+          hostnames.delete(hostname)
+        end
+
+        ips = entries.flat_map do |address|
+          if (als = address["alias"])
+            _get(als, lookups, hostnames, ttl)
+          else
+            Resolver::Entry.new(address["data"], address["TTL"])
+          end
+        end.compact
+
+        ips unless ips.empty?
+      end
+
+      def _set(hostname, family, entries, lookups, hostnames)
+        # lru cleanup
+        while lookups.size >= MAX_CACHE_SIZE
+          hs = hostnames.shift
+          lookups.delete(hs)
+        end
+        hostnames << hostname
+
+        lookups[hostname] ||= [] # when there's no default proc
+
+        case family
+        when Socket::AF_INET6
+          lookups[hostname].concat(entries)
+        when Socket::AF_INET
+          lookups[hostname].unshift(*entries)
+        end
+        entries.each do |entry|
+          name = entry["name"]
+          next unless name != hostname
+
+          lookups[name] ||= []
+
+          case family
+          when Socket::AF_INET6
+            lookups[name] << entry
+          when Socket::AF_INET
+            lookups[name].unshift(entry)
+          end
+        end
+      end
+
+      def _evict(hostname, ip, lookups, hostnames)
+        return unless lookups.key?(hostname)
+
+        entries = lookups[hostname]
+
+        return unless entries
+
+        entries.delete_if { |entry| entry["data"] == ip }
+
+        return unless entries.empty?
+
+        lookups.delete(hostname)
+        hostnames.delete(hostname)
+      end
+    end
+  end
+end

data/lib/httpx/resolver/cache/memory.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Resolver::Cache
+    # Implementation of a thread-safe in-memory LRU resolver cache.
+    class Memory < Base
+      def initialize
+        super
+        @hostnames = []
+        @lookups = Hash.new { |h, k| h[k] = [] }
+        @lookup_mutex = Thread::Mutex.new
+      end
+
+      def get(hostname)
+        now = Utils.now
+        synchronize do |lookups, hostnames|
+          _get(hostname, lookups, hostnames, now)
+        end
+      end
+
+      def set(hostname, family, entries)
+        synchronize do |lookups, hostnames|
+          _set(hostname, family, entries, lookups, hostnames)
+        end
+      end
+
+      def evict(hostname, ip)
+        ip = ip.to_s
+
+        synchronize do |lookups, hostnames|
+          _evict(hostname, ip, lookups, hostnames)
+        end
+      end
+
+      private
+
+      def synchronize
+        @lookup_mutex.synchronize { yield(@lookups, @hostnames) }
+      end
+    end
+  end
+end

data/lib/httpx/resolver/cache.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "httpx/resolver/cache/base"
+require "httpx/resolver/cache/memory"
+
+module HTTPX::Resolver
+  # The internal resolvers cache adapters are defined under this namespace.
+  #
+  # Adapters must comply with the Resolver Cache Adapter API and implement the following methods:
+  #
+  # * #resolve: (String hostname) -> Array[HTTPX::Entry]? => resolves hostname to a list of cached IPs (if found in cache or system)
+  # * #get: (String hostname) -> Array[HTTPX::Entry]? => resolves hostname to a list of cached IPs (if found in cache)
+  # * #set: (String hostname, Integer ip_family, Array[dns_result]) -> void => stores the set of results in the cache indexes for
+  #         the hostname and the IP family
+  # * #evict: (String hostname, _ToS ip) -> void => evicts the ip for the hostname from the cache (usually done when no longer reachable)
+  module Cache
+  end
+end

data/lib/httpx/resolver/https.rb

@@ -31,7 +31,7 @@ module HTTPX
       use_get: false,
     }.freeze
 
-    def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close,
+    def_delegators :@resolver_connection, :connecting?, :to_io, :call, :close,
                    :closed?, :deactivate, :terminate, :inflight?, :handle_socket_timeout
 
     def initialize(_, options)
@@ -48,10 +48,14 @@ module HTTPX
       @resolver.lazy_initialize
     end
 
+    def state
+      @resolver_connection ? @resolver_connection.state : :idle
+    end
+
     def <<(connection)
       return if @uri.origin == connection.peer.to_s
 
-      @uri_addresses ||= HTTPX::Resolver.nolookup_resolve(@uri.host) || @resolver.getaddresses(@uri.host)
+      @uri_addresses ||= @options.resolver_cache.resolve(@uri.host) || @resolver.getaddresses(@uri.host)
 
       if @uri_addresses.empty?
         ex = ResolveError.new("Can't resolve DNS server #{@uri.host}")
@@ -247,7 +251,7 @@ module HTTPX
           # eliminate other candidates
           @queries.delete_if { |_, conn| connection == conn }
 
-          Resolver.cached_lookup_set(hostname, @family, addresses) if @resolver_options[:cache]
+          @options.resolver_cache.set(hostname, @family, addresses) if @resolver_options[:cache]
           catch(:coalesced) { emit_addresses(connection, @family, addresses.map { |a| Resolver::Entry.new(a["data"], a["TTL"]) }) }
         end
       end

data/lib/httpx/resolver/multi.rb

@@ -19,8 +19,6 @@ module HTTPX
         resolver.multi = self
         resolver
       end
-
-      @errors = Hash.new { |hs, k| hs[k] = [] }
     end
 
     def state
@@ -47,7 +45,7 @@ module HTTPX
 
     def early_resolve(connection)
       hostname = connection.peer.host
-      addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
+      addresses = @resolver_options[:cache] && (connection.addresses || nolookup_resolve(hostname, connection.options))
       return false unless addresses
 
       ip_families = connection.options.ip_families
@@ -83,5 +81,11 @@ module HTTPX
         @current_session.select_resolver(resolver, @current_selector)
       end
     end
+
+    private
+
+    def nolookup_resolve(hostname, options)
+      options.resolver_cache.resolve(hostname)
+    end
   end
 end

data/lib/httpx/resolver/native.rb

@@ -136,7 +136,11 @@ module HTTPX
     private
 
     def calculate_interests
-      return if @queries.empty?
+      if @queries.empty?
+        return @io.interests if (@socket_type == :tcp) && (@state == :idle)
+
+        return
+      end
 
       return :r if @write_buffer.empty?
 
@@ -417,7 +421,7 @@ module HTTPX
           reset_hostname(name, connection: connection)
           @timeouts.delete(connection.peer.host)
           @connections.delete(connection)
-          Resolver.cached_lookup_set(connection.peer.host, @family, addresses) if @resolver_options[:cache]
+          @options.resolver_cache.set(connection.peer.host, @family, addresses) if @resolver_options[:cache]
           catch(:coalesced) do
             emit_addresses(connection, @family, addresses.map { |a| Resolver::Entry.new(a["data"], a["TTL"]) })
           end

data/lib/httpx/resolver/resolver.rb

@@ -125,7 +125,7 @@ module HTTPX
     end
 
     def early_resolve(connection, hostname: connection.peer.host) # rubocop:disable Naming/PredicateMethod
-      addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
+      addresses = @resolver_options[:cache] && (connection.addresses || @options.resolver_cache.resolve(hostname))
 
       return false unless addresses
 

data/lib/httpx/resolver.rb

@@ -8,150 +8,27 @@ module HTTPX
     extend self
 
     RESOLVE_TIMEOUT = [2, 3].freeze
-    MAX_CACHE_SIZE = 512
-
     require "httpx/resolver/entry"
+    require "httpx/resolver/cache"
     require "httpx/resolver/resolver"
     require "httpx/resolver/system"
     require "httpx/resolver/native"
     require "httpx/resolver/https"
     require "httpx/resolver/multi"
 
-    @lookup_mutex = Thread::Mutex.new
-    @hostnames = []
-    @lookups = Hash.new { |h, k| h[k] = [] }
-
     @identifier_mutex = Thread::Mutex.new
     @identifier = 1
-    @hosts_resolver = Resolv::Hosts.new
 
     def supported_ip_families
-      if in_ractor?
+      if Utils.in_ractor?
         Ractor.store_if_absent(:httpx_supported_ip_families) { find_supported_ip_families }
       else
         @supported_ip_families ||= find_supported_ip_families
       end
     end
 
-    def resolver_for(resolver_type, options)
-      case resolver_type
-      when Symbol
-        meth = :"resolver_#{resolver_type}_class"
-
-        return options.__send__(meth) if options.respond_to?(meth)
-      when Class
-        return resolver_type if resolver_type < Resolver
-      end
-
-      raise Error, "unsupported resolver type (#{resolver_type})"
-    end
-
-    def nolookup_resolve(hostname)
-      ip_resolve(hostname) || cached_lookup(hostname) || hosts_resolve(hostname)
-    end
-
-    # tries to convert +hostname+ into an IPAddr, returns <tt>nil</tt> otherwise.
-    def ip_resolve(hostname)
-      [Entry.new(hostname)]
-    rescue ArgumentError
-    end
-
-    # matches +hostname+ to entries in the hosts file, returns <tt>nil</nil> if none is
-    # found, or there is no hosts file.
-    def hosts_resolve(hostname)
-      ips = if in_ractor?
-        Ractor.store_if_absent(:httpx_hosts_resolver) { Resolv::Hosts.new }
-      else
-        @hosts_resolver
-      end.getaddresses(hostname)
-
-      return if ips.empty?
-
-      ips.map { |ip| Entry.new(ip) }
-    rescue IOError
-    end
-
-    def cached_lookup(hostname)
-      now = Utils.now
-      lookup_synchronize do |lookups, hostnames|
-        lookup(hostname, lookups, hostnames, now)
-      end
-    end
-
-    def cached_lookup_set(hostname, family, entries)
-      lookup_synchronize do |lookups, hostnames|
-        # lru cleanup
-        while lookups.size >= MAX_CACHE_SIZE
-          hs = hostnames.shift
-          lookups.delete(hs)
-        end
-        hostnames << hostname
-
-        case family
-        when Socket::AF_INET6
-          lookups[hostname].concat(entries)
-        when Socket::AF_INET
-          lookups[hostname].unshift(*entries)
-        end
-        entries.each do |entry|
-          name = entry["name"]
-          next unless name != hostname
-
-          case family
-          when Socket::AF_INET6
-            lookups[name] << entry
-          when Socket::AF_INET
-            lookups[name].unshift(entry)
-          end
-        end
-      end
-    end
-
-    def cached_lookup_evict(hostname, ip)
-      ip = ip.to_s
-
-      lookup_synchronize do |lookups, hostnames|
-        entries = lookups[hostname]
-
-        return unless entries
-
-        entries.delete_if { |entry| entry["data"] == ip }
-
-        if entries.empty?
-          lookups.delete(hostname)
-          hostnames.delete(hostname)
-        end
-      end
-    end
-
-    # do not use directly!
-    def lookup(hostname, lookups, hostnames, ttl)
-      return unless lookups.key?(hostname)
-
-      entries = lookups[hostname]
-
-      entries.delete_if do |address|
-        address["TTL"] < ttl
-      end
-
-      if entries.empty?
-        lookups.delete(hostname)
-        hostnames.delete(hostname)
-      end
-
-      ips = entries.flat_map do |address|
-        if (als = address["alias"])
-          lookup(als, lookups, hostnames, ttl)
-        else
-          Entry.new(address["data"], address["TTL"])
-        end
-      end.compact
-
-      ips unless ips.empty?
-    end
-
     def generate_id
-      if in_ractor?
+      if Utils.in_ractor?
         identifier = Ractor.store_if_absent(:httpx_resolver_identifier) { -1 }
         Ractor.current[:httpx_resolver_identifier] = (identifier + 1) & 0xFFFF
       else
@@ -213,16 +90,6 @@ module HTTPX
 
     private
 
-    def lookup_synchronize
-      if in_ractor?
-        lookups = Ractor.store_if_absent(:httpx_resolver_lookups) { Hash.new { |h, k| h[k] = [] } }
-        hostnames = Ractor.store_if_absent(:httpx_resolver_hostnames) { [] }
-        return yield(lookups, hostnames)
-      end
-
-      @lookup_mutex.synchronize { yield(@lookups, @hostnames) }
-    end
-
     def id_synchronize(&block)
       @identifier_mutex.synchronize(&block)
     end
@@ -240,18 +107,5 @@ module HTTPX
         [Socket::AF_INET]
       end.freeze
     end
-
-    if defined?(Ractor) &&
-       # no ractor support for 3.0
-       RUBY_VERSION >= "3.1.0"
-
-      def in_ractor?
-        Ractor.main != Ractor.current
-      end
-    else
-      def in_ractor?
-        false
-      end
-    end
   end
 end

data/lib/httpx/response/body.rb

@@ -59,13 +59,11 @@ module HTTPX
 
       chunk = decode_chunk(chunk)
 
-      size = chunk.bytesize
-      @length += size
       transition(:open)
       @buffer.write(chunk)
 
       @response.emit(:chunk_received, chunk)
-      size
+      chunk.bytesize
     end
 
     # reads a chunk from the payload (implementation of the IO reader protocol).
@@ -209,6 +207,8 @@ module HTTPX
         chunk = inflater.call(chunk)
       end if @inflaters
 
+      @length += chunk.bytesize
+
       chunk
     end
 

data/lib/httpx/selector.rb

@@ -127,13 +127,15 @@ module HTTPX
     private
 
     def select(interval, &block)
-      has_no_selectables = @selectables.empty?
       # do not cause an infinite loop here.
       #
       # this may happen if timeout calculation actually triggered an error which causes
       # the connections to be reaped (such as the total timeout error) before #select
       # gets called.
-      return if interval.nil? && has_no_selectables
+      if @selectables.empty?
+        sleep(interval) if interval
+        return
+      end
 
       # @type var r: (selectable | Array[selectable])?
       # @type var w: (selectable | Array[selectable])?
@@ -171,7 +173,7 @@ module HTTPX
         when Array
           select_many(r, w, interval, &block)
         when nil
-          return unless interval && has_no_selectables
+          return unless interval && @selectables.any?
 
           # no selectables
           # TODO: replace with sleep?

data/lib/httpx/session.rb

@@ -177,7 +177,7 @@ module HTTPX
 
     # returns the HTTPX::Connection through which the +request+ should be sent through.
     def find_connection(request_uri, selector, options)
-      log(level: 2) { "finding connection for ##{request_uri}..." }
+      log(level: 2) { "finding connection for #{request_uri}..." }
       if (connection = selector.find_connection(request_uri, options))
         connection.idling if connection.state == :closed
         connection.log(level: 2) { "found connection##{connection.object_id}(#{connection.state}) in selector##{selector.object_id}" }

data/lib/httpx/timers.rb

@@ -29,13 +29,15 @@ module HTTPX
     end
 
     def wait_interval
-      drop_elapsed!
-
       return if @intervals.empty?
 
+      first_interval = @intervals.first
+
+      drop_elapsed!(0) if first_interval.elapsed?(0)
+
       @next_interval_at = Utils.now
 
-      @intervals.first.interval
+      first_interval.interval
     end
 
     def fire(error = nil)
@@ -46,20 +48,12 @@ module HTTPX
 
       drop_elapsed!(elapsed_time)
 
-      @intervals = @intervals.drop_while { |interval| interval.elapse(elapsed_time) <= 0 }
-
       @next_interval_at = nil if @intervals.empty?
     end
 
     private
 
-    def drop_elapsed!(elapsed_time = 0)
-      # check first, if not elapsed, then return
-      first_interval = @intervals.first
-
-      return unless first_interval && first_interval.elapsed?(elapsed_time)
-
-      # TODO: would be nice to have a drop_while!
+    def drop_elapsed!(elapsed_time)
       @intervals = @intervals.drop_while { |interval| interval.elapse(elapsed_time) <= 0 }
     end
 

data/lib/httpx/transcoder/gzip.rb

@@ -8,6 +8,7 @@ module HTTPX
       class Deflater < Transcoder::Deflater
         def initialize(body)
           @compressed_chunk = "".b
+          @deflater = nil
           super
         end
 
@@ -28,8 +29,12 @@ module HTTPX
 
         private
 
-        def write(chunk)
-          @compressed_chunk << chunk
+        def write(*chunks)
+          chunks.sum do |chunk|
+            chunk = chunk.to_s
+            @compressed_chunk << chunk
+            chunk.bytesize
+          end
         end
 
         def compressed_chunk

data/lib/httpx/transcoder/multipart/decoder.rb

@@ -12,7 +12,6 @@ module HTTPX
         def initialize(filename, content_type)
           @original_filename = filename
           @content_type = content_type
-          @current = nil
           @file = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
           super(@file)
         end
@@ -39,6 +38,7 @@ module HTTPX
           @parts = {}
           @intermediate_boundary = "--#{@boundary}"
           @state = :idle
+          @current = nil
         end
 
         def call(response, *)

data/lib/httpx/transcoder/multipart.rb

@@ -19,7 +19,7 @@ module HTTPX::Transcoder
 
     def multipart_value?(value)
       value.respond_to?(:read) ||
-        (value.respond_to?(:to_hash) &&
+        (value.is_a?(Hash) &&
           value.key?(:body) &&
           (value.key?(:filename) || value.key?(:content_type)))
     end

data/lib/httpx/utils.rb

@@ -71,5 +71,18 @@ module HTTPX
       uri.non_ascii_hostname = non_ascii_hostname
       uri
     end
+
+    if defined?(Ractor) &&
+       # no ractor support for 3.0
+       RUBY_VERSION >= "3.1.0"
+
+      def in_ractor?
+        Ractor.main != Ractor.current
+      end
+    else
+      def in_ractor?
+        false
+      end
+    end
   end
 end

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "1.7.0"
+  VERSION = "1.7.1"
 end

data/sig/altsvc.rbs

@@ -5,19 +5,22 @@ module HTTPX
 
       def send: (Request request) -> void
 
-      def match?: (URI::Generic uri, Options options) -> bool
+      def match?: (http_uri uri, Options options) -> bool
 
       private
 
-      def match_altsvcs?: (URI::Generic uri) -> bool
+      def match_altsvcs?: (http_uri uri) -> bool
 
-      def match_altsvc_options?: (URI::Generic uri, Options options) -> bool
+      def match_altsvc_options?: (http_uri uri, Options options) -> bool
 
-      def altsvc_match?: (uri uri, uri other_uri) -> bool
+      def altsvc_match?: (http_uri uri, uri other_uri) -> bool
     end
 
     type altsvc_params = Hash[String, untyped]
 
+    self.@altsvc_mutex: Thread::Mutex
+    self.@altsvcs: Hash[String, Array[altsvc_params]]
+
     def self?.cached_altsvc: (String origin) -> Array[altsvc_params]
 
     def self?.cached_altsvc_set: (String origin, altsvc_params) -> void