httpx 1.6.2 → 1.7.0

data/lib/httpx/resolver.rb

@@ -5,7 +5,10 @@ require "resolv"
 
 module HTTPX
   module Resolver
+    extend self
+
     RESOLVE_TIMEOUT = [2, 3].freeze
+    MAX_CACHE_SIZE = 512
 
     require "httpx/resolver/entry"
     require "httpx/resolver/resolver"
@@ -15,26 +18,19 @@ module HTTPX
     require "httpx/resolver/multi"
 
     @lookup_mutex = Thread::Mutex.new
+    @hostnames = []
     @lookups = Hash.new { |h, k| h[k] = [] }
 
     @identifier_mutex = Thread::Mutex.new
     @identifier = 1
     @hosts_resolver = Resolv::Hosts.new
 
-    module_function
-
     def supported_ip_families
-      @supported_ip_families ||= begin
-        # https://github.com/ruby/resolv/blob/095f1c003f6073730500f02acbdbc55f83d70987/lib/resolv.rb#L408
-        list = Socket.ip_address_list
-        if list.any? { |a| a.ipv6? && !a.ipv6_loopback? && !a.ipv6_linklocal? }
-          [Socket::AF_INET6, Socket::AF_INET]
-        else
-          [Socket::AF_INET]
-        end
-      rescue NotImplementedError
-        [Socket::AF_INET]
-      end.freeze
+      if in_ractor?
+        Ractor.store_if_absent(:httpx_supported_ip_families) { find_supported_ip_families }
+      else
+        @supported_ip_families ||= find_supported_ip_families
+      end
     end
 
     def resolver_for(resolver_type, options)
@@ -63,7 +59,12 @@ module HTTPX
     # matches +hostname+ to entries in the hosts file, returns <tt>nil</nil> if none is
     # found, or there is no hosts file.
     def hosts_resolve(hostname)
-      ips = @hosts_resolver.getaddresses(hostname)
+      ips = if in_ractor?
+        Ractor.store_if_absent(:httpx_hosts_resolver) { Resolv::Hosts.new }
+      else
+        @hosts_resolver
+      end.getaddresses(hostname)
+
       return if ips.empty?
 
       ips.map { |ip| Entry.new(ip) }
@@ -72,13 +73,20 @@ module HTTPX
 
     def cached_lookup(hostname)
       now = Utils.now
-      lookup_synchronize do |lookups|
-        lookup(hostname, lookups, now)
+      lookup_synchronize do |lookups, hostnames|
+        lookup(hostname, lookups, hostnames, now)
       end
     end
 
     def cached_lookup_set(hostname, family, entries)
-      lookup_synchronize do |lookups|
+      lookup_synchronize do |lookups, hostnames|
+        # lru cleanup
+        while lookups.size >= MAX_CACHE_SIZE
+          hs = hostnames.shift
+          lookups.delete(hs)
+        end
+        hostnames << hostname
+
         case family
         when Socket::AF_INET6
           lookups[hostname].concat(entries)
@@ -86,13 +94,14 @@ module HTTPX
           lookups[hostname].unshift(*entries)
         end
         entries.each do |entry|
-          next unless entry["name"] != hostname
+          name = entry["name"]
+          next unless name != hostname
 
           case family
           when Socket::AF_INET6
-            lookups[entry["name"]] << entry
+            lookups[name] << entry
           when Socket::AF_INET
-            lookups[entry["name"]].unshift(entry)
+            lookups[name].unshift(entry)
           end
         end
       end
@@ -101,26 +110,38 @@ module HTTPX
     def cached_lookup_evict(hostname, ip)
       ip = ip.to_s
 
-      lookup_synchronize do |lookups|
+      lookup_synchronize do |lookups, hostnames|
         entries = lookups[hostname]
 
         return unless entries
 
-        lookups.delete_if { |entry| entry["data"] == ip }
+        entries.delete_if { |entry| entry["data"] == ip }
+
+        if entries.empty?
+          lookups.delete(hostname)
+          hostnames.delete(hostname)
+        end
       end
     end
 
     # do not use directly!
-    def lookup(hostname, lookups, ttl)
+    def lookup(hostname, lookups, hostnames, ttl)
       return unless lookups.key?(hostname)
 
-      entries = lookups[hostname] = lookups[hostname].select do |address|
-        address["TTL"] > ttl
+      entries = lookups[hostname]
+
+      entries.delete_if do |address|
+        address["TTL"] < ttl
+      end
+
+      if entries.empty?
+        lookups.delete(hostname)
+        hostnames.delete(hostname)
       end
 
       ips = entries.flat_map do |address|
         if (als = address["alias"])
-          lookup(als, lookups, ttl)
+          lookup(als, lookups, hostnames, ttl)
         else
           Entry.new(address["data"], address["TTL"])
         end
@@ -130,7 +151,12 @@ module HTTPX
     end
 
     def generate_id
-      id_synchronize { @identifier = (@identifier + 1) & 0xFFFF }
+      if in_ractor?
+        identifier = Ractor.store_if_absent(:httpx_resolver_identifier) { -1 }
+        Ractor.current[:httpx_resolver_identifier] = (identifier + 1) & 0xFFFF
+      else
+        id_synchronize { @identifier = (@identifier + 1) & 0xFFFF }
+      end
     end
 
     def encode_dns_query(hostname, type: Resolv::DNS::Resource::IN::A, message_id: generate_id)
@@ -152,7 +178,14 @@ module HTTPX
 
       return :message_truncated if message.tc == 1
 
-      return :dns_error, message.rcode if message.rcode != Resolv::DNS::RCode::NoError
+      if message.rcode != Resolv::DNS::RCode::NoError
+        case message.rcode
+        when Resolv::DNS::RCode::ServFail
+          return :retriable_error, message.rcode
+        else
+          return :dns_error, message.rcode
+        end
+      end
 
       addresses = []
 
@@ -178,12 +211,47 @@ module HTTPX
       [:ok, addresses]
     end
 
+    private
+
     def lookup_synchronize
-      @lookup_mutex.synchronize { yield(@lookups) }
+      if in_ractor?
+        lookups = Ractor.store_if_absent(:httpx_resolver_lookups) { Hash.new { |h, k| h[k] = [] } }
+        hostnames = Ractor.store_if_absent(:httpx_resolver_hostnames) { [] }
+        return yield(lookups, hostnames)
+      end
+
+      @lookup_mutex.synchronize { yield(@lookups, @hostnames) }
     end
 
     def id_synchronize(&block)
       @identifier_mutex.synchronize(&block)
     end
+
+    def find_supported_ip_families
+      list = Socket.ip_address_list
+
+      begin
+        if list.any? { |a| a.ipv6? && !a.ipv6_loopback? && !a.ipv6_linklocal? }
+          [Socket::AF_INET6, Socket::AF_INET]
+        else
+          [Socket::AF_INET]
+        end
+      rescue NotImplementedError
+        [Socket::AF_INET]
+      end.freeze
+    end
+
+    if defined?(Ractor) &&
+       # no ractor support for 3.0
+       RUBY_VERSION >= "3.1.0"
+
+      def in_ractor?
+        Ractor.main != Ractor.current
+      end
+    else
+      def in_ractor?
+        false
+      end
+    end
   end
 end

data/lib/httpx/response/body.rb

@@ -138,6 +138,8 @@ module HTTPX
       else
         IO.copy_stream(@buffer, dest)
       end
+    ensure
+      close
     end
 
     # closes/cleans the buffer, resets everything

data/lib/httpx/response.rb

@@ -211,16 +211,19 @@ module HTTPX
 
     def initialize(header_value)
       @header_value = header_value
+      @mime_type = @charset = nil
+      @initialized = false
     end
 
     # returns the mime type declared in the header.
     #
     #   ContentType.new("application/json; charset=utf-8").mime_type #=> "application/json"
     def mime_type
-      return @mime_type if defined?(@mime_type)
+      return @mime_type if @initialized
 
-      m = @header_value.to_s[MIME_TYPE_RE, 1]
-      m && @mime_type = m.strip.downcase
+      load
+
+      @mime_type
     end
 
     # returns the charset declared in the header.
@@ -228,10 +231,23 @@ module HTTPX
     #   ContentType.new("application/json; charset=utf-8").charset #=> "utf-8"
     #   ContentType.new("text/plain").charset #=> nil
     def charset
-      return @charset if defined?(@charset)
+      return @charset if @initialized
+
+      load
+
+      @charset
+    end
+
+    private
+
+    def load
+      m = @header_value.to_s[MIME_TYPE_RE, 1]
+      m && @mime_type = m.strip.downcase
+
+      c = @header_value.to_s[CHARSET_RE, 1]
+      c && @charset = c.strip.delete('"')
 
-      m = @header_value.to_s[CHARSET_RE, 1]
-      m && @charset = m.strip.delete('"')
+      @initialized = true
     end
   end
 

data/lib/httpx/selector.rb

@@ -51,7 +51,7 @@ module HTTPX
 
         begin
           select(timeout) do |c|
-            c.log(level: 2) { "[#{c.state}] selected#{" after #{timeout} secs" unless timeout.nil?}..." }
+            c.log(level: 2) { "[#{c.state}] selected from selector##{object_id} #{" after #{timeout} secs" unless timeout.nil?}..." }
 
             c.call
           end
@@ -61,17 +61,6 @@ module HTTPX
           @timers.fire(e)
         end
       end
-    rescue StandardError => e
-      each_connection do |c|
-        c.emit(:error, e)
-      end
-    rescue Exception # rubocop:disable Lint/RescueException
-      each_connection do |conn|
-        conn.force_reset
-        conn.disconnect
-      end
-
-      raise
     end
 
     def terminate
@@ -157,7 +146,9 @@ module HTTPX
 
         next(is_closed) if is_closed
 
-        io.log(level: 2) { "[#{io.state}] registering for select (#{interests})#{" for #{interval} seconds" unless interval.nil?}" }
+        io.log(level: 2) do
+          "[#{io.state}] registering in selector##{object_id} for select (#{interests})#{" for #{interval} seconds" unless interval.nil?}"
+        end
 
         if READABLE.include?(interests)
           r = r.nil? ? io : (Array(r) << io)
@@ -206,7 +197,27 @@ module HTTPX
     end
 
     def select_many(r, w, interval, &block)
-      readers, writers = ::IO.select(r, w, nil, interval)
+      begin
+        readers, writers = ::IO.select(r, w, nil, interval)
+      rescue IOError => e
+        (Array(r) + Array(w)).each do |sel|
+          # TODO: is there a way to cheaply find the IO associated with the error?
+          sel.on_error(e)
+          sel.force_close(true)
+        end
+      rescue StandardError => e
+        (Array(r) + Array(w)).each do |sel|
+          sel.on_error(e)
+        end
+
+        return
+      rescue Exception => e # rubocop:disable Lint/RescueException
+        (Array(r) + Array(w)).each do |sel|
+          sel.force_close(true)
+        end
+
+        raise e
+      end
 
       if readers.nil? && writers.nil? && interval
         [*r, *w].each { |io| io.handle_socket_timeout(interval) }
@@ -228,12 +239,25 @@ module HTTPX
     end
 
     def select_one(io, interests, interval)
-      result =
-        case interests
-        when :r then io.to_io.wait_readable(interval)
-        when :w then io.to_io.wait_writable(interval)
-        when :rw then rw_wait(io, interval)
-        end
+      begin
+        result =
+          case interests
+          when :r then io.to_io.wait_readable(interval)
+          when :w then io.to_io.wait_writable(interval)
+          when :rw then rw_wait(io, interval)
+          end
+      rescue IOError => e
+        io.on_error(e)
+        io.force_close(true)
+      rescue StandardError => e
+        io.on_error(e)
+
+        return
+      rescue Exception => e # rubocop:disable Lint/RescueException
+        io.force_close(true)
+
+        raise e
+      end
 
       unless result || interval.nil?
         io.handle_socket_timeout(interval) unless @is_timer_interval

data/lib/httpx/session.rb

@@ -120,13 +120,16 @@ module HTTPX
     end
 
     def select_connection(connection, selector)
-      pin_connection(connection, selector)
+      pin(connection, selector)
+      connection.log(level: 2) do
+        "registering into selector##{selector.object_id}"
+      end
       selector.register(connection)
     end
 
-    def pin_connection(connection, selector)
-      connection.current_session = self
-      connection.current_selector = selector
+    def pin(conn_or_resolver, selector)
+      conn_or_resolver.current_session = self
+      conn_or_resolver.current_selector = selector
     end
 
     alias_method :select_resolver, :select_connection
@@ -137,9 +140,6 @@ module HTTPX
       end
       selector.deregister(connection)
 
-      # when connections coalesce
-      return if connection.state == :idle
-
       return if cloned
 
       return if @closing && connection.state == :closed
@@ -177,6 +177,7 @@ module HTTPX
 
     # returns the HTTPX::Connection through which the +request+ should be sent through.
     def find_connection(request_uri, selector, options)
+      log(level: 2) { "finding connection for ##{request_uri}..." }
       if (connection = selector.find_connection(request_uri, options))
         connection.idling if connection.state == :closed
         connection.log(level: 2) { "found connection##{connection.object_id}(#{connection.state}) in selector##{selector.object_id}" }
@@ -194,7 +195,7 @@ module HTTPX
         if options.io
           select_connection(connection, selector)
         else
-          pin_connection(connection, selector)
+          pin(connection, selector)
         end
       when :closing, :closed
         connection.idling
@@ -205,7 +206,7 @@ module HTTPX
           resolve_connection(connection, selector)
         end
       else
-        pin_connection(connection, selector)
+        pin(connection, selector)
       end
 
       connection
@@ -224,13 +225,7 @@ module HTTPX
 
     # tries deactivating connections in the +selector+, deregistering the ones that have been deactivated.
     def deactivate(selector)
-      selector.each_connection.select do |c|
-        c.deactivate
-
-        c.state == :inactive
-      end.each do |c| # rubocop:disable Style/MultilineBlockChain
-        deselect_connection(c, selector)
-      end
+      selector.each_connection.to_a.each(&:deactivate)
     end
 
     # callback executed when an HTTP/2 promise frame has been received.
@@ -395,6 +390,7 @@ module HTTPX
 
       resolver = find_resolver_for(connection, selector)
 
+      pin(connection, selector)
       resolver.early_resolve(connection) || resolver.lazy_resolve(connection)
     end
 
@@ -402,6 +398,9 @@ module HTTPX
       from_pool = false
       found_connection = selector.find_mergeable_connection(connection) || begin
         from_pool = true
+        connection.log(level: 2) do
+          "try finding a mergeable connection in pool##{@pool.object_id}"
+        end
         @pool.checkout_mergeable_connection(connection)
       end
 
@@ -409,29 +408,21 @@ module HTTPX
 
       connection.log(level: 2) do
         "try coalescing from #{from_pool ? "pool##{@pool.object_id}" : "selector##{selector.object_id}"} " \
-          "(conn##{found_connection.object_id}[#{found_connection.origin}])"
+          "(connection##{found_connection.object_id}[#{found_connection.origin}])"
       end
 
       coalesce_connections(found_connection, connection, selector, from_pool)
     end
 
-    def on_resolver_close(resolver, selector)
-      return if resolver.closed?
-
-      deselect_resolver(resolver, selector)
-      resolver.close unless resolver.closed?
-    end
-
     def find_resolver_for(connection, selector)
       if (resolver = selector.find_resolver(connection.options))
-        resolver.log(level: 2) { "found resolver##{connection.object_id}(#{connection.state}) in selector##{selector.object_id}" }
+        resolver.log(level: 2) { "found resolver##{resolver.object_id}(#{resolver.state}) in selector##{selector.object_id}" }
         return resolver
       end
 
       resolver = @pool.checkout_resolver(connection.options)
-      resolver.log(level: 2) { "found resolver##{connection.object_id}(#{connection.state}) in pool##{@pool.object_id}" }
-      resolver.current_session = self
-      resolver.current_selector = selector
+      resolver.log(level: 2) { "found resolver##{resolver.object_id}(#{resolver.state}) in pool##{@pool.object_id}" }
+      pin(resolver, selector)
 
       resolver
     end
@@ -446,14 +437,13 @@ module HTTPX
           conn1.log(level: 2) { "check-in connection##{conn1.object_id}(#{conn1.state}) in pool##{@pool.object_id}" }
           @pool.checkin_connection(conn1)
         end
-        return false
+        return
       end
 
-      conn2.log(level: 2) { "coalescing with conn##{conn1.object_id}[#{conn1.origin}])" }
-      conn2.coalesce!(conn1)
+      conn2.log(level: 2) { "coalescing with connection##{conn1.object_id}[#{conn1.origin}])" }
       select_connection(conn1, selector) if from_pool
+      conn2.coalesce!(conn1)
       conn2.disconnect
-      true
     end
 
     def get_current_selector
@@ -482,6 +472,7 @@ module HTTPX
       th.thread_variable_get(:httpx_persistent_selector_store)
     end
 
+    Options.freeze
     @default_options = Options.new
     @default_options.freeze
     @plugins = []
@@ -503,7 +494,6 @@ module HTTPX
       #
       def plugin(pl, options = nil, &block)
         label = pl
-        # raise Error, "Cannot add a plugin to a frozen config" if frozen?
         pl = Plugins.load_plugin(pl) if pl.is_a?(Symbol)
         raise ArgumentError, "Invalid plugin type: #{pl.class.inspect}" unless pl.is_a?(Module)
 

data/lib/httpx/transcoder/body.rb

@@ -12,7 +12,7 @@ module HTTPX::Transcoder
       def initialize(body)
         body = body.open(File::RDONLY, encoding: Encoding::BINARY) if Object.const_defined?(:Pathname) && body.is_a?(Pathname)
         @body = body
-        super(body)
+        super
       end
 
       def bytesize

data/lib/httpx/transcoder/deflate.rb

@@ -10,15 +10,20 @@ module HTTPX
         def deflate(chunk)
           @deflater ||= Zlib::Deflate.new
 
-          if chunk.nil?
-            unless @deflater.closed?
-              last = @deflater.finish
-              @deflater.close
-              last.empty? ? nil : last
-            end
-          else
-            @deflater.deflate(chunk)
+          unless chunk.nil?
+            chunk = @deflater.deflate(chunk)
+
+            # deflate call may return nil, while still
+            # retaining the last chunk in the deflater.
+            return chunk unless chunk.empty?
           end
+
+          return if @deflater.closed?
+
+          last = @deflater.finish
+          @deflater.close
+
+          last unless last.empty?
         end
       end
 

data/lib/httpx/transcoder/json.rb

@@ -64,7 +64,7 @@ module HTTPX::Transcoder
     else
       require "json"
       def json_load(*args); ::JSON.parse(*args); end
-      def json_dump(*args); ::JSON.dump(*args); end
+      def json_dump(*args); ::JSON.generate(*args); end
     end
     # rubocop:enable Style/SingleLineMethods
   end

data/lib/httpx/transcoder/multipart/decoder.rb

@@ -60,7 +60,7 @@ module HTTPX
           when :idle
             raise Error, "payload does not start with boundary" unless @buffer.start_with?("#{@intermediate_boundary}#{CRLF}")
 
-            @buffer = @buffer.byteslice(@intermediate_boundary.bytesize + 2..-1)
+            @buffer = @buffer.byteslice((@intermediate_boundary.bytesize + 2)..-1)
 
             @state = :part_header
           when :part_header
@@ -70,7 +70,7 @@ module HTTPX
             return unless idx
 
             # @type var head: String
-            head = @buffer.byteslice(0..idx + 4 - 1)
+            head = @buffer.byteslice(0..(idx + 4 - 1))
 
             @buffer = @buffer.byteslice(head.bytesize..-1)
 
@@ -107,8 +107,8 @@ module HTTPX
             idx = @buffer.index(body_separator)
 
             if idx
-              payload = @buffer.byteslice(0..idx - 1)
-              @buffer = @buffer.byteslice(idx + body_separator.bytesize..-1)
+              payload = @buffer.byteslice(0..(idx - 1))
+              @buffer = @buffer.byteslice((idx + body_separator.bytesize)..-1)
               part << payload
               part.rewind if part.respond_to?(:rewind)
               @state = :parse_boundary

data/lib/httpx/transcoder/multipart/encoder.rb

@@ -53,7 +53,7 @@ module HTTPX
 
       def to_parts(form)
         params = form.each_with_object([]) do |(key, val), aux|
-          Transcoder.normalize_keys(key, val, MULTIPART_VALUE_COND) do |k, v|
+          Transcoder::Multipart.normalize_keys(key, val) do |k, v|
             next if v.nil?
 
             value, content_type, filename = Part.call(v)

data/lib/httpx/transcoder/multipart.rb

@@ -7,20 +7,28 @@ require_relative "multipart/mime_type_detector"
 
 module HTTPX::Transcoder
   module Multipart
-    MULTIPART_VALUE_COND = lambda do |value|
+    module_function
+
+    def multipart?(form_data)
+      form_data.any? do |_, v|
+        multipart_value?(v) ||
+          (v.respond_to?(:to_ary) && v.to_ary.any? { |av| multipart_value?(av) }) ||
+          (v.respond_to?(:to_hash) && v.to_hash.any? { |_, e| multipart_value?(e) })
+      end
+    end
+
+    def multipart_value?(value)
       value.respond_to?(:read) ||
         (value.respond_to?(:to_hash) &&
           value.key?(:body) &&
           (value.key?(:filename) || value.key?(:content_type)))
     end
 
-    module_function
-
-    def multipart?(form_data)
-      form_data.any? do |_, v|
-        Multipart::MULTIPART_VALUE_COND.call(v) ||
-          (v.respond_to?(:to_ary) && v.to_ary.any?(&Multipart::MULTIPART_VALUE_COND)) ||
-          (v.respond_to?(:to_hash) && v.to_hash.any? { |_, e| Multipart::MULTIPART_VALUE_COND.call(e) })
+    def normalize_keys(key, value, transcoder = self, &block)
+      if multipart_value?(value)
+        block.call(key.to_s, value)
+      else
+        HTTPX::Transcoder.normalize_keys(key, value, transcoder, &block)
       end
     end
 

data/lib/httpx/transcoder/utils/body_reader.rb

@@ -28,8 +28,7 @@ module HTTPX
         begin
           chunk = @body.next
           if outbuf
-            outbuf.clear.force_encoding(Encoding::BINARY)
-            outbuf << chunk
+            outbuf.replace(chunk)
           else
             outbuf = chunk
           end

data/lib/httpx/transcoder/utils/deflater.rb

@@ -31,8 +31,7 @@ module HTTPX
         return unless compressed_chunk
 
         if outbuf
-          outbuf.clear.force_encoding(Encoding::BINARY)
-          outbuf << compressed_chunk
+          outbuf.replace(compressed_chunk)
         else
           compressed_chunk
         end