httpx 1.5.1 → 1.6.1

data/lib/httpx/plugins/retries.rb

@@ -57,6 +57,8 @@ module HTTPX
       # :retry_on :: callable which alternatively defines a different rule for when a response is to be retried
       #              (i.e. <tt>->(res) { ... }</tt>).
       module OptionsMethods
+        private
+
         def option_retry_after(value)
           # return early if callable
           unless value.respond_to?(:call)

data/lib/httpx/plugins/ssrf_filter.rb

@@ -91,6 +91,8 @@ module HTTPX
       #
       # :allowed_schemes :: list of URI schemes allowed (defaults to <tt>["https", "http"]</tt>)
       module OptionsMethods
+        private
+
         def option_allowed_schemes(value)
           Array(value)
         end
@@ -129,8 +131,6 @@ module HTTPX
         end
 
         def addresses=(addrs)
-          addrs = addrs.map { |addr| addr.is_a?(IPAddr) ? addr : IPAddr.new(addr) }
-
           addrs.reject!(&SsrfFilter.method(:unsafe_ip_address?))
 
           raise ServerSideRequestForgeryError, "#{@origin.host} has no public IP addresses" if addrs.empty?

data/lib/httpx/plugins/stream_bidi.rb

@@ -42,7 +42,7 @@ module HTTPX
         %i[join_headers join_trailers join_body].each do |lock_meth|
           class_eval(<<-METH, __FILE__, __LINE__ + 1)
             # lock.aware version of +#{lock_meth}+
-            def #{lock_meth}(*)                # def join_headers(*)
+            private def #{lock_meth}(*)                # private def join_headers(*)
               return super if @lock.owned?
 
               # small race condition between
@@ -119,7 +119,7 @@ module HTTPX
       class Signal
         def initialize
           @closed = false
-          @pipe_read, @pipe_write = ::IO.pipe
+          @pipe_read, @pipe_write = IO.pipe
         end
 
         def state
@@ -127,7 +127,7 @@ module HTTPX
         end
 
         # noop
-        def log(**); end
+        def log(**, &_); end
 
         def to_io
           @pipe_read.to_io

data/lib/httpx/plugins/upgrade/h2.rb

@@ -22,6 +22,16 @@ module HTTPX
       module ConnectionMethods
         using URIExtensions
 
+        def interests
+          return super unless connecting? && @parser
+
+          connect
+
+          return @io.interests if connecting?
+
+          super
+        end
+
         def upgrade_to_h2
           prev_parser = @parser
 
@@ -30,7 +40,7 @@ module HTTPX
             @inflight -= prev_parser.requests.size
           end
 
-          @parser = Connection::HTTP2.new(@write_buffer, @options)
+          @parser = @options.http2_class.new(@write_buffer, @options)
           set_parser_callbacks(@parser)
           @upgrade_protocol = "h2"
 

data/lib/httpx/plugins/upgrade.rb

@@ -20,6 +20,8 @@ module HTTPX
       end
 
       module OptionsMethods
+        private
+
         def option_upgrade_handlers(value)
           raise TypeError, ":upgrade_handlers must be a Hash" unless value.is_a?(Hash)
 
@@ -65,6 +67,12 @@ module HTTPX
       module ConnectionMethods
         attr_reader :upgrade_protocol, :hijacked
 
+        def initialize(*)
+          super
+
+          @upgrade_protocol = nil
+        end
+
         def hijack_io
           @hijacked = true
 

data/lib/httpx/pool.rb

@@ -2,6 +2,8 @@
 
 require "httpx/selector"
 require "httpx/connection"
+require "httpx/connection/http2"
+require "httpx/connection/http1"
 require "httpx/resolver"
 
 module HTTPX
@@ -51,18 +53,21 @@ module HTTPX
             # this takes precedence over per-origin
             @max_connections_cond.wait(@connection_mtx, @pool_timeout)
 
-            acquire_connection(uri, options) || begin
-              if @connections_counter == @max_connections
-                # if no matching usable connection was found, the pool will make room and drop a closed connection. if none is found,
-                # this means that all of them are persistent or being used, so raise a timeout error.
-                conn = @connections.find { |c| c.state == :closed }
+            if (conn = acquire_connection(uri, options))
+              return conn
+            end
+
+            if @connections_counter == @max_connections
+              # if no matching usable connection was found, the pool will make room and drop a closed connection. if none is found,
+              # this means that all of them are persistent or being used, so raise a timeout error.
+              conn = @connections.find { |c| c.state == :closed }
 
-                raise PoolTimeoutError.new(@pool_timeout,
-                                           "Timed out after #{@pool_timeout} seconds while waiting for a connection") unless conn
+              raise PoolTimeoutError.new(@pool_timeout,
+                                         "Timed out after #{@pool_timeout} seconds while waiting for a connection") unless conn
 
-                drop_connection(conn)
-              end
+              drop_connection(conn)
             end
+
           end
 
           if @origin_counters[uri.origin] == @max_connections_per_origin
@@ -110,7 +115,7 @@ module HTTPX
 
     def checkout_resolver(options)
       resolver_type = options.resolver_class
-      resolver_type = Resolver.resolver_for(resolver_type)
+      resolver_type = Resolver.resolver_for(resolver_type, options)
 
       @resolver_mtx.synchronize do
         resolvers = @resolvers[resolver_type]

data/lib/httpx/request/body.rb

@@ -56,7 +56,7 @@ module HTTPX
           block.call(chunk)
         end
         # TODO: use copy_stream once bug is resolved: https://bugs.ruby-lang.org/issues/21131
-        # ::IO.copy_stream(body, ProcIO.new(block))
+        # IO.copy_stream(body, ProcIO.new(block))
       elsif body.respond_to?(:each)
         body.each(&block)
       else
@@ -127,8 +127,13 @@ module HTTPX
           # @type var body: bodyIO
           Transcoder::Body.encode(body)
         elsif (form = params.delete(:form))
-          # @type var form: Transcoder::urlencoded_input
-          Transcoder::Form.encode(form)
+          if Transcoder::Multipart.multipart?(form)
+            # @type var form: Transcoder::multipart_input
+            Transcoder::Multipart.encode(form)
+          else
+            # @type var form: Transcoder::urlencoded_input
+            Transcoder::Form.encode(form)
+          end
         elsif (json = params.delete(:json))
           # @type var body: _ToJson
           Transcoder::JSON.encode(json)

data/lib/httpx/request.rb

@@ -8,6 +8,7 @@ module HTTPX
   # as well as maintaining the state machine which manages streaming the request onto the wire.
   class Request
     extend Forwardable
+    include Loggable
     include Callbacks
     using URIExtensions
 
@@ -102,13 +103,16 @@ module HTTPX
       raise UnsupportedSchemeError, "#{@uri}: #{@uri.scheme}: unsupported URI scheme" unless ALLOWED_URI_SCHEMES.include?(@uri.scheme)
 
       @state = :idle
-      @response = nil
-      @peer_address = nil
+      @response = @peer_address = @context = @informational_status = nil
       @ping = false
       @persistent = @options.persistent
       @active_timeouts = []
     end
 
+    def complete!(response = @response)
+      emit(:complete, response)
+    end
+
     # whether request has been buffered with a ping
     def ping?
       @ping
@@ -173,17 +177,23 @@ module HTTPX
     def response=(response)
       return unless response
 
-      if response.is_a?(Response) && response.status < 200
-        # deal with informational responses
+      case response
+      when Response
+        if response.status < 200
+          # deal with informational responses
 
-        if response.status == 100 && @headers.key?("expect")
-          @informational_status = response.status
-          return
-        end
+          if response.status == 100 && @headers.key?("expect")
+            @informational_status = response.status
+            return
+          end
+
+          # 103 Early Hints advertises resources in document to browsers.
+          # not very relevant for an HTTP client, discard.
+          return if response.status >= 103
 
-        # 103 Early Hints advertises resources in document to browsers.
-        # not very relevant for an HTTP client, discard.
-        return if response.status >= 103
+        end
+      when ErrorResponse
+        response.error.connection = nil if response.error.respond_to?(:connection=)
       end
 
       @response = response
@@ -293,6 +303,7 @@ module HTTPX
         return if @state == :expect
 
       end
+      log(level: 3) { "#{@state}] -> #{nextstate}" }
       @state = nextstate
       emit(@state, self)
       nil

data/lib/httpx/resolver/entry.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "ipaddr"
+
+module HTTPX
+  module Resolver
+    class Entry < SimpleDelegator
+      attr_reader :address
+
+      def self.convert(address)
+        new(address, rescue_on_convert: true)
+      end
+
+      def initialize(address, expires_in = Float::INFINITY, rescue_on_convert: false)
+        @expires_in = expires_in
+        @address = address.is_a?(IPAddr) ? address : IPAddr.new(address.to_s)
+        super(@address)
+      rescue IPAddr::InvalidAddressError
+        raise unless rescue_on_convert
+
+        @address = address.to_s
+        super(@address)
+      end
+
+      def expired?
+        @expires_in < Utils.now
+      end
+    end
+  end
+end

data/lib/httpx/resolver/https.rb

@@ -59,6 +59,8 @@ module HTTPX
       resolve(connection)
     end
 
+    # This is already indirectly monitored bt the HTTP connection. In order to skip
+    # monitoring, this method returns <tt>true</tt>.
     def closed?
       true
     end
@@ -202,7 +204,7 @@ module HTTPX
           @queries.delete_if { |_, conn| connection == conn }
 
           Resolver.cached_lookup_set(hostname, @family, addresses) if @resolver_options[:cache]
-          catch(:coalesced) { emit_addresses(connection, @family, addresses.map { |addr| addr["data"] }) }
+          catch(:coalesced) { emit_addresses(connection, @family, addresses.map { |a| Resolver::Entry.new(a["data"], a["TTL"]) }) }
         end
       end
       return if @connections.empty?

data/lib/httpx/resolver/multi.rb

@@ -11,12 +11,13 @@ module HTTPX
     attr_reader :resolvers, :options
 
     def initialize(resolver_type, options)
-      @current_selector = nil
-      @current_session = nil
+      @current_selector = @current_session = nil
       @options = options
       @resolver_options = @options.resolver_options
 
-      @resolvers = options.ip_families.map do |ip_family|
+      ip_families = options.ip_families || Resolver.supported_ip_families
+
+      @resolvers = ip_families.map do |ip_family|
         resolver = resolver_type.new(ip_family, options)
         resolver.multi = self
         resolver
@@ -35,6 +36,10 @@ module HTTPX
       @resolvers.each { |r| r.__send__(__method__, s) }
     end
 
+    def log(*args, **kwargs, &blk)
+      @resolvers.each { |r| r.__send__(__method__, *args, **kwargs, &blk) }
+    end
+
     def closed?
       @resolvers.all?(&:closed?)
     end
@@ -64,8 +69,12 @@ module HTTPX
       addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
       return false unless addresses
 
+      ip_families = connection.options.ip_families || Resolver.supported_ip_families
+
       resolved = false
       addresses.group_by(&:family).sort { |(f1, _), (f2, _)| f2 <=> f1 }.each do |family, addrs|
+        next unless ip_families.include?(family)
+
         # try to match the resolver by family. However, there are cases where that's not possible, as when
         # the system does not have IPv6 connectivity, but it does support IPv6 via loopback/link-local.
         resolver = @resolvers.find { |r| r.family == family } || @resolvers.first
@@ -82,7 +91,11 @@ module HTTPX
     end
 
     def lazy_resolve(connection)
+      ip_families = connection.options.ip_families || Resolver.supported_ip_families
+
       @resolvers.each do |resolver|
+        next unless ip_families.include?(resolver.family)
+
         resolver << @current_session.try_clone_connection(connection, @current_selector, resolver.family)
         next if resolver.empty?
 

data/lib/httpx/resolver/native.rb

@@ -42,6 +42,7 @@ module HTTPX
       @read_buffer = "".b
       @write_buffer = Buffer.new(@resolver_options[:packet_size])
       @state = :idle
+      @timer = nil
     end
 
     def close
@@ -104,11 +105,11 @@ module HTTPX
     private
 
     def calculate_interests
-      return :w unless @write_buffer.empty?
+      return if @queries.empty?
 
-      return :r unless @queries.empty?
+      return :r if @write_buffer.empty?
 
-      nil
+      :w
     end
 
     def consume
@@ -153,6 +154,8 @@ module HTTPX
       @timer = @current_selector.after(timeout) do
         next unless @connections.include?(connection)
 
+        @timer = nil
+
         do_retry(h, connection, timeout)
       end
     end
@@ -270,6 +273,8 @@ module HTTPX
     def parse(buffer)
       @timer.cancel
 
+      @timer = nil
+
       code, result = Resolver.decode_dns_answer(buffer)
 
       case code
@@ -370,7 +375,9 @@ module HTTPX
           @timeouts.delete(connection.peer.host)
           @connections.delete(connection)
           Resolver.cached_lookup_set(connection.peer.host, @family, addresses) if @resolver_options[:cache]
-          catch(:coalesced) { emit_addresses(connection, @family, addresses.map { |addr| addr["data"] }) }
+          catch(:coalesced) do
+            emit_addresses(connection, @family, addresses.map { |a| Resolver::Entry.new(a["data"], a["TTL"]) })
+          end
         end
       end
       close_or_resolve
@@ -383,7 +390,8 @@ module HTTPX
 
       raise Error, "no URI to resolve" unless connection
 
-      return unless @write_buffer.empty?
+      # do not buffer query if previous is still in the buffer or awaiting reply/retry
+      return unless @write_buffer.empty? && @timer.nil?
 
       hostname ||= @queries.key(connection)
 
@@ -444,7 +452,7 @@ module HTTPX
       when :tcp
         log { "resolver #{FAMILY_TYPES[@record_type]}: server: tcp://#{ip}:#{port}..." }
         origin = URI("tcp://#{ip}:#{port}")
-        TCP.new(origin, [ip], @options)
+        TCP.new(origin, [Resolver::Entry.new(ip)], @options)
       end
     end
 
@@ -480,6 +488,7 @@ module HTTPX
         @write_buffer.clear
         @read_buffer.clear
       end
+      log(level: 3) { "#{@state} -> #{nextstate}" }
       @state = nextstate
     rescue Errno::ECONNREFUSED,
            Errno::EADDRNOTAVAIL,

data/lib/httpx/resolver/resolver.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "resolv"
-require "ipaddr"
 
 module HTTPX
   # Base class for all internal internet name resolvers. It handles basic blocks
@@ -69,24 +68,29 @@ module HTTPX
     end
 
     def emit_addresses(connection, family, addresses, early_resolve = false)
-      addresses.map! do |address|
-        address.is_a?(IPAddr) ? address : IPAddr.new(address.to_s)
-      end
+      addresses.map! { |address| address.is_a?(Resolver::Entry) ? address : Resolver::Entry.new(address) }
 
       # double emission check, but allow early resolution to work
-      return if !early_resolve && connection.addresses && !addresses.intersect?(connection.addresses)
+      conn_addrs = connection.addresses
+      return if !early_resolve && conn_addrs && (!conn_addrs.empty? && !addresses.intersect?(!conn_addrs))
 
       log do
         "resolver #{FAMILY_TYPES[RECORD_TYPES[family]]}: " \
           "answer #{connection.peer.host}: #{addresses.inspect} (early resolve: #{early_resolve})"
       end
 
-      if !early_resolve && # do not apply resolution delay for non-dns name resolution
-         @current_selector && # just in case...
-         family == Socket::AF_INET && # resolution delay only applies to IPv4
-         !connection.io && # connection already has addresses and initiated/ended handshake
-         connection.options.ip_families.size > 1 && # no need to delay if not supporting dual stack IP
-         addresses.first.to_s != connection.peer.host.to_s # connection URL host is already the IP (early resolve included perhaps?)
+      # do not apply resolution delay for non-dns name resolution
+      if !early_resolve &&
+         # just in case...
+         @current_selector &&
+         # resolution delay only applies to IPv4
+         family == Socket::AF_INET &&
+         # connection already has addresses and initiated/ended handshake
+         !connection.io &&
+         # no need to delay if not supporting dual stack / multi-homed IP
+         (connection.options.ip_families || Resolver.supported_ip_families).size > 1 &&
+         # connection URL host is already the IP (early resolve included perhaps?)
+         addresses.first.to_s != connection.peer.host.to_s
         log { "resolver #{FAMILY_TYPES[RECORD_TYPES[family]]}: applying resolution delay..." }
 
         @current_selector.after(0.05) do

data/lib/httpx/resolver/system.rb

@@ -127,7 +127,7 @@ module HTTPX
       when :open
         return unless @state == :idle
 
-        @pipe_read, @pipe_write = ::IO.pipe
+        @pipe_read, @pipe_write = IO.pipe
       when :closed
         return unless @state == :open
 
@@ -187,7 +187,9 @@ module HTTPX
 
       transition(:open)
 
-      connection.options.ip_families.each do |family|
+      ip_families = connection.options.ip_families || Resolver.supported_ip_families
+
+      ip_families.each do |family|
         @queries << [family, connection]
       end
       async_resolve(connection, hostname, scheme)
@@ -195,7 +197,7 @@ module HTTPX
     end
 
     def async_resolve(connection, hostname, scheme)
-      families = connection.options.ip_families
+      families = connection.options.ip_families || Resolver.supported_ip_families
       log { "resolver: query for #{hostname}" }
       timeouts = @timeouts[connection.peer.host]
       resolve_timeout = timeouts.first

data/lib/httpx/resolver.rb

@@ -1,12 +1,13 @@
 # frozen_string_literal: true
 
+require "socket"
 require "resolv"
-require "ipaddr"
 
 module HTTPX
   module Resolver
     RESOLVE_TIMEOUT = [2, 3].freeze
 
+    require "httpx/resolver/entry"
     require "httpx/resolver/resolver"
     require "httpx/resolver/system"
     require "httpx/resolver/native"
@@ -18,36 +19,54 @@ module HTTPX
 
     @identifier_mutex = Thread::Mutex.new
     @identifier = 1
-    @system_resolver = Resolv::Hosts.new
+    @hosts_resolver = Resolv::Hosts.new
 
     module_function
 
-    def resolver_for(resolver_type)
+    def supported_ip_families
+      @supported_ip_families ||= begin
+        # https://github.com/ruby/resolv/blob/095f1c003f6073730500f02acbdbc55f83d70987/lib/resolv.rb#L408
+        list = Socket.ip_address_list
+        if list.any? { |a| a.ipv6? && !a.ipv6_loopback? && !a.ipv6_linklocal? }
+          [Socket::AF_INET6, Socket::AF_INET]
+        else
+          [Socket::AF_INET]
+        end
+      rescue NotImplementedError
+        [Socket::AF_INET]
+      end.freeze
+    end
+
+    def resolver_for(resolver_type, options)
       case resolver_type
-      when :native then Native
-      when :system then System
-      when :https then HTTPS
-      else
-        return resolver_type if resolver_type.is_a?(Class) && resolver_type < Resolver
+      when Symbol
+        meth = :"resolver_#{resolver_type}_class"
 
-        raise Error, "unsupported resolver type (#{resolver_type})"
+        return options.__send__(meth) if options.respond_to?(meth)
+      when Class
+        return resolver_type if resolver_type < Resolver
       end
+
+      raise Error, "unsupported resolver type (#{resolver_type})"
     end
 
     def nolookup_resolve(hostname)
-      ip_resolve(hostname) || cached_lookup(hostname) || system_resolve(hostname)
+      ip_resolve(hostname) || cached_lookup(hostname) || hosts_resolve(hostname)
     end
 
+    # tries to convert +hostname+ into an IPAddr, returns <tt>nil</tt> otherwise.
     def ip_resolve(hostname)
-      [IPAddr.new(hostname)]
+      [Entry.new(hostname)]
     rescue ArgumentError
     end
 
-    def system_resolve(hostname)
-      ips = @system_resolver.getaddresses(hostname)
+    # matches +hostname+ to entries in the hosts file, returns <tt>nil</nil> if none is
+    # found, or there is no hosts file.
+    def hosts_resolve(hostname)
+      ips = @hosts_resolver.getaddresses(hostname)
       return if ips.empty?
 
-      ips.map { |ip| IPAddr.new(ip) }
+      ips.map { |ip| Entry.new(ip) }
     rescue IOError
     end
 
@@ -59,10 +78,6 @@ module HTTPX
     end
 
     def cached_lookup_set(hostname, family, entries)
-      now = Utils.now
-      entries.each do |entry|
-        entry["TTL"] += now
-      end
       lookup_synchronize do |lookups|
         case family
         when Socket::AF_INET6
@@ -83,6 +98,18 @@ module HTTPX
       end
     end
 
+    def cached_lookup_evict(hostname, ip)
+      ip = ip.to_s
+
+      lookup_synchronize do |lookups|
+        entries = lookups[hostname]
+
+        return unless entries
+
+        lookups.delete_if { |entry| entry["data"] == ip }
+      end
+    end
+
     # do not use directly!
     def lookup(hostname, lookups, ttl)
       return unless lookups.key?(hostname)
@@ -95,7 +122,7 @@ module HTTPX
         if (als = address["alias"])
           lookup(als, lookups, ttl)
         else
-          IPAddr.new(address["data"])
+          Entry.new(address["data"], address["TTL"])
         end
       end.compact
 
@@ -129,19 +156,20 @@ module HTTPX
 
       addresses = []
 
+      now = Utils.now
       message.each_answer do |question, _, value|
         case value
         when Resolv::DNS::Resource::IN::CNAME
           addresses << {
             "name" => question.to_s,
-            "TTL" => value.ttl,
+            "TTL" => (now + value.ttl),
             "alias" => value.name.to_s,
           }
         when Resolv::DNS::Resource::IN::A,
              Resolv::DNS::Resource::IN::AAAA
           addresses << {
             "name" => question.to_s,
-            "TTL" => value.ttl,
+            "TTL" => (now + value.ttl),
             "data" => value.address.to_s,
           }
         end

data/lib/httpx/response/body.rb

@@ -136,7 +136,7 @@ module HTTPX
       if dest.respond_to?(:path) && @buffer.respond_to?(:path)
         FileUtils.mv(@buffer.path, dest.path)
       else
-        ::IO.copy_stream(@buffer, dest)
+        IO.copy_stream(@buffer, dest)
       end
     end