httpx 1.4.0 → 1.4.4

data/lib/httpx/request.rb

@@ -11,6 +11,8 @@ module HTTPX
     include Callbacks
     using URIExtensions
 
+    ALLOWED_URI_SCHEMES = %w[https http].freeze
+
     # default value used for "user-agent" header, when not overridden.
     USER_AGENT = "httpx.rb/#{VERSION}".freeze # rubocop:disable Style/RedundantFreeze
 
@@ -43,9 +45,14 @@ module HTTPX
 
     attr_writer :persistent
 
+    attr_reader :active_timeouts
+
     # will be +true+ when request body has been completely flushed.
     def_delegator :@body, :empty?
 
+    # closes the body
+    def_delegator :@body, :close
+
     # initializes the instance with the given +verb+ (an upppercase String, ex. 'GEt'),
     # an absolute or relative +uri+ (either as String or URI::HTTP object), the
     # request +options+ (instance of HTTPX::Options) and an optional Hash of +params+.
@@ -92,23 +99,37 @@ module HTTPX
         @uri = origin.merge("#{base_path}#{@uri}")
       end
 
+      raise UnsupportedSchemeError, "#{@uri}: #{@uri.scheme}: unsupported URI scheme" unless ALLOWED_URI_SCHEMES.include?(@uri.scheme)
+
       @state = :idle
       @response = nil
       @peer_address = nil
+      @ping = false
       @persistent = @options.persistent
+      @active_timeouts = []
+    end
+
+    # whether request has been buffered with a ping
+    def ping?
+      @ping
+    end
+
+    # marks the request as having been buffered with a ping
+    def ping!
+      @ping = true
     end
 
-    # the read timeout defined for this requet.
+    # the read timeout defined for this request.
     def read_timeout
       @options.timeout[:read_timeout]
     end
 
-    # the write timeout defined for this requet.
+    # the write timeout defined for this request.
     def write_timeout
       @options.timeout[:write_timeout]
     end
 
-    # the request timeout defined for this requet.
+    # the request timeout defined for this request.
     def request_timeout
       @options.timeout[:request_timeout]
     end
@@ -239,10 +260,13 @@ module HTTPX
       case nextstate
       when :idle
         @body.rewind
+        @ping = false
         @response = nil
         @drainer = nil
+        @active_timeouts.clear
       when :headers
         return unless @state == :idle
+
       when :body
         return unless @state == :headers ||
                       @state == :expect
@@ -263,6 +287,7 @@ module HTTPX
         return unless @state == :body
       when :done
         return if @state == :expect
+
       end
       @state = nextstate
       emit(@state, self)
@@ -273,6 +298,15 @@ module HTTPX
     def expects?
       @headers["expect"] == "100-continue" && @informational_status == 100 && !@response
     end
+
+    def set_timeout_callback(event, &callback)
+      clb = once(event, &callback)
+
+      # reset timeout callbacks when requests get rerouted to a different connection
+      once(:idle) do
+        callbacks(event).delete(clb)
+      end
+    end
   end
 end
 

data/lib/httpx/resolver/https.rb

@@ -82,7 +82,9 @@ module HTTPX
 
       if hostname.nil?
         hostname = connection.peer.host
-        log { "resolver: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}" } if connection.peer.non_ascii_hostname
+        log do
+          "resolver #{FAMILY_TYPES[@record_type]}: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}"
+        end if connection.peer.non_ascii_hostname
 
         hostname = @resolver.generate_candidates(hostname).each do |name|
           @queries[name.to_s] = connection
@@ -90,7 +92,7 @@ module HTTPX
       else
         @queries[hostname] = connection
       end
-      log { "resolver: query #{FAMILY_TYPES[RECORD_TYPES[@family]]} for #{hostname}" }
+      log { "resolver #{FAMILY_TYPES[@record_type]}: query for #{hostname}" }
 
       begin
         request = build_request(hostname)

data/lib/httpx/resolver/native.rb

@@ -35,6 +35,7 @@ module HTTPX
       @_timeouts = Array(@resolver_options[:timeouts])
       @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
       @connections = []
+      @name = nil
       @queries = {}
       @read_buffer = "".b
       @write_buffer = Buffer.new(@resolver_options[:packet_size])
@@ -58,19 +59,6 @@ module HTTPX
       when :open
         consume
       end
-      nil
-    rescue Errno::EHOSTUNREACH => e
-      @ns_index += 1
-      nameserver = @nameserver
-      if nameserver && @ns_index < nameserver.size
-        log { "resolver: failed resolving on nameserver #{@nameserver[@ns_index - 1]} (#{e.message})" }
-        transition(:idle)
-        @timeouts.clear
-      else
-        handle_error(e)
-      end
-    rescue NativeResolveError => e
-      handle_error(e)
     end
 
     def interests
@@ -105,9 +93,7 @@ module HTTPX
       @timeouts.values_at(*hosts).reject(&:empty?).map(&:first).min
     end
 
-    def handle_socket_timeout(interval)
-      do_retry(interval)
-    end
+    def handle_socket_timeout(interval); end
 
     private
 
@@ -120,54 +106,94 @@ module HTTPX
     end
 
     def consume
-      dread if calculate_interests == :r
-      do_retry
-      dwrite if calculate_interests == :w
+      loop do
+        dread if calculate_interests == :r
+
+        break unless calculate_interests == :w
+
+        # do_retry
+        dwrite
+
+        break unless calculate_interests == :r
+      end
+    rescue Errno::EHOSTUNREACH => e
+      @ns_index += 1
+      nameserver = @nameserver
+      if nameserver && @ns_index < nameserver.size
+        log do
+          "resolver #{FAMILY_TYPES[@record_type]}: " \
+            "failed resolving on nameserver #{@nameserver[@ns_index - 1]} (#{e.message})"
+        end
+        transition(:idle)
+        @timeouts.clear
+        retry
+      else
+        handle_error(e)
+        emit(:close, self)
+      end
+    rescue NativeResolveError => e
+      handle_error(e)
+      close_or_resolve
+      retry unless closed?
     end
 
-    def do_retry(loop_time = nil)
-      return if @queries.empty? || !@start_timeout
+    def schedule_retry
+      h = @name
 
-      loop_time ||= Utils.elapsed_time(@start_timeout)
+      return unless h
 
-      query = @queries.first
+      connection = @queries[h]
 
-      return unless query
+      timeouts = @timeouts[h]
+      timeout = timeouts.shift
 
-      h, connection = query
-      host = connection.peer.host
-      timeout = (@timeouts[host][0] -= loop_time)
+      @timer = @current_selector.after(timeout) do
+        next unless @connections.include?(connection)
 
-      return unless timeout <= 0
+        do_retry(h, connection, timeout)
+      end
+    end
 
-      @timeouts[host].shift
+    def do_retry(h, connection, interval)
+      timeouts = @timeouts[h]
 
-      if !@timeouts[host].empty?
-        log { "resolver: timeout after #{timeout}s, retry(#{@timeouts[host].first}) #{host}..." }
+      if !timeouts.empty?
+        log do
+          "resolver #{FAMILY_TYPES[@record_type]}: timeout after #{interval}s, retry (with #{timeouts.first}s) #{h}..."
+        end
         # must downgrade to tcp AND retry on same host as last
         downgrade_socket
         resolve(connection, h)
       elsif @ns_index + 1 < @nameserver.size
         # try on the next nameserver
         @ns_index += 1
-        log { "resolver: failed resolving #{host} on nameserver #{@nameserver[@ns_index - 1]} (timeout error)" }
+        log do
+          "resolver #{FAMILY_TYPES[@record_type]}: failed resolving #{h} on nameserver #{@nameserver[@ns_index - 1]} (timeout error)"
+        end
         transition(:idle)
         @timeouts.clear
         resolve(connection, h)
       else
 
-        @timeouts.delete(host)
+        @timeouts.delete(h)
         reset_hostname(h, reset_candidates: false)
 
-        return unless @queries.empty?
+        unless @queries.empty?
+          resolve(connection)
+          return
+        end
 
         @connections.delete(connection)
+
+        host = connection.peer.host
+
         # This loop_time passed to the exception is bogus. Ideally we would pass the total
         # resolve timeout, including from the previous retries.
-        ex = ResolveTimeoutError.new(loop_time, "Timed out while resolving #{connection.peer.host}")
+        ex = ResolveTimeoutError.new(interval, "Timed out while resolving #{host}")
         ex.set_backtrace(ex ? ex.backtrace : caller)
         emit_resolve_error(connection, host, ex)
-        emit(:close, self)
+
+        close_or_resolve
       end
     end
 
@@ -216,7 +242,7 @@ module HTTPX
           parse(@read_buffer)
         end
 
-        return if @state == :closed
+        return if @state == :closed || !@write_buffer.empty?
       end
     end
 
@@ -234,11 +260,15 @@ module HTTPX
 
         return unless siz.positive?
 
+        schedule_retry if @write_buffer.empty?
+
         return if @state == :closed
       end
     end
 
     def parse(buffer)
+      @timer.cancel
+
       code, result = Resolver.decode_dns_answer(buffer)
 
       case code
@@ -249,15 +279,17 @@ module HTTPX
         hostname, connection = @queries.first
         reset_hostname(hostname, reset_candidates: false)
 
-        unless @queries.value?(connection)
+        other_candidate, _ = @queries.find { |_, conn| conn == connection }
+
+        if other_candidate
+          resolve(connection, other_candidate)
+        else
           @connections.delete(connection)
           ex = NativeResolveError.new(connection, connection.peer.host, "name or service not known")
           ex.set_backtrace(ex ? ex.backtrace : caller)
           emit_resolve_error(connection, connection.peer.host, ex)
-          emit(:close, self)
+          close_or_resolve
         end
-
-        resolve
       when :message_truncated
         # TODO: what to do if it's already tcp??
         return if @socket_type == :tcp
@@ -312,8 +344,10 @@ module HTTPX
           connection = @queries.delete(name)
         end
 
-        if address.key?("alias") # CNAME
-          hostname_alias = address["alias"]
+        alias_addresses, addresses = addresses.partition { |addr| addr.key?("alias") }
+
+        if addresses.empty? && !alias_addresses.empty? # CNAME
+          hostname_alias = alias_addresses.first["alias"]
           # clean up intermediate queries
           @timeouts.delete(name) unless connection.peer.host == name
 
@@ -326,7 +360,7 @@ module HTTPX
               transition(:idle)
               transition(:open)
             end
-            log { "resolver: ALIAS #{hostname_alias} for #{name}" }
+            log { "resolver #{FAMILY_TYPES[@record_type]}: ALIAS #{hostname_alias} for #{name}" }
             resolve(connection, hostname_alias)
             return
           end
@@ -338,12 +372,14 @@ module HTTPX
           catch(:coalesced) { emit_addresses(connection, @family, addresses.map { |addr| addr["data"] }) }
         end
       end
-      return emit(:close, self) if @connections.empty?
-
-      resolve
+      close_or_resolve
     end
 
-    def resolve(connection = @connections.first, hostname = nil)
+    def resolve(connection = nil, hostname = nil)
+      @connections.shift until @connections.empty? || @connections.first.state != :closed
+
+      connection ||= @connections.find { |c| !@queries.value?(c) }
+
       raise Error, "no URI to resolve" unless connection
 
       return unless @write_buffer.empty?
@@ -352,7 +388,10 @@ module HTTPX
 
       if hostname.nil?
         hostname = connection.peer.host
-        log { "resolver: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}" } if connection.peer.non_ascii_hostname
+        log do
+          "resolver #{FAMILY_TYPES[@record_type]}: " \
+            "resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}"
+        end if connection.peer.non_ascii_hostname
 
         hostname = generate_candidates(hostname).each do |name|
           @queries[name] = connection
@@ -360,14 +399,17 @@ module HTTPX
       else
         @queries[hostname] = connection
       end
-      log { "resolver: query #{@record_type.name.split("::").last} for #{hostname}" }
+
+      @name = hostname
+
+      log { "resolver #{FAMILY_TYPES[@record_type]}: query for #{hostname}" }
       begin
         @write_buffer << encode_dns_query(hostname)
       rescue Resolv::DNS::EncodeError => e
         reset_hostname(hostname, connection: connection)
         @connections.delete(connection)
         emit_resolve_error(connection, hostname, e)
-        emit(:close, self) if @connections.empty?
+        close_or_resolve
       end
     end
 
@@ -397,10 +439,10 @@ module HTTPX
 
       case @socket_type
       when :udp
-        log { "resolver: server: udp://#{ip}:#{port}..." }
+        log { "resolver #{FAMILY_TYPES[@record_type]}: server: udp://#{ip}:#{port}..." }
         UDP.new(ip, port, @options)
       when :tcp
-        log { "resolver: server: tcp://#{ip}:#{port}..." }
+        log { "resolver #{FAMILY_TYPES[@record_type]}: server: tcp://#{ip}:#{port}..." }
         origin = URI("tcp://#{ip}:#{port}")
         TCP.new(origin, [ip], @options)
       end
@@ -448,6 +490,7 @@ module HTTPX
       # these errors may happen during TCP handshake
       # treat them as resolve errors.
       handle_error(e)
+      emit(:close, self)
     end
 
     def handle_error(error)
@@ -462,13 +505,15 @@ module HTTPX
           @connections.delete(connection)
           emit_resolve_error(connection, host, error)
         end
+
+        while (connection = @connections.shift)
+          emit_resolve_error(connection, host, error)
+        end
       end
-      emit(:close, self) if @connections.empty?
     end
 
     def reset_hostname(hostname, connection: @queries.delete(hostname), reset_candidates: true)
       @timeouts.delete(hostname)
-      @timeouts.delete(hostname)
 
       return unless connection && reset_candidates
 
@@ -478,5 +523,16 @@ module HTTPX
       # reset timeouts
       @timeouts.delete_if { |h, _| candidates.include?(h) }
     end
+
+    def close_or_resolve
+      # drop already closed connections
+      @connections.shift until @connections.empty? || @connections.first.state != :closed
+
+      if (@connections - @queries.values).empty?
+        emit(:close, self)
+      else
+        resolve
+      end
+    end
   end
 end

data/lib/httpx/resolver/resolver.rb

@@ -72,17 +72,22 @@ module HTTPX
       # double emission check, but allow early resolution to work
       return if !early_resolve && connection.addresses && !addresses.intersect?(connection.addresses)
 
-      log { "resolver: answer #{FAMILY_TYPES[RECORD_TYPES[family]]} #{connection.peer.host}: #{addresses.inspect}" }
-      if @current_selector && # if triggered by early resolve, session may not be here yet
-         !connection.io &&
-         connection.options.ip_families.size > 1 &&
-         family == Socket::AF_INET &&
-         addresses.first.to_s != connection.peer.host.to_s
-        log { "resolver: A response, applying resolution delay..." }
+      log do
+        "resolver #{FAMILY_TYPES[RECORD_TYPES[family]]}: " \
+          "answer #{connection.peer.host}: #{addresses.inspect} (early resolve: #{early_resolve})"
+      end
+
+      if !early_resolve && # do not apply resolution delay for non-dns name resolution
+         @current_selector && # just in case...
+         family == Socket::AF_INET && # resolution delay only applies to IPv4
+         !connection.io && # connection already has addresses and initiated/ended handshake
+         connection.options.ip_families.size > 1 && # no need to delay if not supporting dual stack IP
+         addresses.first.to_s != connection.peer.host.to_s # connection URL host is already the IP (early resolve included perhaps?)
+        log { "resolver #{FAMILY_TYPES[RECORD_TYPES[family]]}: applying resolution delay..." }
+
         @current_selector.after(0.05) do
-          unless connection.state == :closed ||
-                 # double emission check
-                 (connection.addresses && addresses.intersect?(connection.addresses))
+          # double emission check
+          unless connection.addresses && addresses.intersect?(connection.addresses)
             emit_resolved_connection(connection, addresses, early_resolve)
           end
         end
@@ -97,6 +102,8 @@ module HTTPX
       begin
         connection.addresses = addresses
 
+        return if connection.state == :closed
+
         emit(:resolve, connection)
       rescue StandardError => e
         if early_resolve
@@ -146,7 +153,7 @@ module HTTPX
     end
 
     def emit_connection_error(connection, error)
-      return connection.emit(:connect_error, error) if connection.connecting? && connection.callbacks_for?(:connect_error)
+      return connection.handle_connect_error(error) if connection.connecting?
 
       connection.emit(:error, error)
     end

data/lib/httpx/resolver/system.rb

@@ -1,12 +1,10 @@
 # frozen_string_literal: true
 
-require "forwardable"
 require "resolv"
 
 module HTTPX
   class Resolver::System < Resolver::Resolver
     using URIExtensions
-    extend Forwardable
 
     RESOLV_ERRORS = [Resolv::ResolvError,
                      Resolv::DNS::Requester::RequestError,
@@ -24,8 +22,6 @@ module HTTPX
 
     attr_reader :state
 
-    def_delegator :@connections, :empty?
-
     def initialize(options)
       super(nil, options)
       @resolver_options = @options.resolver_options
@@ -162,7 +158,9 @@ module HTTPX
 
       hostname = connection.peer.host
       scheme = connection.origin.scheme
-      log { "resolver: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}" } if connection.peer.non_ascii_hostname
+      log do
+        "resolver: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}"
+      end if connection.peer.non_ascii_hostname
 
       transition(:open)
 

data/lib/httpx/response.rb

@@ -52,9 +52,6 @@ module HTTPX
     # copies the response body to a different location.
     def_delegator :@body, :copy_to
 
-    # closes the body.
-    def_delegator :@body, :close
-
     # the corresponding request uri.
     def_delegator :@request, :uri
 
@@ -74,6 +71,12 @@ module HTTPX
       @content_type = nil
     end
 
+    # closes the respective +@request+ and +@body+.
+    def close
+      @request.close
+      @body.close
+    end
+
     # merges headers defined in +h+ into the response headers.
     def merge_headers(h)
       @headers = @headers.merge(h)
@@ -264,7 +267,7 @@ module HTTPX
 
     # closes the error resources.
     def close
-      @response.close if @response && @response.respond_to?(:close)
+      @response.close if @response
     end
 
     # always true for error responses.
@@ -279,6 +282,8 @@ module HTTPX
 
     # buffers lost chunks to error response
     def <<(data)
+      return unless @response
+
       @response << data
     end
   end

data/lib/httpx/selector.rb

@@ -19,6 +19,7 @@ module HTTPX
     def initialize
       @timers = Timers.new
       @selectables = []
+      @is_timer_interval = false
     end
 
     def each(&blk)
@@ -43,7 +44,11 @@ module HTTPX
     rescue StandardError => e
       emit_error(e)
     rescue Exception # rubocop:disable Lint/RescueException
-      each_connection(&:force_reset)
+      each_connection do |conn|
+        conn.force_reset
+        conn.disconnect
+      end
+
       raise
     end
 
@@ -92,10 +97,6 @@ module HTTPX
       end
     end
 
-    def empty?
-      @selectables.empty?
-    end
-
     # deregisters +io+ from selectables.
     def deregister(io)
       @selectables.delete(io)
@@ -129,24 +130,22 @@ module HTTPX
       # first, we group IOs based on interest type. On call to #interests however,
       # things might already happen, and new IOs might be registered, so we might
       # have to start all over again. We do this until we group all selectables
-      begin
-        @selectables.delete_if do |io|
-          interests = io.interests
+      @selectables.delete_if do |io|
+        interests = io.interests
 
-          (r ||= []) << io if READABLE.include?(interests)
-          (w ||= []) << io if WRITABLE.include?(interests)
+        (r ||= []) << io if READABLE.include?(interests)
+        (w ||= []) << io if WRITABLE.include?(interests)
 
-          io.state == :closed
-        end
+        io.state == :closed
+      end
 
-        # TODO: what to do if there are no selectables?
+      # TODO: what to do if there are no selectables?
 
-        readers, writers = IO.select(r, w, nil, interval)
+      readers, writers = IO.select(r, w, nil, interval)
 
-        if readers.nil? && writers.nil? && interval
-          [*r, *w].each { |io| io.handle_socket_timeout(interval) }
-          return
-        end
+      if readers.nil? && writers.nil? && interval
+        [*r, *w].each { |io| io.handle_socket_timeout(interval) }
+        return
       end
 
       if writers
@@ -178,7 +177,7 @@ module HTTPX
       end
 
       unless result || interval.nil?
-        io.handle_socket_timeout(interval)
+        io.handle_socket_timeout(interval) unless @is_timer_interval
         return
       end
       # raise TimeoutError.new(interval, "timed out while waiting on select")
@@ -190,10 +189,21 @@ module HTTPX
     end
 
     def next_timeout
-      [
-        @timers.wait_interval,
-        @selectables.filter_map(&:timeout).min,
-      ].compact.min
+      @is_timer_interval = false
+
+      timer_interval = @timers.wait_interval
+
+      connection_interval = @selectables.filter_map(&:timeout).min
+
+      return connection_interval unless timer_interval
+
+      if connection_interval.nil? || timer_interval <= connection_interval
+        @is_timer_interval = true
+
+        return timer_interval
+      end
+
+      connection_interval
     end
 
     def emit_error(e)