httpx 1.4.3 → 1.5.0

data/lib/httpx/request.rb

@@ -104,21 +104,32 @@ module HTTPX
       @state = :idle
       @response = nil
       @peer_address = nil
+      @ping = false
       @persistent = @options.persistent
       @active_timeouts = []
     end
 
-    # the read timeout defined for this requet.
+    # whether request has been buffered with a ping
+    def ping?
+      @ping
+    end
+
+    # marks the request as having been buffered with a ping
+    def ping!
+      @ping = true
+    end
+
+    # the read timeout defined for this request.
     def read_timeout
       @options.timeout[:read_timeout]
     end
 
-    # the write timeout defined for this requet.
+    # the write timeout defined for this request.
     def write_timeout
       @options.timeout[:write_timeout]
     end
 
-    # the request timeout defined for this requet.
+    # the request timeout defined for this request.
     def request_timeout
       @options.timeout[:request_timeout]
     end
@@ -144,6 +155,10 @@ module HTTPX
       :w
     end
 
+    def can_buffer?
+      @state != :done
+    end
+
     # merges +h+ into the instance of HTTPX::Headers of the request.
     def merge_headers(h)
       @headers = @headers.merge(h)
@@ -211,7 +226,7 @@ module HTTPX
       return @query if defined?(@query)
 
       query = []
-      if (q = @query_params)
+      if (q = @query_params) && !q.empty?
         query << Transcoder::Form.encode(q)
       end
       query << @uri.query if @uri.query
@@ -236,7 +251,7 @@ module HTTPX
 
     # :nocov:
     def inspect
-      "#<HTTPX::Request:#{object_id} " \
+      "#<#{self.class}:#{object_id} " \
         "#{@verb} " \
         "#{uri} " \
         "@headers=#{@headers} " \
@@ -249,6 +264,7 @@ module HTTPX
       case nextstate
       when :idle
         @body.rewind
+        @ping = false
         @response = nil
         @drainer = nil
         @active_timeouts.clear

data/lib/httpx/resolver/https.rb

@@ -2,11 +2,14 @@
 
 require "resolv"
 require "uri"
-require "cgi"
 require "forwardable"
 require "httpx/base64"
 
 module HTTPX
+  # Implementation of a DoH name resolver (https://www.youtube.com/watch?v=unMXvnY2FNM).
+  # It wraps an HTTPX::Connection object which integrates with the main session in the
+  # same manner as other performed HTTP requests.
+  #
   class Resolver::HTTPS < Resolver::Resolver
     extend Forwardable
     using URIExtensions
@@ -27,14 +30,13 @@ module HTTPX
       use_get: false,
     }.freeze
 
-    def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close, :terminate, :inflight?
+    def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close, :terminate, :inflight?, :handle_socket_timeout
 
     def initialize(_, options)
       super
       @resolver_options = DEFAULTS.merge(@options.resolver_options)
       @queries = {}
       @requests = {}
-      @connections = []
       @uri = URI(@resolver_options[:uri])
       @uri_addresses = nil
       @resolver = Resolv::DNS.new
@@ -75,7 +77,11 @@ module HTTPX
 
     private
 
-    def resolve(connection = @connections.first, hostname = nil)
+    def resolve(connection = nil, hostname = nil)
+      @connections.shift until @connections.empty? || @connections.first.state != :closed
+
+      connection ||= @connections.first
+
       return unless connection
 
       hostname ||= @queries.key(connection)

data/lib/httpx/resolver/native.rb

@@ -4,6 +4,9 @@ require "forwardable"
 require "resolv"
 
 module HTTPX
+  # Implements a pure ruby name resolver, which abides by the Selectable API.
+  # It delegates DNS payload encoding/decoding to the +resolv+ stlid gem.
+  #
   class Resolver::Native < Resolver::Resolver
     extend Forwardable
     using URIExtensions
@@ -34,7 +37,6 @@ module HTTPX
       @search = Array(@resolver_options[:search]).map { |srch| srch.scan(/[^.]+/) }
       @_timeouts = Array(@resolver_options[:timeouts])
       @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
-      @connections = []
       @name = nil
       @queries = {}
       @read_buffer = "".b
@@ -46,6 +48,10 @@ module HTTPX
       transition(:closed)
     end
 
+    def terminate
+      emit(:close, self)
+    end
+
     def closed?
       @state == :closed
     end
@@ -120,10 +126,7 @@ module HTTPX
       @ns_index += 1
       nameserver = @nameserver
       if nameserver && @ns_index < nameserver.size
-        log do
-          "resolver #{FAMILY_TYPES[@record_type]}: " \
-            "failed resolving on nameserver #{@nameserver[@ns_index - 1]} (#{e.message})"
-        end
+        log { "resolver #{FAMILY_TYPES[@record_type]}: failed resolving on nameserver #{@nameserver[@ns_index - 1]} (#{e.message})" }
         transition(:idle)
         @timeouts.clear
         retry
@@ -158,9 +161,7 @@ module HTTPX
       timeouts = @timeouts[h]
 
       if !timeouts.empty?
-        log do
-          "resolver #{FAMILY_TYPES[@record_type]}: timeout after #{interval}s, retry (with #{timeouts.first}s) #{h}..."
-        end
+        log { "resolver #{FAMILY_TYPES[@record_type]}: timeout after #{interval}s, retry (with #{timeouts.first}s) #{h}..." }
         # must downgrade to tcp AND retry on same host as last
         downgrade_socket
         resolve(connection, h)
@@ -388,10 +389,9 @@ module HTTPX
 
       if hostname.nil?
         hostname = connection.peer.host
-        log do
-          "resolver #{FAMILY_TYPES[@record_type]}: " \
-            "resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}"
-        end if connection.peer.non_ascii_hostname
+        if connection.peer.non_ascii_hostname
+          log { "resolver #{FAMILY_TYPES[@record_type]}: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}" }
+        end
 
         hostname = generate_candidates(hostname).each do |name|
           @queries[name] = connection
@@ -507,7 +507,7 @@ module HTTPX
         end
 
         while (connection = @connections.shift)
-          emit_resolve_error(connection, host, error)
+          emit_resolve_error(connection, connection.peer.host, error)
         end
       end
     end

data/lib/httpx/resolver/resolver.rb

@@ -4,6 +4,9 @@ require "resolv"
 require "ipaddr"
 
 module HTTPX
+  # Base class for all internal internet name resolvers. It handles basic blocks
+  # from the Selectable API.
+  #
   class Resolver::Resolver
     include Callbacks
     include Loggable
@@ -36,6 +39,7 @@ module HTTPX
       @family = family
       @record_type = RECORD_TYPES[family]
       @options = options
+      @connections = []
 
       set_resolver_callbacks
     end

data/lib/httpx/resolver/system.rb

@@ -3,6 +3,15 @@
 require "resolv"
 
 module HTTPX
+  # Implementation of a synchronous name resolver which relies on the system resolver,
+  # which is lib'c getaddrinfo function (abstracted in ruby via Addrinfo.getaddrinfo).
+  #
+  # Its main advantage is relying on the reference implementation for name resolution
+  # across most/all OSs which deploy ruby (it's what TCPSocket also uses), its main
+  # disadvantage is the inability to set timeouts / check socket for readiness events,
+  # hence why it relies on using the Timeout module, which poses a lot of problems for
+  # the selector loop, specially when network is unstable.
+  #
   class Resolver::System < Resolver::Resolver
     using URIExtensions
 
@@ -23,14 +32,13 @@ module HTTPX
     attr_reader :state
 
     def initialize(options)
-      super(nil, options)
+      super(0, options)
       @resolver_options = @options.resolver_options
       resolv_options = @resolver_options.dup
       timeouts = resolv_options.delete(:timeouts) || Resolver::RESOLVE_TIMEOUT
       @_timeouts = Array(timeouts)
       @timeouts = Hash.new { |tims, host| tims[host] = @_timeouts.dup }
       resolv_options.delete(:cache)
-      @connections = []
       @queries = []
       @ips = []
       @pipe_mutex = Thread::Mutex.new
@@ -100,7 +108,14 @@ module HTTPX
     def handle_socket_timeout(interval)
       error = HTTPX::ResolveTimeoutError.new(interval, "timed out while waiting on select")
       error.set_backtrace(caller)
-      on_error(error)
+      @queries.each do |host, connection|
+        @connections.delete(connection)
+        emit_resolve_error(connection, host, error)
+      end
+
+      while (connection = @connections.shift)
+        emit_resolve_error(connection, connection.peer.host, error)
+      end
     end
 
     private
@@ -131,19 +146,22 @@ module HTTPX
         case event
         when DONE
           *pair, addrs = @pipe_mutex.synchronize { @ips.pop }
-          @queries.delete(pair)
-          _, connection = pair
-          @connections.delete(connection)
+          if pair
+            @queries.delete(pair)
+            family, connection = pair
+            @connections.delete(connection)
 
-          family, connection = pair
-          catch(:coalesced) { emit_addresses(connection, family, addrs) }
+            catch(:coalesced) { emit_addresses(connection, family, addrs) }
+          end
         when ERROR
           *pair, error = @pipe_mutex.synchronize { @ips.pop }
-          @queries.delete(pair)
-          @connections.delete(connection)
+          if pair && error
+            @queries.delete(pair)
+            @connections.delete(connection)
 
-          _, connection = pair
-          emit_resolve_error(connection, connection.peer.host, error)
+            _, connection = pair
+            emit_resolve_error(connection, connection.peer.host, error)
+          end
         end
       end
 
@@ -152,11 +170,16 @@ module HTTPX
       resolve
     end
 
-    def resolve(connection = @connections.first)
+    def resolve(connection = nil, hostname = nil)
+      @connections.shift until @connections.empty? || @connections.first.state != :closed
+
+      connection ||= @connections.first
+
       raise Error, "no URI to resolve" unless connection
+
       return unless @queries.empty?
 
-      hostname = connection.peer.host
+      hostname ||= connection.peer.host
       scheme = connection.origin.scheme
       log do
         "resolver: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}"

data/lib/httpx/resolver.rb

@@ -92,8 +92,8 @@ module HTTPX
       end
 
       ips = entries.flat_map do |address|
-        if address.key?("alias")
-          lookup(address["alias"], lookups, ttl)
+        if (als = address["alias"])
+          lookup(als, lookups, ttl)
         else
           IPAddr.new(address["data"])
         end

data/lib/httpx/response/body.rb

@@ -11,6 +11,9 @@ module HTTPX
     # Array of encodings contained in the response "content-encoding" header.
     attr_reader :encodings
 
+    attr_reader :buffer
+    protected :buffer
+
     # initialized with the corresponding HTTPX::Response +response+ and HTTPX::Options +options+.
     def initialize(response, options)
       @response = response
@@ -148,18 +151,17 @@ module HTTPX
     end
 
     def ==(other)
-      object_id == other.object_id || begin
-        if other.respond_to?(:read)
-          _with_same_buffer_pos { FileUtils.compare_stream(@buffer, other) }
-        else
-          to_s == other.to_s
-        end
-      end
+      super || case other
+               when Response::Body
+                 @buffer == other.buffer
+               else
+                 @buffer = other
+               end
     end
 
     # :nocov:
     def inspect
-      "#<HTTPX::Response::Body:#{object_id} " \
+      "#<#{self.class}:#{object_id} " \
         "@state=#{@state} " \
         "@length=#{@length}>"
     end
@@ -226,19 +228,6 @@ module HTTPX
       @state = nextstate
     end
 
-    def _with_same_buffer_pos # :nodoc:
-      return yield unless @buffer && @buffer.respond_to?(:pos)
-
-      # @type ivar @buffer: StringIO | Tempfile
-      current_pos = @buffer.pos
-      @buffer.rewind
-      begin
-        yield
-      ensure
-        @buffer.pos = current_pos
-      end
-    end
-
     class << self
       def initialize_inflater_by_encoding(encoding, response, **kwargs) # :nodoc:
         case encoding

data/lib/httpx/response/buffer.rb

@@ -7,6 +7,9 @@ require "tempfile"
 module HTTPX
   # wraps and delegates to an internal buffer, which can be a StringIO or a Tempfile.
   class Response::Buffer < SimpleDelegator
+    attr_reader :buffer
+    protected :buffer
+
     # initializes buffer with the +threshold_size+ over which the payload gets buffer to a tempfile,
     # the initial +bytesize+, and the +encoding+.
     def initialize(threshold_size:, bytesize: 0, encoding: Encoding::BINARY)
@@ -20,7 +23,14 @@ module HTTPX
     def initialize_dup(other)
       super
 
-      @buffer = other.instance_variable_get(:@buffer).dup
+      # create new descriptor in READ-ONLY mode
+      @buffer =
+        case other.buffer
+        when StringIO
+          StringIO.new(other.buffer.string, mode: File::RDONLY)
+        else
+          other.buffer.class.new(other.buffer.path, encoding: Encoding::BINARY, mode: File::RDONLY)
+        end
     end
 
     # size in bytes of the buffered content.
@@ -46,7 +56,7 @@ module HTTPX
         end
       when Tempfile
         rewind
-        content = _with_same_buffer_pos { @buffer.read }
+        content = @buffer.read
         begin
           content.force_encoding(@encoding)
         rescue ArgumentError # ex: unknown encoding name - utf
@@ -61,6 +71,30 @@ module HTTPX
       @buffer.unlink if @buffer.respond_to?(:unlink)
     end
 
+    def ==(other)
+      super || begin
+        return false unless other.is_a?(Response::Buffer)
+
+        if @buffer.nil?
+          other.buffer.nil?
+        elsif @buffer.respond_to?(:read) &&
+              other.respond_to?(:read)
+          buffer_pos = @buffer.pos
+          other_pos = other.buffer.pos
+          @buffer.rewind
+          other.buffer.rewind
+          begin
+            FileUtils.compare_stream(@buffer, other.buffer)
+          ensure
+            @buffer.pos = buffer_pos
+            other.buffer.pos = other_pos
+          end
+        else
+          to_s == other.to_s
+        end
+      end
+    end
+
     private
 
     # initializes the buffer into a StringIO, or turns it into a Tempfile when the threshold
@@ -82,15 +116,5 @@ module HTTPX
 
       __setobj__(@buffer)
     end
-
-    def _with_same_buffer_pos # :nodoc:
-      current_pos = @buffer.pos
-      @buffer.rewind
-      begin
-        yield
-      ensure
-        @buffer.pos = current_pos
-      end
-    end
   end
 end

data/lib/httpx/response.rb

@@ -71,6 +71,14 @@ module HTTPX
       @content_type = nil
     end
 
+    # dupped initialization
+    def initialize_dup(orig)
+      super
+      # if a response gets dupped, the body handle must also get dupped to prevent
+      # two responses from using the same file handle to read.
+      @body = orig.body.dup
+    end
+
     # closes the respective +@request+ and +@body+.
     def close
       @request.close
@@ -126,7 +134,7 @@ module HTTPX
 
     # :nocov:
     def inspect
-      "#<Response:#{object_id} " \
+      "#<#{self.class}:#{object_id} " \
         "HTTP/#{version} " \
         "@status=#{@status} " \
         "@headers=#{@headers} " \
@@ -275,6 +283,8 @@ module HTTPX
       true
     end
 
+    def finish!; end
+
     # raises the wrapped exception.
     def raise_for_status
       raise @error

data/lib/httpx/selector.rb

@@ -35,14 +35,21 @@ module HTTPX
         end
 
         begin
-          select(timeout, &:call)
+          select(timeout) do |c|
+            c.log(level: 2) { "[#{c.state}] selected#{" after #{timeout} secs" unless timeout.nil?}..." }
+
+            c.call
+          end
+
           @timers.fire
         rescue TimeoutError => e
           @timers.fire(e)
         end
       end
     rescue StandardError => e
-      emit_error(e)
+      each_connection do |c|
+        c.emit(:error, e)
+      end
     rescue Exception # rubocop:disable Lint/RescueException
       each_connection do |conn|
         conn.force_reset
@@ -77,9 +84,10 @@ module HTTPX
       return enum_for(__method__) unless block
 
       @selectables.each do |c|
-        if c.is_a?(Resolver::Resolver)
+        case c
+        when Resolver::Resolver
           c.each_connection(&block)
-        else
+        when Connection
           yield c
         end
       end
@@ -133,6 +141,8 @@ module HTTPX
       @selectables.delete_if do |io|
         interests = io.interests
 
+        io.log(level: 2) { "[#{io.state}] registering for select (#{interests})#{" for #{interval} seconds" unless interval.nil?}" }
+
         (r ||= []) << io if READABLE.include?(interests)
         (w ||= []) << io if WRITABLE.include?(interests)
 
@@ -169,6 +179,8 @@ module HTTPX
 
       interests = io.interests
 
+      io.log(level: 2) { "[#{io.state}] registering for select (#{interests})#{" for #{interval} seconds" unless interval.nil?}" }
+
       result = case interests
                when :r then io.to_io.wait_readable(interval)
                when :w then io.to_io.wait_writable(interval)
@@ -205,13 +217,5 @@ module HTTPX
 
       connection_interval
     end
-
-    def emit_error(e)
-      @selectables.each do |c|
-        next if c.is_a?(Resolver::Resolver)
-
-        c.emit(:error, e)
-      end
-    end
   end
 end