httpx 1.2.6 → 1.4.4

data/lib/httpx/resolver.rb

@@ -53,8 +53,8 @@ module HTTPX
 
     def cached_lookup(hostname)
       now = Utils.now
-      @lookup_mutex.synchronize do
-        lookup(hostname, now)
+      lookup_synchronize do |lookups|
+        lookup(hostname, lookups, now)
       end
     end
 
@@ -63,37 +63,37 @@ module HTTPX
       entries.each do |entry|
         entry["TTL"] += now
       end
-      @lookup_mutex.synchronize do
+      lookup_synchronize do |lookups|
         case family
         when Socket::AF_INET6
-          @lookups[hostname].concat(entries)
+          lookups[hostname].concat(entries)
         when Socket::AF_INET
-          @lookups[hostname].unshift(*entries)
+          lookups[hostname].unshift(*entries)
         end
         entries.each do |entry|
           next unless entry["name"] != hostname
 
           case family
           when Socket::AF_INET6
-            @lookups[entry["name"]] << entry
+            lookups[entry["name"]] << entry
           when Socket::AF_INET
-            @lookups[entry["name"]].unshift(entry)
+            lookups[entry["name"]].unshift(entry)
           end
         end
       end
     end
 
     # do not use directly!
-    def lookup(hostname, ttl)
-      return unless @lookups.key?(hostname)
+    def lookup(hostname, lookups, ttl)
+      return unless lookups.key?(hostname)
 
-      entries = @lookups[hostname] = @lookups[hostname].select do |address|
+      entries = lookups[hostname] = lookups[hostname].select do |address|
         address["TTL"] > ttl
       end
 
       ips = entries.flat_map do |address|
         if address.key?("alias")
-          lookup(address["alias"], ttl)
+          lookup(address["alias"], lookups, ttl)
         else
           IPAddr.new(address["data"])
         end
@@ -103,12 +103,11 @@ module HTTPX
     end
 
     def generate_id
-      @identifier_mutex.synchronize { @identifier = (@identifier + 1) & 0xFFFF }
+      id_synchronize { @identifier = (@identifier + 1) & 0xFFFF }
     end
 
     def encode_dns_query(hostname, type: Resolv::DNS::Resource::IN::A, message_id: generate_id)
-      Resolv::DNS::Message.new.tap do |query|
-        query.id = message_id
+      Resolv::DNS::Message.new(message_id).tap do |query|
         query.rd = 1
         query.add_question(hostname, type)
       end.encode
@@ -150,5 +149,13 @@ module HTTPX
 
       [:ok, addresses]
     end
+
+    def lookup_synchronize
+      @lookup_mutex.synchronize { yield(@lookups) }
+    end
+
+    def id_synchronize(&block)
+      @identifier_mutex.synchronize(&block)
+    end
   end
 end

data/lib/httpx/response/body.rb

@@ -17,12 +17,23 @@ module HTTPX
       @headers = response.headers
       @options = options
       @window_size = options.window_size
-      @encoding = response.content_type.charset || Encoding::BINARY
       @encodings = []
       @length = 0
       @buffer = nil
       @reader = nil
       @state = :idle
+
+      # initialize response encoding
+      @encoding = if (enc = response.content_type.charset)
+        begin
+          Encoding.find(enc)
+        rescue ArgumentError
+          Encoding::BINARY
+        end
+      else
+        Encoding::BINARY
+      end
+
       initialize_inflaters
     end
 

data/lib/httpx/response.rb

@@ -52,9 +52,6 @@ module HTTPX
     # copies the response body to a different location.
     def_delegator :@body, :copy_to
 
-    # closes the body.
-    def_delegator :@body, :close
-
     # the corresponding request uri.
     def_delegator :@request, :uri
 
@@ -74,6 +71,12 @@ module HTTPX
       @content_type = nil
     end
 
+    # closes the respective +@request+ and +@body+.
+    def close
+      @request.close
+      @body.close
+    end
+
     # merges headers defined in +h+ into the response headers.
     def merge_headers(h)
       @headers = @headers.merge(h)
@@ -166,10 +169,12 @@ module HTTPX
       decode(Transcoder::Form)
     end
 
-    # decodes the response payload into a Nokogiri::XML::Node object **if** the payload is valid
-    # "application/xml" (requires the "nokogiri" gem).
     def xml
-      decode(Transcoder::Xml)
+      # TODO: remove at next major version.
+      warn "DEPRECATION WARNING: calling `.#{__method__}` on plain HTTPX responses is deprecated. " \
+           "Use HTTPX.plugin(:xml) sessions and call `.#{__method__}` in its responses instead."
+      require "httpx/plugins/xml"
+      decode(Plugins::XML::Transcoder)
     end
 
     private
@@ -247,11 +252,11 @@ module HTTPX
     # the IP address of the peer server.
     def_delegator :@request, :peer_address
 
-    def initialize(request, error, options)
+    def initialize(request, error)
       @request = request
       @response = request.response if request.response.is_a?(Response)
       @error = error
-      @options = Options.new(options)
+      @options = request.options
       log_exception(@error)
     end
 
@@ -262,7 +267,7 @@ module HTTPX
 
     # closes the error resources.
     def close
-      @response.close if @response && @response.respond_to?(:close)
+      @response.close if @response
     end
 
     # always true for error responses.
@@ -277,6 +282,8 @@ module HTTPX
 
     # buffers lost chunks to error response
     def <<(data)
+      return unless @response
+
       @response << data
     end
   end

data/lib/httpx/selector.rb

@@ -2,71 +2,142 @@
 
 require "io/wait"
 
-class HTTPX::Selector
-  READABLE = %i[rw r].freeze
-  WRITABLE = %i[rw w].freeze
+module HTTPX
+  class Selector
+    extend Forwardable
 
-  private_constant :READABLE
-  private_constant :WRITABLE
+    READABLE = %i[rw r].freeze
+    WRITABLE = %i[rw w].freeze
 
-  def initialize
-    @selectables = []
-  end
+    private_constant :READABLE
+    private_constant :WRITABLE
 
-  # deregisters +io+ from selectables.
-  def deregister(io)
-    @selectables.delete(io)
-  end
+    def_delegator :@timers, :after
 
-  # register +io+.
-  def register(io)
-    return if @selectables.include?(io)
+    def_delegator :@selectables, :empty?
 
-    @selectables << io
-  end
+    def initialize
+      @timers = Timers.new
+      @selectables = []
+      @is_timer_interval = false
+    end
 
-  private
+    def each(&blk)
+      @selectables.each(&blk)
+    end
 
-  def select_many(interval, &block)
-    selectables, r, w = nil
+    def next_tick
+      catch(:jump_tick) do
+        timeout = next_timeout
+        if timeout && timeout.negative?
+          @timers.fire
+          throw(:jump_tick)
+        end
 
-    # first, we group IOs based on interest type. On call to #interests however,
-    # things might already happen, and new IOs might be registered, so we might
-    # have to start all over again. We do this until we group all selectables
-    begin
-      loop do
         begin
-          r = nil
-          w = nil
+          select(timeout, &:call)
+          @timers.fire
+        rescue TimeoutError => e
+          @timers.fire(e)
+        end
+      end
+    rescue StandardError => e
+      emit_error(e)
+    rescue Exception # rubocop:disable Lint/RescueException
+      each_connection do |conn|
+        conn.force_reset
+        conn.disconnect
+      end
 
-          selectables = @selectables
-          @selectables = []
+      raise
+    end
 
-          selectables.delete_if do |io|
-            interests = io.interests
+    def terminate
+      # array may change during iteration
+      selectables = @selectables.reject(&:inflight?)
 
-            (r ||= []) << io if READABLE.include?(interests)
-            (w ||= []) << io if WRITABLE.include?(interests)
+      selectables.each(&:terminate)
 
-            io.state == :closed
-          end
+      until selectables.empty?
+        next_tick
 
-          if @selectables.empty?
-            @selectables = selectables
+        selectables &= @selectables
+      end
+    end
 
-            # do not run event loop if there's nothing to wait on.
-            # this might happen if connect failed and connection was unregistered.
-            return if (!r || r.empty?) && (!w || w.empty?) && !selectables.empty?
+    def find_resolver(options)
+      res = @selectables.find do |c|
+        c.is_a?(Resolver::Resolver) && options == c.options
+      end
+
+      res.multi if res
+    end
 
-            break
-          else
-            @selectables.concat(selectables)
-          end
-        rescue StandardError
-          @selectables = selectables if selectables
-          raise
+    def each_connection(&block)
+      return enum_for(__method__) unless block
+
+      @selectables.each do |c|
+        if c.is_a?(Resolver::Resolver)
+          c.each_connection(&block)
+        else
+          yield c
         end
       end
+    end
+
+    def find_connection(request_uri, options)
+      each_connection.find do |connection|
+        connection.match?(request_uri, options)
+      end
+    end
+
+    def find_mergeable_connection(connection)
+      each_connection.find do |ch|
+        ch != connection && ch.mergeable?(connection)
+      end
+    end
+
+    # deregisters +io+ from selectables.
+    def deregister(io)
+      @selectables.delete(io)
+    end
+
+    # register +io+.
+    def register(io)
+      return if @selectables.include?(io)
+
+      @selectables << io
+    end
+
+    private
+
+    def select(interval, &block)
+      # do not cause an infinite loop here.
+      #
+      # this may happen if timeout calculation actually triggered an error which causes
+      # the connections to be reaped (such as the total timeout error) before #select
+      # gets called.
+      return if interval.nil? && @selectables.empty?
+
+      return select_one(interval, &block) if @selectables.size == 1
+
+      select_many(interval, &block)
+    end
+
+    def select_many(interval, &block)
+      r, w = nil
+
+      # first, we group IOs based on interest type. On call to #interests however,
+      # things might already happen, and new IOs might be registered, so we might
+      # have to start all over again. We do this until we group all selectables
+      @selectables.delete_if do |io|
+        interests = io.interests
+
+        (r ||= []) << io if READABLE.include?(interests)
+        (w ||= []) << io if WRITABLE.include?(interests)
+
+        io.state == :closed
+      end
 
       # TODO: what to do if there are no selectables?
 
@@ -76,63 +147,71 @@ class HTTPX::Selector
         [*r, *w].each { |io| io.handle_socket_timeout(interval) }
         return
       end
-    rescue IOError, SystemCallError
-      @selectables.reject!(&:closed?)
-      retry
-    end
 
-    if writers
-      readers.each do |io|
-        yield io
+      if writers
+        readers.each do |io|
+          yield io
 
-        # so that we don't yield 2 times
-        writers.delete(io)
-      end if readers
+          # so that we don't yield 2 times
+          writers.delete(io)
+        end if readers
 
-      writers.each(&block)
-    else
-      readers.each(&block) if readers
+        writers.each(&block)
+      else
+        readers.each(&block) if readers
+      end
     end
-  end
 
-  def select_one(interval)
-    io = @selectables.first
+    def select_one(interval)
+      io = @selectables.first
 
-    return unless io
+      return unless io
 
-    interests = io.interests
+      interests = io.interests
 
-    result = case interests
-             when :r then io.to_io.wait_readable(interval)
-             when :w then io.to_io.wait_writable(interval)
-             when :rw then io.to_io.wait(interval, :read_write)
-             when nil then return
-    end
+      result = case interests
+               when :r then io.to_io.wait_readable(interval)
+               when :w then io.to_io.wait_writable(interval)
+               when :rw then io.to_io.wait(interval, :read_write)
+               when nil then return
+      end
 
-    unless result || interval.nil?
-      io.handle_socket_timeout(interval)
-      return
+      unless result || interval.nil?
+        io.handle_socket_timeout(interval) unless @is_timer_interval
+        return
+      end
+      # raise TimeoutError.new(interval, "timed out while waiting on select")
+
+      yield io
+      # rescue IOError, SystemCallError
+      #   @selectables.reject!(&:closed?)
+      #   raise unless @selectables.empty?
     end
-    # raise HTTPX::TimeoutError.new(interval, "timed out while waiting on select")
 
-    yield io
-  rescue IOError, SystemCallError
-    @selectables.reject!(&:closed?)
-    raise unless @selectables.empty?
-  end
+    def next_timeout
+      @is_timer_interval = false
 
-  def select(interval, &block)
-    # do not cause an infinite loop here.
-    #
-    # this may happen if timeout calculation actually triggered an error which causes
-    # the connections to be reaped (such as the total timeout error) before #select
-    # gets called.
-    return if interval.nil? && @selectables.empty?
+      timer_interval = @timers.wait_interval
 
-    return select_one(interval, &block) if @selectables.size == 1
+      connection_interval = @selectables.filter_map(&:timeout).min
 
-    select_many(interval, &block)
-  end
+      return connection_interval unless timer_interval
+
+      if connection_interval.nil? || timer_interval <= connection_interval
+        @is_timer_interval = true
+
+        return timer_interval
+      end
+
+      connection_interval
+    end
+
+    def emit_error(e)
+      @selectables.each do |c|
+        next if c.is_a?(Resolver::Resolver)
 
-  public :select
+        c.emit(:error, e)
+      end
+    end
+  end
 end