httpx 1.6.1 → 1.6.3

data/lib/httpx/resolver/resolver.rb

@@ -7,7 +7,6 @@ module HTTPX
   # from the Selectable API.
   #
   class Resolver::Resolver
-    include Callbacks
     include Loggable
 
     using ArrayExtensions::Intersect
@@ -39,8 +38,6 @@ module HTTPX
       @record_type = RECORD_TYPES[family]
       @options = options
       @connections = []
-
-      set_resolver_callbacks
     end
 
     def each_connection(&block)
@@ -55,6 +52,12 @@ module HTTPX
 
     alias_method :terminate, :close
 
+    def force_close(*args)
+      while (connection = @connections.shift)
+        connection.force_close(*args)
+      end
+    end
+
     def closed?
       true
     end
@@ -72,7 +75,7 @@ module HTTPX
 
       # double emission check, but allow early resolution to work
       conn_addrs = connection.addresses
-      return if !early_resolve && conn_addrs && (!conn_addrs.empty? && !addresses.intersect?(!conn_addrs))
+      return if !early_resolve && conn_addrs && (!conn_addrs.empty? && !addresses.intersect?(conn_addrs))
 
       log do
         "resolver #{FAMILY_TYPES[RECORD_TYPES[family]]}: " \
@@ -104,25 +107,23 @@ module HTTPX
       end
     end
 
-    private
-
-    def emit_resolved_connection(connection, addresses, early_resolve)
-      begin
-        connection.addresses = addresses
-
-        return if connection.state == :closed
-
-        emit(:resolve, connection)
-      rescue StandardError => e
-        if early_resolve
-          connection.force_reset
-          throw(:resolve_error, e)
-        else
-          emit(:error, connection, e)
+    def handle_error(error)
+      if error.respond_to?(:connection) &&
+         error.respond_to?(:host)
+        @connections.delete(error.connection)
+        emit_resolve_error(error.connection, error.host, error)
+      else
+        while (connection = @connections.shift)
+          emit_resolve_error(connection, connection.peer.host, error)
         end
       end
     end
 
+    def on_error(error)
+      handle_error(error)
+      disconnect
+    end
+
     def early_resolve(connection, hostname: connection.peer.host)
       addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
 
@@ -137,6 +138,25 @@ module HTTPX
       true
     end
 
+    private
+
+    def emit_resolved_connection(connection, addresses, early_resolve)
+      begin
+        connection.addresses = addresses
+
+        return if connection.state == :closed
+
+        resolve_connection(connection)
+      rescue StandardError => e
+        if early_resolve
+          connection.force_close
+          throw(:resolve_error, e)
+        else
+          emit_connection_error(connection, e)
+        end
+      end
+    end
+
     def emit_resolve_error(connection, hostname = connection.peer.host, ex = nil)
       emit_connection_error(connection, resolve_error(hostname, ex))
     end
@@ -150,12 +170,6 @@ module HTTPX
       error
     end
 
-    def set_resolver_callbacks
-      on(:resolve, &method(:resolve_connection))
-      on(:error, &method(:emit_connection_error))
-      on(:close, &method(:close_resolver))
-    end
-
     def resolve_connection(connection)
       @current_session.__send__(:on_resolver_connection, connection, @current_selector)
     end
@@ -163,11 +177,14 @@ module HTTPX
     def emit_connection_error(connection, error)
       return connection.handle_connect_error(error) if connection.connecting?
 
-      connection.emit(:error, error)
+      connection.on_error(error)
     end
 
-    def close_resolver(resolver)
-      @current_session.__send__(:on_resolver_close, resolver, @current_selector)
+    def disconnect
+      return if closed?
+
+      close
+      @current_session.deselect_resolver(self, @current_selector)
     end
   end
 end

data/lib/httpx/resolver/system.rb

@@ -56,13 +56,21 @@ module HTTPX
     end
 
     def empty?
-      true
+      @connections.empty?
     end
 
     def close
       transition(:closed)
     end
 
+    def force_close(*)
+      close
+      @queries.clear
+      @timeouts.clear
+      @ips.clear
+      super
+    end
+
     def closed?
       @state == :closed
     end
@@ -86,36 +94,42 @@ module HTTPX
     end
 
     def timeout
-      return unless @queries.empty?
-
       _, connection = @queries.first
 
       return unless connection
 
-      @timeouts[connection.peer.host].first
+      timeouts = @timeouts[connection.peer.host]
+
+      return if timeouts.empty?
+
+      log(level: 2) { "resolver #{FAMILY_TYPES[@record_type]}: next timeout #{timeouts.first} secs... (#{timeouts.size - 1} left)" }
+
+      timeouts.first
     end
 
-    def <<(connection)
+    def lazy_resolve(connection)
       @connections << connection
       resolve
-    end
 
-    def early_resolve(connection, **)
-      self << connection
-      true
+      return if empty?
+
+      @current_session.select_resolver(self, @current_selector)
     end
 
+    def early_resolve(connection, **); end
+
     def handle_socket_timeout(interval)
       error = HTTPX::ResolveTimeoutError.new(interval, "timed out while waiting on select")
       error.set_backtrace(caller)
-      @queries.each do |host, connection|
-        @connections.delete(connection)
-        emit_resolve_error(connection, host, error)
+      @queries.each do |_, connection| # rubocop:disable Style/HashEachMethods
+        emit_resolve_error(connection, connection.peer.host, error) if @connections.delete(connection)
       end
 
       while (connection = @connections.shift)
         emit_resolve_error(connection, connection.peer.host, error)
       end
+
+      close_or_resolve
     end
 
     private
@@ -140,34 +154,38 @@ module HTTPX
     def consume
       return if @connections.empty?
 
-      if @pipe_read.wait_readable
-        event = @pipe_read.getbyte
+      event = @pipe_read.read_nonblock(1, exception: false)
 
-        case event
-        when DONE
-          *pair, addrs = @pipe_mutex.synchronize { @ips.pop }
-          if pair
-            @queries.delete(pair)
-            family, connection = pair
-            @connections.delete(connection)
+      return if event == :wait_readable
 
-            catch(:coalesced) { emit_addresses(connection, family, addrs) }
-          end
-        when ERROR
-          *pair, error = @pipe_mutex.synchronize { @ips.pop }
-          if pair && error
-            @queries.delete(pair)
-            @connections.delete(connection)
-
-            _, connection = pair
-            emit_resolve_error(connection, connection.peer.host, error)
-          end
+      raise ResolveError, "socket pipe closed unexpectedly" if event.nil?
+
+      case event.unpack1("C")
+      when DONE
+        *pair, addrs = @pipe_mutex.synchronize { @ips.pop }
+        if pair
+          @queries.delete(pair)
+          family, connection = pair
+          @connections.delete(connection)
+
+          catch(:coalesced) { emit_addresses(connection, family, addrs) }
+        end
+      when ERROR
+        *pair, error = @pipe_mutex.synchronize { @ips.pop }
+        if pair && error
+          @queries.delete(pair)
+          _, connection = pair
+          @connections.delete(connection)
+
+          emit_resolve_error(connection, connection.peer.host, error)
         end
       end
 
-      return emit(:close, self) if @connections.empty?
+      return disconnect if @connections.empty?
 
       resolve
+    rescue StandardError => e
+      on_error(e)
     end
 
     def resolve(connection = nil, hostname = nil)
@@ -240,16 +258,22 @@ module HTTPX
           end
         end
       end
+      Thread.pass
     end
 
-    def __addrinfo_resolve(host, scheme)
-      Addrinfo.getaddrinfo(host, scheme, Socket::AF_UNSPEC, Socket::SOCK_STREAM)
-    end
+    def close_or_resolve
+      # drop already closed connections
+      @connections.shift until @connections.empty? || @connections.first.state != :closed
 
-    def emit_connection_error(_, error)
-      throw(:resolve_error, error)
+      if (@connections - @queries.map(&:last)).empty?
+        disconnect
+      else
+        resolve
+      end
     end
 
-    def close_resolver(resolver); end
+    def __addrinfo_resolve(host, scheme)
+      Addrinfo.getaddrinfo(host, scheme, Socket::AF_UNSPEC, Socket::SOCK_STREAM)
+    end
   end
 end

data/lib/httpx/selector.rb

@@ -51,7 +51,7 @@ module HTTPX
 
         begin
           select(timeout) do |c|
-            c.log(level: 2) { "[#{c.state}] selected#{" after #{timeout} secs" unless timeout.nil?}..." }
+            c.log(level: 2) { "[#{c.state}] selected from selector##{object_id} #{" after #{timeout} secs" unless timeout.nil?}..." }
 
             c.call
           end
@@ -61,17 +61,6 @@ module HTTPX
           @timers.fire(e)
         end
       end
-    rescue StandardError => e
-      each_connection do |c|
-        c.emit(:error, e)
-      end
-    rescue Exception # rubocop:disable Lint/RescueException
-      each_connection do |conn|
-        conn.force_reset
-        conn.disconnect
-      end
-
-      raise
     end
 
     def terminate
@@ -157,7 +146,9 @@ module HTTPX
 
         next(is_closed) if is_closed
 
-        io.log(level: 2) { "[#{io.state}] registering for select (#{interests})#{" for #{interval} seconds" unless interval.nil?}" }
+        io.log(level: 2) do
+          "[#{io.state}] registering in selector##{object_id} for select (#{interests})#{" for #{interval} seconds" unless interval.nil?}"
+        end
 
         if READABLE.include?(interests)
           r = r.nil? ? io : (Array(r) << io)
@@ -206,7 +197,21 @@ module HTTPX
     end
 
     def select_many(r, w, interval, &block)
-      readers, writers = ::IO.select(r, w, nil, interval)
+      begin
+        readers, writers = ::IO.select(r, w, nil, interval)
+      rescue StandardError => e
+        (Array(r) + Array(w)).each do |sel|
+          sel.on_error(e)
+        end
+
+        return
+      rescue Exception => e # rubocop:disable Lint/RescueException
+        (Array(r) + Array(w)).each do |sel|
+          sel.force_close(true)
+        end
+
+        raise e
+      end
 
       if readers.nil? && writers.nil? && interval
         [*r, *w].each { |io| io.handle_socket_timeout(interval) }
@@ -228,12 +233,22 @@ module HTTPX
     end
 
     def select_one(io, interests, interval)
-      result =
-        case interests
-        when :r then io.to_io.wait_readable(interval)
-        when :w then io.to_io.wait_writable(interval)
-        when :rw then rw_wait(io, interval)
-        end
+      begin
+        result =
+          case interests
+          when :r then io.to_io.wait_readable(interval)
+          when :w then io.to_io.wait_writable(interval)
+          when :rw then rw_wait(io, interval)
+          end
+      rescue StandardError => e
+        io.on_error(e)
+
+        return
+      rescue Exception => e # rubocop:disable Lint/RescueException
+        io.force_close(true)
+
+        raise e
+      end
 
       unless result || interval.nil?
         io.handle_socket_timeout(interval) unless @is_timer_interval

data/lib/httpx/session.rb

@@ -120,13 +120,16 @@ module HTTPX
     end
 
     def select_connection(connection, selector)
-      pin_connection(connection, selector)
+      pin(connection, selector)
+      connection.log(level: 2) do
+        "registering into selector##{selector.object_id}"
+      end
       selector.register(connection)
     end
 
-    def pin_connection(connection, selector)
-      connection.current_session = self
-      connection.current_selector = selector
+    def pin(conn_or_resolver, selector)
+      conn_or_resolver.current_session = self
+      conn_or_resolver.current_selector = selector
     end
 
     alias_method :select_resolver, :select_connection
@@ -137,9 +140,6 @@ module HTTPX
       end
       selector.deregister(connection)
 
-      # when connections coalesce
-      return if connection.state == :idle
-
       return if cloned
 
       return if @closing && connection.state == :closed
@@ -194,7 +194,7 @@ module HTTPX
         if options.io
           select_connection(connection, selector)
         else
-          pin_connection(connection, selector)
+          pin(connection, selector)
         end
       when :closing, :closed
         connection.idling
@@ -205,7 +205,7 @@ module HTTPX
           resolve_connection(connection, selector)
         end
       else
-        pin_connection(connection, selector)
+        pin(connection, selector)
       end
 
       connection
@@ -224,13 +224,7 @@ module HTTPX
 
     # tries deactivating connections in the +selector+, deregistering the ones that have been deactivated.
     def deactivate(selector)
-      selector.each_connection.select do |c|
-        c.deactivate
-
-        c.state == :inactive
-      end.each do |c| # rubocop:disable Style/MultilineBlockChain
-        deselect_connection(c, selector)
-      end
+      selector.each_connection.each(&:deactivate)
     end
 
     # callback executed when an HTTP/2 promise frame has been received.
@@ -395,6 +389,7 @@ module HTTPX
 
       resolver = find_resolver_for(connection, selector)
 
+      pin(connection, selector)
       resolver.early_resolve(connection) || resolver.lazy_resolve(connection)
     end
 
@@ -402,6 +397,9 @@ module HTTPX
       from_pool = false
       found_connection = selector.find_mergeable_connection(connection) || begin
         from_pool = true
+        connection.log(level: 2) do
+          "try finding a mergeable connection in pool##{@pool.object_id}"
+        end
         @pool.checkout_mergeable_connection(connection)
       end
 
@@ -409,19 +407,12 @@ module HTTPX
 
       connection.log(level: 2) do
         "try coalescing from #{from_pool ? "pool##{@pool.object_id}" : "selector##{selector.object_id}"} " \
-          "(conn##{found_connection.object_id}[#{found_connection.origin}])"
+          "(connection##{found_connection.object_id}[#{found_connection.origin}])"
       end
 
       coalesce_connections(found_connection, connection, selector, from_pool)
     end
 
-    def on_resolver_close(resolver, selector)
-      return if resolver.closed?
-
-      deselect_resolver(resolver, selector)
-      resolver.close unless resolver.closed?
-    end
-
     def find_resolver_for(connection, selector)
       if (resolver = selector.find_resolver(connection.options))
         resolver.log(level: 2) { "found resolver##{connection.object_id}(#{connection.state}) in selector##{selector.object_id}" }
@@ -430,8 +421,7 @@ module HTTPX
 
       resolver = @pool.checkout_resolver(connection.options)
       resolver.log(level: 2) { "found resolver##{connection.object_id}(#{connection.state}) in pool##{@pool.object_id}" }
-      resolver.current_session = self
-      resolver.current_selector = selector
+      pin(resolver, selector)
 
       resolver
     end
@@ -449,9 +439,9 @@ module HTTPX
         return false
       end
 
-      conn2.log(level: 2) { "coalescing with conn##{conn1.object_id}[#{conn1.origin}])" }
-      conn2.coalesce!(conn1)
+      conn2.log(level: 2) { "coalescing with connection##{conn1.object_id}[#{conn1.origin}])" }
       select_connection(conn1, selector) if from_pool
+      conn2.coalesce!(conn1)
       conn2.disconnect
       true
     end

data/lib/httpx/transcoder/deflate.rb

@@ -10,15 +10,20 @@ module HTTPX
         def deflate(chunk)
           @deflater ||= Zlib::Deflate.new
 
-          if chunk.nil?
-            unless @deflater.closed?
-              last = @deflater.finish
-              @deflater.close
-              last.empty? ? nil : last
-            end
-          else
-            @deflater.deflate(chunk)
+          unless chunk.nil?
+            chunk = @deflater.deflate(chunk)
+
+            # deflate call may return nil, while still
+            # retaining the last chunk in the deflater.
+            return chunk unless chunk.empty?
           end
+
+          return if @deflater.closed?
+
+          last = @deflater.finish
+          @deflater.close
+
+          last unless last.empty?
         end
       end
 

data/lib/httpx/transcoder/utils/body_reader.rb

@@ -28,8 +28,7 @@ module HTTPX
         begin
           chunk = @body.next
           if outbuf
-            outbuf.clear.force_encoding(Encoding::BINARY)
-            outbuf << chunk
+            outbuf.replace(chunk)
           else
             outbuf = chunk
           end

data/lib/httpx/transcoder/utils/deflater.rb

@@ -31,8 +31,7 @@ module HTTPX
         return unless compressed_chunk
 
         if outbuf
-          outbuf.clear.force_encoding(Encoding::BINARY)
-          outbuf << compressed_chunk
+          outbuf.replace(compressed_chunk)
         else
           compressed_chunk
         end

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "1.6.1"
+  VERSION = "1.6.3"
 end

data/sig/connection.rbs

@@ -46,6 +46,7 @@ module HTTPX
     @exhausted: bool
     @cloned: bool
     @coalesced_connection: instance?
+    @altsvc_connection: instance?
     @sibling: instance?
     @main_sibling: bool
 
@@ -66,6 +67,8 @@ module HTTPX
 
     def coalesce!: (instance connection) -> void
 
+    def coalesced?: () -> boolish
+
     def coalescable?: (Connection connection) -> bool
 
     def create_idle: (?Hash[Symbol, untyped] options) -> instance
@@ -76,7 +79,9 @@ module HTTPX
 
     def connecting?: () -> bool
 
-    def io_connected?: () -> bool
+    def current_context?: () -> bool
+
+    def io_connected?: () -> boolish
 
     def inflight?: () -> boolish
 
@@ -90,6 +95,8 @@ module HTTPX
 
     def close: () -> void
 
+    def force_close: (?bool delete_pending) -> void
+
     def force_reset: (?bool cloned) -> void
 
     def reset: () -> void
@@ -112,6 +119,8 @@ module HTTPX
 
     def disconnect: () -> void
 
+    def on_error: (HTTPX::TimeoutError | Error | StandardError error, ?Request? request) -> void
+
     private
 
     def initialize: (http_uri uri, Options options) -> void
@@ -130,6 +139,8 @@ module HTTPX
 
     def send_request_to_parser: (Request request) -> void
 
+    def enqueue_pending_requests_from_parser: (HTTP1 | HTTP2 parser) -> void
+
     def build_parser: (?String protocol) -> (Object & _Parser)
 
     def set_parser_callbacks: (HTTP1 | HTTP2 parser) -> void
@@ -142,8 +153,6 @@ module HTTPX
 
     def build_socket: (?Array[Resolver::Entry]? addrs) -> (TCP | SSL | UNIX)
 
-    def on_error: (HTTPX::TimeoutError | Error | StandardError error, ?Request? request) -> void
-
     def handle_error: (StandardError error, ?Request? request) -> void
 
     def close_sibling: () -> void

data/sig/loggable.rbs

@@ -12,6 +12,10 @@ module HTTPX
 
     def log_exception: (Exception error, ?level: Integer, ?color: Symbol, ?debug_level: Integer, ?debug: _IOLogger?) -> void
 
-    def log_redact: (_ToS text, ?bool should_redact) -> String
+    def log_redact_headers: (_ToS text) -> String
+
+    def log_redact_body: (_ToS text) -> String
+
+    def log_redact: (_ToS text, bool should_redact) -> String
   end
 end

data/sig/options.rbs

@@ -17,9 +17,11 @@ module HTTPX
 
     DEFAULT_OPTIONS: Hash[Symbol, untyped]
     REQUEST_BODY_IVARS: Array[Symbol]
+    USER_AGENT: String
 
     type timeout_type = :connect_timeout | :settings_timeout | :close_handshake_timeout | :operation_timeout | :keep_alive_timeout | :read_timeout | :write_timeout | :request_timeout
     type timeout = Hash[timeout_type, Numeric?]
+    type redact_value = :headers | :body | bool
 
     def self.new: (?options) -> instance
 
@@ -146,6 +148,8 @@ module HTTPX
 
     def initialize: (?options options) -> void
 
+    def do_initialize: (?options options) -> void
+
     def access_option: (Hash[Symbol, untyped] | Object | nil obj, Symbol k, Hash[Symbol, Symbol]? ivar_map) -> untyped
 
     # integer
@@ -175,7 +179,7 @@ module HTTPX
     def option_io: (io_option value) -> io_option
     def option_fallback_protocol: (String value) -> String
     def option_debug: (_IOLogger value) -> _IOLogger
-    def option_debug_redact: (bool value) -> bool
+    def option_debug_redact: (redact_value value) -> redact_value
     def option_compress_request_body: (bool value) -> bool
     def option_decompress_response_body: (bool value) -> bool
     def option_persistent: (bool value) -> bool

data/sig/plugins/callbacks.rbs

@@ -31,6 +31,9 @@ module HTTPX
 
         def emit_or_callback_error: (*untyped) -> void
       end
+
+      module ConnectionMethods
+      end
     end
 
     type sessionCallbacks = Session & Callbacks::InstanceMethods