httpx 1.3.4 → 1.4.1

data/lib/httpx/pool.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "forwardable"
 require "httpx/selector"
 require "httpx/connection"
 require "httpx/resolver"
@@ -8,110 +7,31 @@ require "httpx/resolver"
 module HTTPX
   class Pool
     using ArrayExtensions::FilterMap
-    extend Forwardable
+    using URIExtensions
 
-    def_delegator :@timers, :after
+    POOL_TIMEOUT = 5
 
-    def initialize
-      @resolvers = {}
-      @timers = Timers.new
-      @selector = Selector.new
-      @connections = []
-    end
-
-    def wrap
-      connections = @connections
+    # Sets up the connection pool with the given +options+, which can be the following:
+    #
+    # :max_connections_per_origin :: the maximum number of connections held in the pool pointing to a given origin.
+    # :pool_timeout :: the number of seconds to wait for a connection to a given origin (before raising HTTPX::PoolTimeoutError)
+    #
+    def initialize(options)
+      @max_connections_per_origin = options.fetch(:max_connections_per_origin, Float::INFINITY)
+      @pool_timeout = options.fetch(:pool_timeout, POOL_TIMEOUT)
+      @resolvers = Hash.new { |hs, resolver_type| hs[resolver_type] = [] }
+      @resolver_mtx = Thread::Mutex.new
       @connections = []
-
-      begin
-        yield self
-      ensure
-        @connections.unshift(*connections)
-      end
-    end
-
-    def empty?
-      @connections.empty?
-    end
-
-    def next_tick
-      catch(:jump_tick) do
-        timeout = next_timeout
-        if timeout && timeout.negative?
-          @timers.fire
-          throw(:jump_tick)
-        end
-
-        begin
-          @selector.select(timeout, &:call)
-          @timers.fire
-        rescue TimeoutError => e
-          @timers.fire(e)
-        end
-      end
-    rescue StandardError => e
-      @connections.each do |connection|
-        connection.emit(:error, e)
-      end
-    rescue Exception # rubocop:disable Lint/RescueException
-      @connections.each(&:force_reset)
-      raise
-    end
-
-    def close(connections = @connections)
-      return if connections.empty?
-
-      connections = connections.reject(&:inflight?)
-      connections.each(&:terminate)
-      next_tick until connections.none? { |c| c.state != :idle && @connections.include?(c) }
-
-      # close resolvers
-      outstanding_connections = @connections
-      resolver_connections = @resolvers.each_value.flat_map(&:connections).compact
-      outstanding_connections -= resolver_connections
-
-      return unless outstanding_connections.empty?
-
-      @resolvers.each_value do |resolver|
-        resolver.close unless resolver.closed?
-      end
-      # for https resolver
-      resolver_connections.each(&:terminate)
-      next_tick until resolver_connections.none? { |c| c.state != :idle && @connections.include?(c) }
-    end
-
-    def init_connection(connection, _options)
-      connection.timers = @timers
-      connection.on(:activate) do
-        select_connection(connection)
-      end
-      connection.on(:exhausted) do
-        case connection.state
-        when :closed
-          connection.idling
-          @connections << connection
-          select_connection(connection)
-        when :closing
-          connection.once(:close) do
-            connection.idling
-            @connections << connection
-            select_connection(connection)
-          end
-        end
-      end
-      connection.on(:close) do
-        unregister_connection(connection)
-      end
-      connection.on(:terminate) do
-        unregister_connection(connection, true)
-      end
-      resolve_connection(connection) unless connection.family
+      @connection_mtx = Thread::Mutex.new
+      @origin_counters = Hash.new(0)
+      @origin_conds = Hash.new { |hs, orig| hs[orig] = ConditionVariable.new }
     end
 
-    def deactivate(*connections)
-      connections.each do |connection|
-        connection.deactivate
-        deselect_connection(connection) if connection.state == :inactive
+    def pop_connection
+      @connection_mtx.synchronize do
+        conn = @connections.shift
+        @origin_conds.delete(conn.origin) if conn && (@origin_counters[conn.origin.to_s] -= 1).zero?
+        conn
       end
     end
 
@@ -119,185 +39,94 @@ module HTTPX
     # Many hostnames are reachable through the same IP, so we try to
     # maximize pipelining by opening as few connections as possible.
     #
-    def find_connection(uri, options)
-      conn = @connections.find do |connection|
-        connection.match?(uri, options)
-      end
+    def checkout_connection(uri, options)
+      return checkout_new_connection(uri, options) if options.io
 
-      return unless conn
+      @connection_mtx.synchronize do
+        acquire_connection(uri, options) || begin
+          if @origin_counters[uri.origin] == @max_connections_per_origin
 
-      case conn.state
-      when :closed
-        conn.idling
-        select_connection(conn)
-      when :closing
-        conn.once(:close) do
-          conn.idling
-          select_connection(conn)
-        end
-      end
+            @origin_conds[uri.origin].wait(@connection_mtx, @pool_timeout)
 
-      conn
-    end
-
-    private
-
-    def resolve_connection(connection)
-      @connections << connection unless @connections.include?(connection)
-
-      if connection.addresses || connection.open?
-        #
-        # there are two cases in which we want to activate initialization of
-        # connection immediately:
-        #
-        # 1. when the connection already has addresses, i.e. it doesn't need to
-        #    resolve a name (not the same as name being an IP, yet)
-        # 2. when the connection is initialized with an external already open IO.
-        #
-        connection.once(:connect_error, &connection.method(:handle_error))
-        on_resolver_connection(connection)
-        return
-      end
+            return acquire_connection(uri, options) || raise(PoolTimeoutError.new(uri.origin, @pool_timeout))
+          end
 
-      find_resolver_for(connection) do |resolver|
-        resolver << try_clone_connection(connection, resolver.family)
-        next if resolver.empty?
+          @origin_counters[uri.origin] += 1
 
-        select_connection(resolver)
+          checkout_new_connection(uri, options)
+        end
       end
     end
 
-    def try_clone_connection(connection, family)
-      connection.family ||= family
-
-      return connection if connection.family == family
-
-      new_connection = connection.class.new(connection.origin, connection.options)
-      new_connection.family = family
+    def checkin_connection(connection)
+      return if connection.options.io
 
-      connection.once(:tcp_open) { new_connection.force_reset }
-      connection.once(:connect_error) do |err|
-        if new_connection.connecting?
-          new_connection.merge(connection)
-          connection.emit(:cloned, new_connection)
-          connection.force_reset
-        else
-          connection.__send__(:handle_error, err)
-        end
-      end
+      @connection_mtx.synchronize do
+        @connections << connection
 
-      new_connection.once(:tcp_open) do |new_conn|
-        if new_conn != connection
-          new_conn.merge(connection)
-          connection.force_reset
-        end
-      end
-      new_connection.once(:connect_error) do |err|
-        if connection.connecting?
-          # main connection has the requests
-          connection.merge(new_connection)
-          new_connection.emit(:cloned, connection)
-          new_connection.force_reset
-        else
-          new_connection.__send__(:handle_error, err)
-        end
+        @origin_conds[connection.origin.to_s].signal
       end
-
-      init_connection(new_connection, connection.options)
-      new_connection
     end
 
-    def on_resolver_connection(connection)
-      @connections << connection unless @connections.include?(connection)
-      found_connection = @connections.find do |ch|
-        ch != connection && ch.mergeable?(connection)
-      end
-      return register_connection(connection) unless found_connection
+    def checkout_mergeable_connection(connection)
+      return if connection.options.io
 
-      if found_connection.open?
-        coalesce_connections(found_connection, connection)
-        throw(:coalesced, found_connection) unless @connections.include?(connection)
-      else
-        found_connection.once(:open) do
-          coalesce_connections(found_connection, connection)
+      @connection_mtx.synchronize do
+        idx = @connections.find_index do |ch|
+          ch != connection && ch.mergeable?(connection)
         end
+        @connections.delete_at(idx) if idx
       end
     end
 
-    def on_resolver_error(connection, error)
-      return connection.emit(:connect_error, error) if connection.connecting? && connection.callbacks_for?(:connect_error)
-
-      connection.emit(:error, error)
+    def reset_resolvers
+      @resolver_mtx.synchronize { @resolvers.clear }
     end
 
-    def on_resolver_close(resolver)
-      resolver_type = resolver.class
-      return if resolver.closed?
+    def checkout_resolver(options)
+      resolver_type = options.resolver_class
+      resolver_type = Resolver.resolver_for(resolver_type)
 
-      @resolvers.delete(resolver_type)
+      @resolver_mtx.synchronize do
+        resolvers = @resolvers[resolver_type]
 
-      deselect_connection(resolver)
-      resolver.close unless resolver.closed?
+        idx = resolvers.find_index do |res|
+          res.options == options
+        end
+        resolvers.delete_at(idx) if idx
+      end || checkout_new_resolver(resolver_type, options)
     end
 
-    def register_connection(connection)
-      select_connection(connection)
-    end
+    def checkin_resolver(resolver)
+      @resolver_mtx.synchronize do
+        resolvers = @resolvers[resolver.class]
 
-    def unregister_connection(connection, cleanup = !connection.used?)
-      @connections.delete(connection) if cleanup
-      deselect_connection(connection)
-    end
+        resolver = resolver.multi
 
-    def select_connection(connection)
-      @selector.register(connection)
+        resolvers << resolver unless resolvers.include?(resolver)
+      end
     end
 
-    def deselect_connection(connection)
-      @selector.deregister(connection)
-    end
+    private
 
-    def coalesce_connections(conn1, conn2)
-      return register_connection(conn2) unless conn1.coalescable?(conn2)
+    def acquire_connection(uri, options)
+      idx = @connections.find_index do |connection|
+        connection.match?(uri, options)
+      end
 
-      conn2.emit(:tcp_open, conn1)
-      conn1.merge(conn2)
-      @connections.delete(conn2)
+      @connections.delete_at(idx) if idx
     end
 
-    def next_timeout
-      [
-        @timers.wait_interval,
-        *@resolvers.values.reject(&:closed?).filter_map(&:timeout),
-        *@connections.filter_map(&:timeout),
-      ].compact.min
+    def checkout_new_connection(uri, options)
+      options.connection_class.new(uri, options)
     end
 
-    def find_resolver_for(connection)
-      connection_options = connection.options
-      resolver_type = connection_options.resolver_class
-      resolver_type = Resolver.resolver_for(resolver_type)
-
-      @resolvers[resolver_type] ||= begin
-        resolver_manager = if resolver_type.multi?
-          Resolver::Multi.new(resolver_type, connection_options)
-        else
-          resolver_type.new(connection_options)
-        end
-        resolver_manager.on(:resolve, &method(:on_resolver_connection))
-        resolver_manager.on(:error, &method(:on_resolver_error))
-        resolver_manager.on(:close, &method(:on_resolver_close))
-        resolver_manager
-      end
-
-      manager = @resolvers[resolver_type]
-
-      (manager.is_a?(Resolver::Multi) && manager.early_resolve(connection)) || manager.resolvers.each do |resolver|
-        resolver.pool = self
-        yield resolver
+    def checkout_new_resolver(resolver_type, options)
+      if resolver_type.multi?
+        Resolver::Multi.new(resolver_type, options)
+      else
+        resolver_type.new(options)
       end
-
-      manager
     end
   end
 end

data/lib/httpx/request/body.rb

@@ -25,20 +25,11 @@ module HTTPX
     #   ..., form: { foo: Pathname.open("path/to/file") }) #=> multipart urlencoded encoder
     #   ..., form: { foo: File.open("path/to/file") }) #=> multipart urlencoded encoder
     #   ..., form: { body: "bla") }) #=> raw data encoder
-    def initialize(headers, options, body: nil, form: nil, json: nil, xml: nil, **params)
-      @headers = headers
+    def initialize(h, options, **params)
+      @headers = h
+      @body = self.class.initialize_body(params)
       @options = options.merge(params)
 
-      @body = if body
-        Transcoder::Body.encode(body)
-      elsif form
-        Transcoder::Form.encode(form)
-      elsif json
-        Transcoder::JSON.encode(json)
-      elsif xml
-        Transcoder::Xml.encode(xml)
-      end
-
       if @body
         if @options.compress_request_body && @headers.key?("content-encoding")
 
@@ -61,7 +52,11 @@ module HTTPX
 
       body = stream(@body)
       if body.respond_to?(:read)
-        ::IO.copy_stream(body, ProcIO.new(block))
+        while (chunk = body.read(16_384))
+          block.call(chunk)
+        end
+        # TODO: use copy_stream once bug is resolved: https://bugs.ruby-lang.org/issues/21131
+        # ::IO.copy_stream(body, ProcIO.new(block))
       elsif body.respond_to?(:each)
         body.each(&block)
       else
@@ -69,6 +64,10 @@ module HTTPX
       end
     end
 
+    def close
+      @body.close if @body.respond_to?(:close)
+    end
+
     # if the +@body+ is rewindable, it rewinnds it.
     def rewind
       return if empty?
@@ -123,6 +122,19 @@ module HTTPX
     # :nocov:
 
     class << self
+      def initialize_body(params)
+        if (body = params.delete(:body))
+          # @type var body: bodyIO
+          Transcoder::Body.encode(body)
+        elsif (form = params.delete(:form))
+          # @type var form: Transcoder::urlencoded_input
+          Transcoder::Form.encode(form)
+        elsif (json = params.delete(:json))
+          # @type var body: _ToJson
+          Transcoder::JSON.encode(json)
+        end
+      end
+
       # returns the +body+ wrapped with the correct deflater accordinng to the given +encodisng+.
       def initialize_deflater_body(body, encoding)
         case encoding
@@ -138,17 +150,4 @@ module HTTPX
       end
     end
   end
-
-  # Wrapper yielder which can be used with functions which expect an IO writer.
-  class ProcIO
-    def initialize(block)
-      @block = block
-    end
-
-    # Implementation the IO write protocol, which yield the given chunk to +@block+.
-    def write(data)
-      @block.call(data.dup)
-      data.bytesize
-    end
-  end
 end

data/lib/httpx/request.rb

@@ -11,8 +11,10 @@ module HTTPX
     include Callbacks
     using URIExtensions
 
+    ALLOWED_URI_SCHEMES = %w[https http].freeze
+
     # default value used for "user-agent" header, when not overridden.
-    USER_AGENT = "httpx.rb/#{VERSION}"
+    USER_AGENT = "httpx.rb/#{VERSION}".freeze # rubocop:disable Style/RedundantFreeze
 
     # the upcased string HTTP verb for this request.
     attr_reader :verb
@@ -92,6 +94,8 @@ module HTTPX
         @uri = origin.merge("#{base_path}#{@uri}")
       end
 
+      raise UnsupportedSchemeError, "#{@uri}: #{@uri.scheme}: unsupported URI scheme" unless ALLOWED_URI_SCHEMES.include?(@uri.scheme)
+
       @state = :idle
       @response = nil
       @peer_address = nil
@@ -263,6 +267,8 @@ module HTTPX
         return unless @state == :body
       when :done
         return if @state == :expect
+
+        @body.close
       end
       @state = nextstate
       emit(@state, self)

data/lib/httpx/resolver/https.rb

@@ -27,7 +27,7 @@ module HTTPX
       use_get: false,
     }.freeze
 
-    def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close, :terminate
+    def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close, :terminate, :inflight?
 
     def initialize(_, options)
       super
@@ -43,7 +43,7 @@ module HTTPX
     end
 
     def <<(connection)
-      return if @uri.origin == connection.origin.to_s
+      return if @uri.origin == connection.peer.to_s
 
       @uri_addresses ||= HTTPX::Resolver.nolookup_resolve(@uri.host) || @resolver.getaddresses(@uri.host)
 
@@ -66,30 +66,25 @@ module HTTPX
     end
 
     def resolver_connection
-      @resolver_connection ||= @pool.find_connection(@uri, @options) || begin
-        @building_connection = true
-        connection = @options.connection_class.new(@uri, @options.merge(ssl: { alpn_protocols: %w[h2] }))
-        @pool.init_connection(connection, @options)
-        # only explicity emit addresses if connection didn't pre-resolve, i.e. it's not an IP.
-        catch(:coalesced) do
-          @building_connection = false
-          emit_addresses(connection, @family, @uri_addresses) unless connection.addresses
-          connection
-        end
+      # TODO: leaks connection object into the pool
+      @resolver_connection ||= @current_session.find_connection(@uri, @current_selector,
+                                                                @options.merge(ssl: { alpn_protocols: %w[h2] })).tap do |conn|
+        emit_addresses(conn, @family, @uri_addresses) unless conn.addresses
       end
     end
 
     private
 
     def resolve(connection = @connections.first, hostname = nil)
-      return if @building_connection
       return unless connection
 
       hostname ||= @queries.key(connection)
 
       if hostname.nil?
-        hostname = connection.origin.host
-        log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
+        hostname = connection.peer.host
+        log do
+          "resolver #{FAMILY_TYPES[@record_type]}: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}"
+        end if connection.peer.non_ascii_hostname
 
         hostname = @resolver.generate_candidates(hostname).each do |name|
           @queries[name.to_s] = connection
@@ -97,7 +92,7 @@ module HTTPX
       else
         @queries[hostname] = connection
       end
-      log { "resolver: query #{FAMILY_TYPES[RECORD_TYPES[@family]]} for #{hostname}" }
+      log { "resolver #{FAMILY_TYPES[@record_type]}: query for #{hostname}" }
 
       begin
         request = build_request(hostname)
@@ -108,7 +103,7 @@ module HTTPX
         @connections << connection
       rescue ResolveError, Resolv::DNS::EncodeError => e
         reset_hostname(hostname)
-        emit_resolve_error(connection, connection.origin.host, e)
+        emit_resolve_error(connection, connection.peer.host, e)
       end
     end
 
@@ -117,7 +112,7 @@ module HTTPX
     rescue StandardError => e
       hostname = @requests.delete(request)
       connection = reset_hostname(hostname)
-      emit_resolve_error(connection, connection.origin.host, e)
+      emit_resolve_error(connection, connection.peer.host, e)
     else
       # @type var response: HTTPX::Response
       parse(request, response)
@@ -156,7 +151,7 @@ module HTTPX
       when :decode_error
         host = @requests.delete(request)
         connection = reset_hostname(host)
-        emit_resolve_error(connection, connection.origin.host, result)
+        emit_resolve_error(connection, connection.peer.host, result)
       end
     end
 
@@ -176,7 +171,7 @@ module HTTPX
               alias_address = answers[address["alias"]]
               if alias_address.nil?
                 reset_hostname(address["name"])
-                if catch(:coalesced) { early_resolve(connection, hostname: address["alias"]) }
+                if early_resolve(connection, hostname: address["alias"])
                   @connections.delete(connection)
                 else
                   resolve(connection, address["alias"])

data/lib/httpx/resolver/multi.rb

@@ -8,27 +8,45 @@ module HTTPX
     include Callbacks
     using ArrayExtensions::FilterMap
 
-    attr_reader :resolvers
+    attr_reader :resolvers, :options
 
     def initialize(resolver_type, options)
+      @current_selector = nil
+      @current_session = nil
       @options = options
       @resolver_options = @options.resolver_options
 
       @resolvers = options.ip_families.map do |ip_family|
         resolver = resolver_type.new(ip_family, options)
-        resolver.on(:resolve, &method(:on_resolver_connection))
-        resolver.on(:error, &method(:on_resolver_error))
-        resolver.on(:close) { on_resolver_close(resolver) }
+        resolver.multi = self
         resolver
       end
 
       @errors = Hash.new { |hs, k| hs[k] = [] }
     end
 
+    def current_selector=(s)
+      @current_selector = s
+      @resolvers.each { |r| r.__send__(__method__, s) }
+    end
+
+    def current_session=(s)
+      @current_session = s
+      @resolvers.each { |r| r.__send__(__method__, s) }
+    end
+
     def closed?
       @resolvers.all?(&:closed?)
     end
 
+    def empty?
+      @resolvers.all?(&:empty?)
+    end
+
+    def inflight?
+      @resolvers.any(&:inflight?)
+    end
+
     def timeout
       @resolvers.filter_map(&:timeout).min
     end
@@ -42,10 +60,11 @@ module HTTPX
     end
 
     def early_resolve(connection)
-      hostname = connection.origin.host
+      hostname = connection.peer.host
       addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
-      return unless addresses
+      return false unless addresses
 
+      resolved = false
       addresses.group_by(&:family).sort { |(f1, _), (f2, _)| f2 <=> f1 }.each do |family, addrs|
         # try to match the resolver by family. However, there are cases where that's not possible, as when
         # the system does not have IPv6 connectivity, but it does support IPv6 via loopback/link-local.
@@ -55,21 +74,20 @@ module HTTPX
 
         # it does not matter which resolver it is, as early-resolve code is shared.
         resolver.emit_addresses(connection, family, addrs, true)
-      end
-    end
 
-    private
+        resolved = true
+      end
 
-    def on_resolver_connection(connection)
-      emit(:resolve, connection)
+      resolved
     end
 
-    def on_resolver_error(connection, error)
-      emit(:error, connection, error)
-    end
+    def lazy_resolve(connection)
+      @resolvers.each do |resolver|
+        resolver << @current_session.try_clone_connection(connection, @current_selector, resolver.family)
+        next if resolver.empty?
 
-    def on_resolver_close(resolver)
-      emit(:close, resolver)
+        @current_session.select_resolver(resolver, @current_selector)
+      end
     end
   end
 end