httpx 1.2.6 → 1.4.4

data/lib/httpx/request.rb

@@ -11,8 +11,10 @@ module HTTPX
     include Callbacks
     using URIExtensions
 
+    ALLOWED_URI_SCHEMES = %w[https http].freeze
+
     # default value used for "user-agent" header, when not overridden.
-    USER_AGENT = "httpx.rb/#{VERSION}"
+    USER_AGENT = "httpx.rb/#{VERSION}".freeze # rubocop:disable Style/RedundantFreeze
 
     # the upcased string HTTP verb for this request.
     attr_reader :verb
@@ -43,16 +45,52 @@ module HTTPX
 
     attr_writer :persistent
 
+    attr_reader :active_timeouts
+
     # will be +true+ when request body has been completely flushed.
     def_delegator :@body, :empty?
 
-    # initializes the instance with the given +verb+, an absolute or relative +uri+, and the
-    # request options.
-    def initialize(verb, uri, options = {})
+    # closes the body
+    def_delegator :@body, :close
+
+    # initializes the instance with the given +verb+ (an upppercase String, ex. 'GEt'),
+    # an absolute or relative +uri+ (either as String or URI::HTTP object), the
+    # request +options+ (instance of HTTPX::Options) and an optional Hash of +params+.
+    #
+    # Besides any of the options documented in HTTPX::Options (which would override or merge with what
+    # +options+ sets), it accepts also the following:
+    #
+    # :params :: hash or array of key-values which will be encoded and set in the query string of request uris.
+    # :body :: to be encoded in the request body payload. can be a String, an IO object (i.e. a File), or an Enumerable.
+    # :form :: hash of array of key-values which will be form-urlencoded- or multipart-encoded in requests body payload.
+    # :json :: hash of array of key-values which will be JSON-encoded in requests body payload.
+    # :xml :: Nokogiri XML nodes which will be encoded in requests body payload.
+    #
+    # :body, :form, :json and :xml are all mutually exclusive, i.e. only one of them gets picked up.
+    def initialize(verb, uri, options, params = EMPTY_HASH)
       @verb    = verb.to_s.upcase
-      @options = Options.new(options)
       @uri     = Utils.to_uri(uri)
-      if @uri.relative?
+
+      @headers = options.headers.dup
+      merge_headers(params.delete(:headers)) if params.key?(:headers)
+
+      @headers["user-agent"] ||= USER_AGENT
+      @headers["accept"]     ||= "*/*"
+
+      # forego compression in the Range request case
+      if @headers.key?("range")
+        @headers.delete("accept-encoding")
+      else
+        @headers["accept-encoding"] ||= options.supported_compression_formats
+      end
+
+      @query_params = params.delete(:params) if params.key?(:params)
+
+      @body = options.request_body_class.new(@headers, options, **params)
+
+      @options = @body.options
+
+      if @uri.relative? || @uri.host.nil?
         origin = @options.origin
         raise(Error, "invalid URI: #{@uri}") unless origin
 
@@ -61,28 +99,37 @@ module HTTPX
         @uri = origin.merge("#{base_path}#{@uri}")
       end
 
-      @headers = @options.headers.dup
-      @headers["user-agent"] ||= USER_AGENT
-      @headers["accept"]     ||= "*/*"
+      raise UnsupportedSchemeError, "#{@uri}: #{@uri.scheme}: unsupported URI scheme" unless ALLOWED_URI_SCHEMES.include?(@uri.scheme)
 
-      @body = @options.request_body_class.new(@headers, @options)
       @state = :idle
       @response = nil
       @peer_address = nil
+      @ping = false
       @persistent = @options.persistent
+      @active_timeouts = []
+    end
+
+    # whether request has been buffered with a ping
+    def ping?
+      @ping
+    end
+
+    # marks the request as having been buffered with a ping
+    def ping!
+      @ping = true
     end
 
-    # the read timeout defied for this requet.
+    # the read timeout defined for this request.
     def read_timeout
       @options.timeout[:read_timeout]
     end
 
-    # the write timeout defied for this requet.
+    # the write timeout defined for this request.
     def write_timeout
       @options.timeout[:write_timeout]
     end
 
-    # the request timeout defied for this requet.
+    # the request timeout defined for this request.
     def request_timeout
       @options.timeout[:request_timeout]
     end
@@ -91,10 +138,12 @@ module HTTPX
       @persistent
     end
 
+    # if the request contains trailer headers
     def trailers?
       defined?(@trailers)
     end
 
+    # returns an instance of HTTPX::Headers containing the trailer headers
     def trailers
       @trailers ||= @options.headers_class.new
     end
@@ -106,6 +155,7 @@ module HTTPX
       :w
     end
 
+    # merges +h+ into the instance of HTTPX::Headers of the request.
     def merge_headers(h)
       @headers = @headers.merge(h)
     end
@@ -172,7 +222,7 @@ module HTTPX
       return @query if defined?(@query)
 
       query = []
-      if (q = @options.params)
+      if (q = @query_params)
         query << Transcoder::Form.encode(q)
       end
       query << @uri.query if @uri.query
@@ -210,10 +260,13 @@ module HTTPX
       case nextstate
       when :idle
         @body.rewind
+        @ping = false
         @response = nil
         @drainer = nil
+        @active_timeouts.clear
       when :headers
         return unless @state == :idle
+
       when :body
         return unless @state == :headers ||
                       @state == :expect
@@ -234,6 +287,7 @@ module HTTPX
         return unless @state == :body
       when :done
         return if @state == :expect
+
       end
       @state = nextstate
       emit(@state, self)
@@ -244,6 +298,15 @@ module HTTPX
     def expects?
       @headers["expect"] == "100-continue" && @informational_status == 100 && !@response
     end
+
+    def set_timeout_callback(event, &callback)
+      clb = once(event, &callback)
+
+      # reset timeout callbacks when requests get rerouted to a different connection
+      once(:idle) do
+        callbacks(event).delete(clb)
+      end
+    end
   end
 end
 

data/lib/httpx/resolver/https.rb

@@ -27,7 +27,7 @@ module HTTPX
       use_get: false,
     }.freeze
 
-    def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close, :terminate
+    def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close, :terminate, :inflight?
 
     def initialize(_, options)
       super
@@ -43,7 +43,7 @@ module HTTPX
     end
 
     def <<(connection)
-      return if @uri.origin == connection.origin.to_s
+      return if @uri.origin == connection.peer.to_s
 
       @uri_addresses ||= HTTPX::Resolver.nolookup_resolve(@uri.host) || @resolver.getaddresses(@uri.host)
 
@@ -66,28 +66,25 @@ module HTTPX
     end
 
     def resolver_connection
-      @resolver_connection ||= @pool.find_connection(@uri, @options) || begin
-        @building_connection = true
-        connection = @options.connection_class.new(@uri, @options.merge(ssl: { alpn_protocols: %w[h2] }))
-        @pool.init_connection(connection, @options)
-        # only explicity emit addresses if connection didn't pre-resolve, i.e. it's not an IP.
-        emit_addresses(connection, @family, @uri_addresses) unless connection.addresses
-        @building_connection = false
-        connection
+      # TODO: leaks connection object into the pool
+      @resolver_connection ||= @current_session.find_connection(@uri, @current_selector,
+                                                                @options.merge(ssl: { alpn_protocols: %w[h2] })).tap do |conn|
+        emit_addresses(conn, @family, @uri_addresses) unless conn.addresses
       end
     end
 
     private
 
     def resolve(connection = @connections.first, hostname = nil)
-      return if @building_connection
       return unless connection
 
       hostname ||= @queries.key(connection)
 
       if hostname.nil?
-        hostname = connection.origin.host
-        log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
+        hostname = connection.peer.host
+        log do
+          "resolver #{FAMILY_TYPES[@record_type]}: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}"
+        end if connection.peer.non_ascii_hostname
 
         hostname = @resolver.generate_candidates(hostname).each do |name|
           @queries[name.to_s] = connection
@@ -95,7 +92,7 @@ module HTTPX
       else
         @queries[hostname] = connection
       end
-      log { "resolver: query #{FAMILY_TYPES[RECORD_TYPES[@family]]} for #{hostname}" }
+      log { "resolver #{FAMILY_TYPES[@record_type]}: query for #{hostname}" }
 
       begin
         request = build_request(hostname)
@@ -106,7 +103,7 @@ module HTTPX
         @connections << connection
       rescue ResolveError, Resolv::DNS::EncodeError => e
         reset_hostname(hostname)
-        emit_resolve_error(connection, connection.origin.host, e)
+        emit_resolve_error(connection, connection.peer.host, e)
       end
     end
 
@@ -115,7 +112,7 @@ module HTTPX
     rescue StandardError => e
       hostname = @requests.delete(request)
       connection = reset_hostname(hostname)
-      emit_resolve_error(connection, connection.origin.host, e)
+      emit_resolve_error(connection, connection.peer.host, e)
     else
       # @type var response: HTTPX::Response
       parse(request, response)
@@ -154,7 +151,7 @@ module HTTPX
       when :decode_error
         host = @requests.delete(request)
         connection = reset_hostname(host)
-        emit_resolve_error(connection, connection.origin.host, result)
+        emit_resolve_error(connection, connection.peer.host, result)
       end
     end
 
@@ -174,7 +171,7 @@ module HTTPX
               alias_address = answers[address["alias"]]
               if alias_address.nil?
                 reset_hostname(address["name"])
-                if catch(:coalesced) { early_resolve(connection, hostname: address["alias"]) }
+                if early_resolve(connection, hostname: address["alias"])
                   @connections.delete(connection)
                 else
                   resolve(connection, address["alias"])
@@ -199,7 +196,7 @@ module HTTPX
           @queries.delete_if { |_, conn| connection == conn }
 
           Resolver.cached_lookup_set(hostname, @family, addresses) if @resolver_options[:cache]
-          emit_addresses(connection, @family, addresses.map { |addr| addr["data"] })
+          catch(:coalesced) { emit_addresses(connection, @family, addresses.map { |addr| addr["data"] }) }
         end
       end
       return if @connections.empty?
@@ -219,7 +216,7 @@ module HTTPX
         uri.query = URI.encode_www_form(params)
         request = rklass.new("GET", uri, @options)
       else
-        request = rklass.new("POST", uri, @options.merge(body: [payload]))
+        request = rklass.new("POST", uri, @options, body: [payload])
         request.headers["content-type"] = "application/dns-message"
       end
       request.headers["accept"] = "application/dns-message"

data/lib/httpx/resolver/multi.rb

@@ -8,27 +8,45 @@ module HTTPX
     include Callbacks
     using ArrayExtensions::FilterMap
 
-    attr_reader :resolvers
+    attr_reader :resolvers, :options
 
     def initialize(resolver_type, options)
+      @current_selector = nil
+      @current_session = nil
       @options = options
       @resolver_options = @options.resolver_options
 
       @resolvers = options.ip_families.map do |ip_family|
         resolver = resolver_type.new(ip_family, options)
-        resolver.on(:resolve, &method(:on_resolver_connection))
-        resolver.on(:error, &method(:on_resolver_error))
-        resolver.on(:close) { on_resolver_close(resolver) }
+        resolver.multi = self
         resolver
       end
 
       @errors = Hash.new { |hs, k| hs[k] = [] }
     end
 
+    def current_selector=(s)
+      @current_selector = s
+      @resolvers.each { |r| r.__send__(__method__, s) }
+    end
+
+    def current_session=(s)
+      @current_session = s
+      @resolvers.each { |r| r.__send__(__method__, s) }
+    end
+
     def closed?
       @resolvers.all?(&:closed?)
     end
 
+    def empty?
+      @resolvers.all?(&:empty?)
+    end
+
+    def inflight?
+      @resolvers.any(&:inflight?)
+    end
+
     def timeout
       @resolvers.filter_map(&:timeout).min
     end
@@ -42,10 +60,11 @@ module HTTPX
     end
 
     def early_resolve(connection)
-      hostname = connection.origin.host
+      hostname = connection.peer.host
       addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
-      return unless addresses
+      return false unless addresses
 
+      resolved = false
       addresses.group_by(&:family).sort { |(f1, _), (f2, _)| f2 <=> f1 }.each do |family, addrs|
         # try to match the resolver by family. However, there are cases where that's not possible, as when
         # the system does not have IPv6 connectivity, but it does support IPv6 via loopback/link-local.
@@ -55,21 +74,20 @@ module HTTPX
 
         # it does not matter which resolver it is, as early-resolve code is shared.
         resolver.emit_addresses(connection, family, addrs, true)
-      end
-    end
 
-    private
+        resolved = true
+      end
 
-    def on_resolver_connection(connection)
-      emit(:resolve, connection)
+      resolved
     end
 
-    def on_resolver_error(connection, error)
-      emit(:error, connection, error)
-    end
+    def lazy_resolve(connection)
+      @resolvers.each do |resolver|
+        resolver << @current_session.try_clone_connection(connection, @current_selector, resolver.family)
+        next if resolver.empty?
 
-    def on_resolver_close(resolver)
-      emit(:close, resolver)
+        @current_session.select_resolver(resolver, @current_selector)
+      end
     end
   end
 end