httpx 1.8.4 → 1.9.1

data/lib/httpx/plugins/tracing.rb

@@ -63,13 +63,21 @@ module HTTPX::Plugins
     end
 
     module RequestMethods
+      # time at which a request started being buffered for sending.
       attr_accessor :init_time
 
+      # when not nil, it means this request was sent to a non-open connection, and contains how much time it took
+      # the connection to connect.
+      attr_accessor :handshake_time
+
+      # time at which the request was first handed to a connection, or last moved back to waiting for one.
+      attr_accessor :enqueue_time
+
       # intercepts request initialization to inject the tracing logic.
       def initialize(*)
         super
 
-        @init_time = nil
+        @init_time = @handshake_time = @enqueue_time = nil
 
         tracer = @options.tracer
 
@@ -79,12 +87,15 @@ module HTTPX::Plugins
           tracer.reset(self)
 
           # request is reset when it's retried.
-          @init_time = nil
+          @init_time = @handshake_time = nil
+
+          # going back to idle means waiting to be sent again.
+          @enqueue_time = ::Time.now.utc
         end
         on(:headers) do
           # the usual request init time (when not including the connection handshake)
           # should be the time the request is buffered the first time.
-          @init_time ||= ::Time.now.utc
+          @init_time = calculate_init_time
 
           tracer.start(self)
         end
@@ -92,15 +103,31 @@ module HTTPX::Plugins
       end
 
       def response=(*)
-        # init_time should be set when it's send to a connection.
-        # However, there are situations where connection initialization fails.
+        # There are situations where connection initialization fails.
         # Example is the :ssrf_filter plugin, which raises an error on
         # initialize if the host is an IP which matches against the known set.
         # in such cases, we'll just set here right here.
-        @init_time ||= ::Time.now.utc
+        @init_time ||= calculate_init_time
 
         super
       end
+
+      private
+
+      def calculate_init_time
+        init_time = ::Time.now.utc
+
+        return init_time unless (handshake_time = @handshake_time)
+
+        init_time -= handshake_time
+
+        enqueue_time = @enqueue_time
+
+        # handshake time is measured for the connection, and may happen before the request was sent to it.
+        return init_time unless enqueue_time && init_time < enqueue_time
+
+        enqueue_time
+      end
     end
 
     # Connection mixin
@@ -111,12 +138,20 @@ module HTTPX::Plugins
         @init_time = nil
       end
 
-      def send_request_to_parser(request)
-        if connecting?
-          @init_time ||= ::Time.now.utc
+      def send(request)
+        # requests may be rerouted to other connections (i.e. when coalescing), so only the first
+        # connection the request is handed to marks it.
+        request.enqueue_time ||= ::Time.now.utc
+
+        super
+      end
 
-          # request span timeframe should include the time it took to connect.
-          request.init_time ||= @init_time
+      def send_request_to_parser(request, *)
+        if connecting? && @init_time
+          # at this point, the connection has connected, but hasn't yet transitioned to :open,
+          # so it's the time the requests which caused this connection to connect, are finally being
+          # sent to the parser, so handshake time should factor into the span time.
+          request.handshake_time ||= ::Time.now.utc - @init_time
         end
 
         super
@@ -135,8 +170,37 @@ module HTTPX::Plugins
         @init_time = nil
       end
 
+      def disconnect
+        return if @exhausted
+
+        return unless @current_session && @current_selector
+
+        # reset init time of connections abandoned while connecting.
+        @init_time = nil if connecting?
+
+        super
+      end
+
       private
 
+      def handle_transition(*)
+        super
+
+        # no longer relevant after connection established.
+        @init_time = nil unless connecting?
+      end
+
+      def handle_error(er, request = nil)
+        if connecting? && @init_time
+          # bookkeep handshake time back into pending requests, which were never initialized with it.
+          handshake_time = ::Time.now.utc - @init_time
+          request.handshake_time ||= handshake_time if request
+          @pending.each { |req| req.handshake_time ||= handshake_time }
+        end
+
+        super
+      end
+
       def connect
         @init_time ||= ::Time.now.utc
 
@@ -151,6 +215,30 @@ module HTTPX::Plugins
         super
       end
     end
+
+    module HTTP2Methods
+      def initialize(*)
+        super
+
+        @handshake_init_time = ::Time.now.utc
+      end
+
+      private
+
+      def on_settings(*)
+        unless @handshake_completed
+          # pending requests must wait for the handshake completion
+          handshake_time = ::Time.now.utc - @handshake_init_time
+
+          @pending.each do |request|
+            request.handshake_time ||= 0.0
+            request.handshake_time += handshake_time
+          end
+        end
+
+        super
+      end
+    end
   end
   register_plugin :tracing, Tracing
 end

data/lib/httpx/plugins/upgrade/h2.rb

@@ -25,7 +25,7 @@ module HTTPX
         def interests
           return super unless connecting? && @parser
 
-          connect
+          try_connect
 
           return @io.interests if connecting?
 
@@ -33,9 +33,9 @@ module HTTPX
         end
 
         def upgrade_to_h2
-          enqueue_pending_requests_from_parser(@parser)
+          @parser.reset_requests
 
-          @parser = @options.http2_class.new(@write_buffer, @options)
+          @parser = @options.http2_class.new(@write_buffer, @pending, @options)
           set_parser_callbacks(@parser)
           @upgrade_protocol = "h2"
 

data/lib/httpx/plugins/upgrade.rb

@@ -30,6 +30,8 @@ module HTTPX
       end
 
       module InstanceMethods
+        private
+
         def fetch_response(request, selector, options)
           response = super
 

data/lib/httpx/plugins/webdav.rb

@@ -13,14 +13,18 @@ module HTTPX
       end
 
       module InstanceMethods
+        # sends a COPY WebDav request for +src+ and +dest+.
         def copy(src, dest)
           request("COPY", src, headers: { "destination" => @options.origin.merge(dest) })
         end
 
+        # sends a MOVE WebDav request for +src+ and +dest+.
         def move(src, dest)
           request("MOVE", src, headers: { "destination" => @options.origin.merge(dest) })
         end
 
+        # sends a LOCK WebDav request for +path+ (for the time set by +timeout+, when passed).
+        # it calls #unlock after the block is executed.
         def lock(path, timeout: nil, &blk)
           headers = {}
           headers["timeout"] = if timeout && timeout.positive?
@@ -51,14 +55,17 @@ module HTTPX
           response
         end
 
+        # sends a UNLOCK WebDav request for +path+, using the given +lock_token+.
         def unlock(path, lock_token)
           request("UNLOCK", path, headers: { "lock-token" => lock_token })
         end
 
+        # sends a MKCOL WebDav request for +dir+.
         def mkcol(dir)
           request("MKCOL", dir)
         end
 
+        # sends a PROPFIND WebDav request for +path+.
         def propfind(path, xml = nil)
           body = case xml
                  when :acl
@@ -73,6 +80,7 @@ module HTTPX
           request("PROPFIND", path, headers: { "depth" => "1" }, xml: body)
         end
 
+        # sends a PROPPATCH WebDav request for +path+.
         def proppatch(path, xml)
           body = "<?xml version=\"1.0\"?>" \
                  "<D:propertyupdate xmlns:D=\"DAV:\" xmlns:Z=\"http://ns.example.com/standards/z39.50/\">#{xml}</D:propertyupdate>"

data/lib/httpx/request/body.rb

@@ -32,6 +32,7 @@ module HTTPX
       @headers = h
       @body = self.class.initialize_body(params)
       @options = options.merge(params)
+      @unbounded_body = nil
 
       if @body
         if @options.compress_request_body && @headers.key?("content-encoding")
@@ -102,7 +103,7 @@ module HTTPX
 
     # returns whether the body yields infinitely.
     def unbounded_body?
-      return @unbounded_body if defined?(@unbounded_body)
+      return @unbounded_body unless @unbounded_body.nil?
 
       @unbounded_body = !@body.nil? && (chunked? || @body.bytesize == Float::INFINITY)
     end

data/lib/httpx/request.rb

@@ -140,22 +140,22 @@ module HTTPX
 
     # the read timeout defined for this request.
     def read_timeout
-      @options.timeout[:read_timeout]
+      @options.timeout.read_timeout
     end
 
     # the write timeout defined for this request.
     def write_timeout
-      @options.timeout[:write_timeout]
+      @options.timeout.write_timeout
     end
 
     # the request timeout defined for this request.
     def request_timeout
-      @options.timeout[:request_timeout]
+      @options.timeout.request_timeout
     end
 
     # the total request timeout defined for this request.
     def total_request_timeout
-      @options.timeout[:total_request_timeout]
+      @options.timeout.total_request_timeout
     end
 
     def persistent?
@@ -180,7 +180,7 @@ module HTTPX
     end
 
     def can_buffer?
-      @state != :done
+      @state != :done && @state != :expect
     end
 
     def started?
@@ -339,7 +339,7 @@ module HTTPX
 
     # whether the request supports the 100-continue handshake and already processed the 100 response.
     def expects?
-      @headers["expect"] == "100-continue" && @informational_status == 100 && !@response
+      @headers.key?("expect") && @headers.get("expect").include?("100-continue") && @informational_status == 100 && !@response
     end
 
     def set_timeout_callback(event, &callback)

data/lib/httpx/resolver/https.rb

@@ -32,7 +32,8 @@ module HTTPX
     }.freeze
 
     def_delegators :@resolver_connection, :connecting?, :to_io, :call, :close,
-                   :closed?, :deactivate, :terminate, :inflight?, :handle_socket_timeout
+                   :closed?, :deactivate, :terminate, :inflight?, :handle_socket_timeout,
+                   :closed_for_select?
 
     def initialize(_, options)
       super
@@ -49,7 +50,7 @@ module HTTPX
     end
 
     def state
-      @resolver_connection ? @resolver_connection.state : :idle
+      @resolver_connection&.state || :idle
     end
 
     def <<(connection)

data/lib/httpx/resolver/native.rb

@@ -38,12 +38,11 @@ module HTTPX
       @search = Array(@resolver_options[:search]).map { |srch| srch.scan(/[^.]+/) }
       @_timeouts = Array(@resolver_options[:timeouts])
       @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
-      @name = nil
+      @name = @message_id = @timer = nil
       @queries = {}
       @read_buffer = "".b
       @write_buffer = Buffer.new(@resolver_options[:packet_size])
       @state = :idle
-      @timer = nil
     end
 
     def close
@@ -52,7 +51,7 @@ module HTTPX
 
     def force_close(*)
       @timer.cancel if @timer
-      @timer = @name = nil
+      @timer = @name = @message_id = nil
       @queries.clear
       @timeouts.clear
       close
@@ -87,6 +86,10 @@ module HTTPX
       end
     end
 
+    def closed_for_select?
+      @state == :closed
+    end
+
     def interests
       case @state
       when :idle
@@ -170,6 +173,7 @@ module HTTPX
       if nameserver && @ns_index < nameserver.size
         log { "resolver #{FAMILY_TYPES[@record_type]}: failed resolving on nameserver #{@nameserver[@ns_index - 1]} (#{e.message})" }
         transition(:idle)
+        transition(:open)
         @timeouts.clear
         retry
       else
@@ -195,7 +199,7 @@ module HTTPX
       @timer = @current_selector.after(timeout) do
         next unless @connections.include?(connection)
 
-        @timer = @name = nil
+        @timer = @name = @message_id = nil
 
         do_retry(h, connection, timeout)
       end
@@ -310,7 +314,13 @@ module HTTPX
     end
 
     def parse(buffer)
-      code, result = Resolver.decode_dns_answer(buffer)
+      code, result, message_id = Resolver.decode_dns_answer(buffer)
+
+      # discard late message answers for no longer inflight queries.
+      if message_id && message_id != @message_id
+        log { "resolver #{FAMILY_TYPES[@record_type]}: discarding answer for a query no longer in flight" }
+        return
+      end
 
       case code
       when :ok
@@ -392,7 +402,7 @@ module HTTPX
           orig_name = name
           # absolute name
           name_labels = Resolv::DNS::Name.create(name).to_a
-          name = @queries.each_key.first { |hname| name_labels == Resolv::DNS::Name.create(hname).to_a }
+          name = @queries.each_key.find { |hname| name_labels == Resolv::DNS::Name.create(hname).to_a }
 
           # probably a retried query for which there's an answer
           unless name
@@ -462,11 +472,10 @@ module HTTPX
         @queries[hostname] = connection
       end
 
-      @name = hostname
-
       log { "resolver #{FAMILY_TYPES[@record_type]}: query for #{hostname}" }
       begin
         @write_buffer << encode_dns_query(hostname)
+        @name = hostname
       rescue Resolv::DNS::EncodeError => e
         reset_hostname(hostname, connection: connection)
         @connections.delete(connection)
@@ -479,6 +488,7 @@ module HTTPX
       message_id = Resolver.generate_id
       msg = Resolver.encode_dns_query(hostname, type: @record_type, message_id: message_id)
       msg[0, 2] = [msg.size, message_id].pack("nn") if @socket_type == :tcp
+      @message_id = message_id
       msg
     end
 
@@ -574,7 +584,7 @@ module HTTPX
     def reset_query
       @timer.cancel
 
-      @timer = @name = nil
+      @timer = @name = @message_id = nil
     end
 
     def reset_hostname(hostname, connection: @queries.delete(hostname), reset_candidates: true)

data/lib/httpx/resolver/resolver.rb

@@ -29,9 +29,9 @@ module HTTPX
 
     attr_reader :family, :options
 
-    attr_writer :current_selector, :current_session
+    attr_writer :current_session
 
-    attr_accessor :multi
+    attr_accessor :current_selector, :multi
 
     def initialize(family, options)
       @family = family

data/lib/httpx/resolver/system.rb

@@ -90,6 +90,10 @@ module HTTPX
       nil
     end
 
+    def closed_for_select?
+      @state == :closed
+    end
+
     def interests
       return if @queries.empty?
 

data/lib/httpx/resolver.rb

@@ -51,16 +51,20 @@ module HTTPX
       end
 
       # no domain was found
-      return :no_domain_found if message.rcode == Resolv::DNS::RCode::NXDomain
+      return :no_domain_found, nil, message.id if message.rcode == Resolv::DNS::RCode::NXDomain
 
-      return :message_truncated if message.tc == 1
+      return :message_truncated, nil, message.id if message.tc == 1
 
       if message.rcode != Resolv::DNS::RCode::NoError
         case message.rcode
-        when Resolv::DNS::RCode::ServFail
-          return :retriable_error, message.rcode
+        when Resolv::DNS::RCode::ServFail,
+             # getaddrinfo also retries these errors, as they're not about the hostname being resolved,
+             # but the DNS resolver state.
+             Resolv::DNS::RCode::NotImp,
+             Resolv::DNS::RCode::Refused
+          return :retriable_error, message.rcode, message.id
         else
-          return :dns_error, message.rcode
+          return :dns_error, message.rcode, message.id
         end
       end
 
@@ -85,7 +89,7 @@ module HTTPX
         end
       end
 
-      [:ok, addresses]
+      [:ok, addresses, message.id]
     end
 
     private

data/lib/httpx/response.rb

@@ -119,7 +119,6 @@ module HTTPX
     # marks the response as finished, freezes the headers.
     def finish!
       @finished = true
-      @headers.freeze
       @request.connection = nil
     end
 
@@ -215,8 +214,7 @@ module HTTPX
 
   # Helper class which decodes the HTTP "content-type" header.
   class ContentType
-    MIME_TYPE_RE = %r{^([^/]+/[^;]+)(?:$|;)}.freeze
-    CHARSET_RE   = /;\s*charset=([^;]+)/i.freeze
+    SPLIT_PATTERN = /[;,]/.freeze
 
     def initialize(header_value)
       @header_value = header_value
@@ -250,11 +248,25 @@ module HTTPX
     private
 
     def load
-      m = @header_value.to_s[MIME_TYPE_RE, 1]
-      m && @mime_type = m.strip.downcase
+      if @header_value && !@header_value.empty?
+        m, c = @header_value.split(SPLIT_PATTERN, 2)
 
-      c = @header_value.to_s[CHARSET_RE, 1]
-      c && @charset = c.strip.delete('"')
+        if m
+          @mime_type = m.strip
+          @mime_type.downcase
+        end
+
+        if c
+          c.strip!
+          c.delete_prefix!("charset=")
+          c.gsub!(/\A"(.*?)(?<!\\)"\z/) do
+            c2 = ::Regexp.last_match(1)
+            c2.gsub!(/\\(.)/, '\1')
+            c2
+          end
+          @charset = c
+        end
+      end
 
       @initialized = true
     end