httpx 1.7.8 → 1.8.1

data/lib/httpx/plugins/stream.rb

@@ -46,8 +46,8 @@ module HTTPX
       end
     end
 
-    def each_line
-      return enum_for(__method__) unless block_given?
+    def each_line(&block)
+      return enum_for(__method__) unless block
 
       line = "".b
 
@@ -55,7 +55,11 @@ module HTTPX
         line << chunk
 
         while (idx = line.index("\n"))
-          yield line.byteslice(0..(idx - 1))
+          if idx.zero?
+            yield ""
+          else
+            yield line.byteslice(0..(idx - 1))
+          end
 
           line = line.byteslice((idx + 1)..-1)
         end
@@ -71,11 +75,11 @@ module HTTPX
       @on_chunk.call(chunk)
     end
 
-    # :nocov:
+    # simplecov:disable
     def inspect
       "#<#{self.class}:#{object_id}>"
     end
-    # :nocov:
+    # simplecov:enable
 
     def to_s
       if @request.response

data/lib/httpx/plugins/stream_bidi.rb

@@ -177,6 +177,8 @@ module HTTPX
           @pipe_read.readpartial(1)
         end
 
+        def initial_call; end
+
         def interests
           return if @closed
 

data/lib/httpx/plugins/tracing.rb

@@ -111,10 +111,11 @@ module HTTPX::Plugins
         @init_time = ::Time.now.utc
       end
 
-      def send(request)
-        # request init time is only the same as the connection init time
-        # if the connection is going through the connection handshake.
-        request.init_time ||= @init_time unless open?
+      def send_request_to_parser(request)
+        if connecting?
+          # request span timeframe should include the time it took to connect.
+          request.init_time ||= @init_time
+        end
 
         super
       end
@@ -126,6 +127,16 @@ module HTTPX::Plugins
         # the connection is back to :idle, and ready to connect again.
         @init_time = ::Time.now.utc
       end
+
+      private
+
+      def ping(request)
+        # if a connection is probed for liveness, the request timeframe should include
+        # it too.
+        request.init_time ||= ::Time.now.utc
+
+        super
+      end
     end
   end
   register_plugin :tracing, Tracing

data/lib/httpx/pool.rb

@@ -173,7 +173,7 @@ module HTTPX
       end
     end
 
-    # :nocov:
+    # simplecov:disable
     def inspect
       "#<#{self.class}:#{object_id} " \
         "@max_connections=#{@max_connections} " \
@@ -181,7 +181,7 @@ module HTTPX
         "@pool_timeout=#{@pool_timeout} " \
         "@connections=#{@connections.size}>"
     end
-    # :nocov:
+    # simplecov:enable
 
     private
 

data/lib/httpx/request/body.rb

@@ -114,12 +114,12 @@ module HTTPX
       @headers.add("transfer-encoding", "chunked")
     end
 
-    # :nocov:
+    # simplecov:disable
     def inspect
       "#<#{self.class}:#{object_id} " \
         "#{unbounded_body? ? "stream" : "@bytesize=#{bytesize}"}>"
     end
-    # :nocov:
+    # simplecov:enable
 
     class << self
       def initialize_body(params)

data/lib/httpx/request.rb

@@ -39,6 +39,10 @@ module HTTPX
     # Exception raised during enumerable body writes.
     attr_reader :drain_error
 
+    # when this request is sent via HTTP/2, it'll use this hash of options to set the priority of the
+    # respective HTTP/2 frame.
+    attr_reader :http2_stream_options
+
     # The IP address from the peer server.
     attr_accessor :peer_address
 
@@ -70,6 +74,7 @@ module HTTPX
     # :form :: hash of array of key-values which will be form-urlencoded- or multipart-encoded in requests body payload.
     # :json :: hash of array of key-values which will be JSON-encoded in requests body payload.
     # :xml :: Nokogiri XML nodes which will be encoded in requests body payload.
+    # :http2_stream_options :: hash of options to be used to set the HTTP/2 priority by sending an initial PRIORITY frame.
     #
     # :body, :form, :json and :xml are all mutually exclusive, i.e. only one of them gets picked up.
     def initialize(verb, uri, options, params = EMPTY_HASH)
@@ -81,6 +86,8 @@ module HTTPX
 
       @query_params = params.delete(:params) if params.key?(:params)
 
+      @http2_stream_options = params.key?(:http2_stream_options) ? params.delete(:http2_stream_options) : EMPTY_HASH
+
       @body = options.request_body_class.new(@headers, options, **params)
 
       @options = @body.options
@@ -100,7 +107,7 @@ module HTTPX
       @connection = @response =
         @drainer = @peer_address =
           @informational_status = @on_response_arrived = nil
-      @ping = false
+      @ping = @started = false
       @persistent = @options.persistent
       @active_timeouts = []
     end
@@ -115,6 +122,7 @@ module HTTPX
 
     def complete!(response = @response)
       emit(:complete, response)
+      reset_timers(true)
     end
 
     # whether request has been buffered with a ping
@@ -142,6 +150,11 @@ module HTTPX
       @options.timeout[:request_timeout]
     end
 
+    # the total request timeout defined for this request.
+    def total_request_timeout
+      @options.timeout[:total_request_timeout]
+    end
+
     def persistent?
       @persistent
     end
@@ -167,6 +180,10 @@ module HTTPX
       @state != :done
     end
 
+    def started?
+      @started
+    end
+
     # merges +h+ into the instance of HTTPX::Headers of the request.
     def merge_headers(h)
       @headers = @headers.merge(h)
@@ -252,21 +269,19 @@ module HTTPX
 
     # consumes and returns the next available chunk of request body that can be sent
     def drain_body
-      return nil if @body.nil?
+      return if @body.nil?
 
       @drainer ||= @body.each
-      chunk = @drainer.next.dup
-
-      emit(:body_chunk, chunk)
-      chunk
+      @drainer.next.dup
     rescue StopIteration
       nil
     rescue StandardError => e
+      # in case an error occurs while emitting body chunks
       @drain_error = e
       nil
     end
 
-    # :nocov:
+    # simplecov:disable
     def inspect
       "#<#{self.class}:#{object_id} " \
         "#{@verb} " \
@@ -274,7 +289,7 @@ module HTTPX
         "@headers=#{@headers} " \
         "@body=#{@body}>"
     end
-    # :nocov:
+    # simplecov:enable
 
     # moves on to the +nextstate+ of the request state machine (when all preconditions are met)
     def transition(nextstate)
@@ -283,10 +298,14 @@ module HTTPX
         @body.rewind
         @ping = false
         @response = @drainer = nil
-        @active_timeouts.clear
+
+        # request may be sent to a different connection and will be
+        # reassigned a new set of timers.
+        reset_timers(false)
       when :headers
         return unless @state == :idle
 
+        @started = true
       when :body
         return unless @state == :headers ||
                       @state == :expect
@@ -309,7 +328,7 @@ module HTTPX
         return if @state == :expect
 
       end
-      log(level: 3) { "#{@state}] -> #{nextstate}" }
+      log(level: 3) { "#{@state} -> #{nextstate}" }
       @state = nextstate
       emit(@state, self)
       nil
@@ -346,6 +365,19 @@ module HTTPX
 
       @on_response_arrived.call
     end
+
+    private
+
+    def reset_timers(reset_total_request_timers)
+      timers = @active_timeouts
+
+      until (timer = timers.shift).nil?
+        next if !reset_total_request_timers && timer.label == :total_request_timeout
+
+        # cancel active timers.
+        timer.cancel
+      end
+    end
   end
 end
 

data/lib/httpx/resolver/cache/base.rb

@@ -106,14 +106,7 @@ module HTTPX
           name = entry["name"]
           next unless name != hostname
 
-          lookups[name] ||= []
-
-          case family
-          when Socket::AF_INET6
-            lookups[name] << entry
-          when Socket::AF_INET
-            lookups[name].unshift(entry)
-          end
+          _set(name, family, [entry], lookups, hostnames)
         end
       end
 

data/lib/httpx/resolver/cache/file.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "pstore"
+require "tmpdir"
+
+module HTTPX
+  module Resolver::Cache
+    # Implementation of a file resolver cache.
+    class File < Base
+      # default path where the resolver cache is stored. It's versioned, as the file may
+      # change format in-between releases, and it'd signal it as corrupted.
+      DEFAULT_PATH = ::File.join(Dir.tmpdir, "httpx-ruby-#{VERSION}.cache")
+
+      def initialize(path = DEFAULT_PATH)
+        super()
+        @store = PStore.new(path, true)
+      end
+
+      def get(hostname)
+        now = Utils.now
+        @store.transaction do
+          lookups = @store[:lookups] || EMPTY_HASH
+          hostnames = @store[:hostnames] || EMPTY
+
+          _get(hostname, lookups, hostnames, now)
+        end
+      end
+
+      def set(hostname, family, entries)
+        @store.transaction do
+          lookups = @store[:lookups] || {}
+          hostnames = @store[:hostnames] || []
+
+          _set(hostname, family, entries, lookups, hostnames)
+
+          @store[:lookups] = lookups
+          @store[:hostnames] = hostnames
+        end
+      end
+
+      def evict(hostname, ip)
+        ip = ip.to_s
+
+        @store.transaction do
+          lookups = @store[:lookups] || EMPTY_HASH
+          hostnames = @store[:hostnames] || EMPTY
+
+          _evict(hostname, ip, lookups, hostnames)
+
+          @store[:lookups] = lookups
+          @store[:hostnames] = hostnames
+        end
+      end
+    end
+  end
+end

data/lib/httpx/resolver/https.rb

@@ -126,27 +126,27 @@ module HTTPX
     end
 
     def on_response(request, response)
-      response.raise_for_status
-    rescue StandardError => e
-      hostname = @requests.delete(request)
-      connection = reset_hostname(hostname)
-      emit_resolve_error(connection, connection.peer.host, e)
-      close_or_resolve
-    else
-      # @type var response: HTTPX::Response
-      if response.status.between?(300, 399) && response.headers.key?("location")
-        hostname = @requests[request]
-        connection = @queries[hostname]
-        location_uri = URI(response.headers["location"])
-        location_uri = response.uri.merge(location_uri) if location_uri.relative?
-
-        # we assume that the DNS server URI changed permanently and move on
-        @uri = location_uri
-        send_request(hostname, connection)
-        return
-      end
+      if (e = response.error)
+        hostname = @requests.delete(request)
+        connection = reset_hostname(hostname)
+        emit_resolve_error(connection, connection.peer.host, e)
+        close_or_resolve
+      else
+        # @type var response: HTTPX::Response
+        if response.status.between?(300, 399) && response.headers.key?("location")
+          hostname = @requests[request]
+          connection = @queries[hostname]
+          location_uri = URI(response.headers["location"])
+          location_uri = response.uri.merge(location_uri) if location_uri.relative?
 
-      parse(request, response)
+          # we assume that the DNS server URI changed permanently and move on
+          @uri = location_uri
+          send_request(hostname, connection)
+          return
+        end
+
+        parse(request, response)
+      end
     ensure
       @requests.delete(request)
     end

data/lib/httpx/resolver/native.rb

@@ -75,6 +75,13 @@ module HTTPX
 
     def call
       case @state
+      when :idle, :closed
+        return if @connections.empty?
+
+        transition(:idle) if @state == :closed
+        transition(:open)
+
+        consume if @state == :open
       when :open
         consume
       end
@@ -512,12 +519,16 @@ module HTTPX
       transition(:open)
     end
 
+    # moves the resolver state machine to the +nextstate+ state (if all conditions are met)-
     def transition(nextstate)
       case nextstate
       when :idle
-        if @io
-          @io.close
+        if (io = @io)
           @io = nil
+          io.close
+
+          # @fiber-switch-guard
+          return if @io
         end
       when :open
         return unless @state == :idle
@@ -531,7 +542,14 @@ module HTTPX
       when :closed
         return if @state == :closed
 
-        @io.close if @io
+        if (io = @io)
+          @io = nil
+          io.close
+
+          # @fiber-switch-guard
+          return if @io
+        end
+
         @start_timeout = nil
         @write_buffer.clear
         @read_buffer.clear

data/lib/httpx/resolver/resolver.rb

@@ -52,6 +52,10 @@ module HTTPX
 
     alias_method :terminate, :close
 
+    def initial_call
+      call
+    end
+
     def force_close(*args)
       while (connection = @connections.shift)
         connection.force_close(*args)

data/lib/httpx/resolver/system.rb

@@ -23,6 +23,9 @@ module HTTPX
     DONE = 1
     ERROR = 2
 
+    class AddrinfoTimeoutError < StandardError
+    end
+
     class << self
       def multi?
         false
@@ -227,7 +230,7 @@ module HTTPX
         begin
           addrs = if resolve_timeout
 
-            Timeout.timeout(resolve_timeout) do
+            Timeout.timeout(resolve_timeout, AddrinfoTimeoutError) do
               __addrinfo_resolve(hostname, scheme)
             end
           else
@@ -246,7 +249,7 @@ module HTTPX
             end
           end
         rescue StandardError => e
-          if e.is_a?(Timeout::Error)
+          if e.is_a?(AddrinfoTimeoutError)
             timeouts.shift
             retry unless timeouts.empty?
             e = ResolveTimeoutError.new(resolve_timeout, e.message)

data/lib/httpx/response/body.rb

@@ -20,10 +20,10 @@ module HTTPX
       @headers = response.headers
       @options = options
       @window_size = options.window_size
+      @max_response_body_size = options.max_response_body_size
       @encodings = []
       @length = 0
-      @buffer = nil
-      @reader = nil
+      @buffer = @reader = nil
       @state = :idle
 
       # initialize response encoding
@@ -59,6 +59,8 @@ module HTTPX
 
       chunk = decode_chunk(chunk)
 
+      raise Error, "maximum response body size exceeded" if @max_response_body_size < @length
+
       transition(:open)
       @buffer.write(chunk)
 
@@ -159,13 +161,13 @@ module HTTPX
                end
     end
 
-    # :nocov:
+    # simplecov:disable
     def inspect
       "#<#{self.class}:#{object_id} " \
         "@state=#{@state} " \
         "@length=#{@length}>"
     end
-    # :nocov:
+    # simplecov:enable
 
     # rewinds the response payload buffer.
     def rewind

data/lib/httpx/response.rb

@@ -68,7 +68,7 @@ module HTTPX
       @headers = @options.headers_class.new(headers)
       @body = @options.response_body_class.new(self, @options)
       @finished = complete?
-      @content_type = nil
+      @content_type = @content_length = nil
     end
 
     # dupped initialization
@@ -88,6 +88,7 @@ module HTTPX
     # merges headers defined in +h+ into the response headers.
     def merge_headers(h)
       @headers = @headers.merge(h)
+      @content_type = @content_length = nil
     end
 
     # writes +data+ chunk into the response body.
@@ -103,6 +104,13 @@ module HTTPX
       @content_type ||= ContentType.new(@headers["content-type"])
     end
 
+    # returns the response content length as advertised in the HTTP Content-Length header value.
+    def content_length
+      return @content_length if defined?(@content_length)
+
+      @content_length = @headers["content-length"]&.to_i
+    end
+
     # returns whether the response has been fully fetched.
     def finished?
       @finished
@@ -133,7 +141,7 @@ module HTTPX
       bodyless? || (@request.verb == "CONNECT" && @status == 200)
     end
 
-    # :nocov:
+    # simplecov:disable
     def inspect
       "#<#{self.class}:#{object_id} " \
         "HTTP/#{version} " \
@@ -141,7 +149,7 @@ module HTTPX
         "@headers=#{@headers} " \
         "@body=#{@body.bytesize}>"
     end
-    # :nocov:
+    # simplecov:enable
 
     # returns an instance of HTTPX::HTTPError if the response has a 4xx or 5xx
     # status code, or nothing.
@@ -259,7 +267,6 @@ module HTTPX
   #   response = HTTPX.get("https://some-domain/path") #=> response is HTTPX::Response or HTTPX::ErrorResponse
   #   response.raise_for_status #=> raises if it wraps an error
   class ErrorResponse
-    include Loggable
     extend Forwardable
 
     # the corresponding HTTPX::Request instance.
@@ -282,7 +289,7 @@ module HTTPX
       @response = request.response if request.response.is_a?(Response)
       @error = error
       @options = request.options
-      log_exception(@error)
+      @request.log_exception(@error)
       finish!
     end
 

data/lib/httpx/selector.rb

@@ -41,6 +41,11 @@ module HTTPX
       @selectables.empty? && @timers.empty?
     end
 
+    # first time the registered selectables are added, there's probably work to do.
+    def initial_call
+      @selectables.each(&:initial_call)
+    end
+
     def next_tick
       catch(:jump_tick) do
         timeout = next_timeout
@@ -134,7 +139,13 @@ module HTTPX
       # the connections to be reaped (such as the total timeout error) before #select
       # gets called.
       if @selectables.empty?
-        sleep(interval) if interval
+        begin
+          sleep(interval)
+        rescue IOError
+          # @fiber-switch-guard
+          # in a fiber scheduler scenario, IOs may be closed by the scheduler and raised in a separate fiber
+          # on wakeup, which includes a sleep call.
+        end if interval
         return
       end
 

data/lib/httpx/session.rb

@@ -319,9 +319,14 @@ module HTTPX
 
       waiting = false
 
-      responses = requests.each_with_index.map do |request, idx|
+      requests.each do |request|
         send_request(request, selector)
+      end
 
+      # do work first
+      selector.initial_call
+
+      responses = requests.each_with_index.map do |request, idx|
         fetch_response(request, selector, request.options).tap do |response|
           if response.nil?
             pending += 1

data/lib/httpx/session_extensions.rb

@@ -20,11 +20,11 @@ module HTTPX
     $VERBOSE = original_verbosity
   end
 
-  # :nocov:
+  # simplecov:disable
   if Session.default_options.debug_level > 2
     proxy_session = plugin(:internal_telemetry)
     remove_const(:Session)
     const_set(:Session, proxy_session.class)
   end
-  # :nocov:
+  # simplecov:enable
 end

data/lib/httpx/timers.rb

@@ -62,6 +62,9 @@ module HTTPX
     end
 
     class Timer
+      # simpler helper which allows classification
+      attr_accessor :label
+
       def initialize(interval, callback)
         @interval = interval
         @callback = callback

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "1.7.8"
+  VERSION = "1.8.1"
 end

data/sig/chainable.rbs

@@ -16,6 +16,7 @@ module HTTPX
               | (:basic_auth, ?options) -> Plugins::sessionBasicAuth
               | (:digest_auth, ?options) -> Plugins::sessionDigestAuth
               | (:ntlm_auth, ?options) -> Plugins::sessionNTLMAuth
+              | (:ntlm_v2_auth, ?options) -> Plugins::sessionNTLMV2Auth
               | (:aws_sdk_authentication, ?options) -> Plugins::sessionAwsSdkAuthentication
               | (:brotli, ?options) -> Session
               | (:cookies, ?options) -> Plugins::sessionCookies
@@ -32,8 +33,10 @@ module HTTPX
               | (:rate_limiter, ?options) -> Plugins::sessionRateLimiter
               | (:stream, ?options) -> Plugins::sessionStream
               | (:stream_bidi, ?options) -> Plugins::sessionStreamBidi
+              | (:server_sent_events, ?options) -> Plugins::sessionServerSentEvents
               | (:aws_sigv4, ?options) -> Plugins::awsSigV4Session
               | (:grpc, ?options) -> Plugins::grpcSession
+              | (:cache, ?options) -> Plugins::sessionCache
               | (:response_cache, ?options) -> Plugins::sessionResponseCache
               | (:circuit_breaker, ?options) -> Plugins::sessionCircuitBreaker
               | (:oauth, ?options) -> Plugins::sessionOAuth

data/sig/connection/http1.rbs

@@ -58,7 +58,7 @@ module HTTPX
 
     def waiting_for_ping?: () -> bool
 
-    def timeout: () -> Numeric?
+    def timeout: () -> interval?
 
     private