httpx 1.8.4 → 1.9.1

data/lib/httpx/headers.rb

@@ -159,8 +159,8 @@ module HTTPX
     # This method is more internal, and for this reason doesn't try
     # to "correct" the user input, i.e. it doesn't downcase the key.
     #
-    def get(field)
-      @headers[field] || EMPTY
+    def get(downcased_key)
+      @headers[downcased_key] || EMPTY
     end
 
     private

data/lib/httpx/io/tcp.rb

@@ -17,7 +17,7 @@ module HTTPX
       @keep_open = false
       @addresses = []
       @ip_index = -1
-      @ip = nil
+      @ip = @attempt_started_at = nil
       @hostname = origin.host
       @options = options
       @fallback_protocol = @options.fallback_protocol
@@ -80,6 +80,27 @@ module HTTPX
       @addresses.any?
     end
 
+    # time left until the attempt to connect to the current address should be abandoned.
+    def connect_attempt_time_left
+      return unless @state == :idle && @ip_index.positive? && (attempt_started_at = @attempt_started_at)
+
+      timeouts = @options.timeout
+      timeout = timeouts.connect_timeout || timeouts.request_timeout || timeouts.total_request_timeout
+
+      return unless timeout && timeout.finite?
+
+      timeout.fdiv(@addresses.size) - Utils.elapsed_time(attempt_started_at)
+    end
+
+    # abandons the connection attempt to the current address.
+    def next_address!(e)
+      @io&.close
+      @ip_index -= 1
+      raise e if @ip_index.negative?
+
+      log { "failed connecting to #{@ip} (#{e.message}), trying next..." }
+    end
+
     def to_io
       @io.to_io
     end
@@ -109,13 +130,8 @@ module HTTPX
       try_connect
     rescue Errno::EHOSTUNREACH,
            Errno::ENETUNREACH => e
-      @ip_index -= 1
-
-      raise e if @ip_index.negative?
-
-      log { "failed connecting to #{@ip} (#{e.message}), evict from cache and trying next..." }
+      next_address!(e)
       @options.resolver_cache.evict(@hostname, @ip)
-
       @io = build_socket
       retry
     rescue Errno::ECONNREFUSED,
@@ -132,7 +148,7 @@ module HTTPX
     rescue Errno::ETIMEDOUT => e
       @ip_index -= 1
 
-      raise ConnectTimeoutError.new(@options.timeout[:connect_timeout], e.message) if @ip_index.negative?
+      raise ConnectTimeoutError.new(@options.timeout.connect_timeout, e.message) if @ip_index.negative?
 
       log { "failed connecting to #{@ip} (#{e.message}), trying next..." }
 
@@ -159,26 +175,29 @@ module HTTPX
     private :try_connect
 
     def read(size, buffer)
-      ret = @io.read_nonblock(size, buffer, exception: false)
-      if ret == :wait_readable
+      siz = @io.read_nonblock(size, buffer, exception: false)
+
+      case siz
+      when String
+        log { "READ: #{buffer.bytesize} bytes..." }
+        buffer.bytesize
+      when :wait_readable, :wait_writable
         buffer.clear
-        return 0
+        0
       end
-      return if ret.nil?
-
-      log { "READ: #{buffer.bytesize} bytes..." }
-      buffer.bytesize
     end
 
     def write(buffer)
       siz = @io.write_nonblock(buffer, exception: false)
-      return 0 if siz == :wait_writable
-      return if siz.nil?
 
-      log { "WRITE: #{siz} bytes..." }
-
-      buffer.shift!(siz)
-      siz
+      case siz
+      when Integer
+        log { "WRITE: #{siz} bytes..." }
+        buffer.shift!(siz)
+        siz
+      when :wait_readable, :wait_writable
+        0
+      end
     end
 
     def close
@@ -234,6 +253,7 @@ module HTTPX
 
     def build_socket
       @ip = @addresses[@ip_index]
+      @attempt_started_at = Utils.now
       Socket.new(@ip.family, :STREAM, 0)
     end
 
@@ -255,6 +275,7 @@ module HTTPX
 
     def log_transition_state(nextstate)
       label = host
+      label = "#{label}:#{@port}" if label
       label = "#{label}(##{@io.fileno})" if nextstate == :connected
       "#{label} #{@state} -> #{nextstate}"
     end

data/lib/httpx/options.rb

@@ -12,12 +12,11 @@ module HTTPX
     SETTINGS_TIMEOUT = 10
     CLOSE_HANDSHAKE_TIMEOUT = 10
     CONNECT_TIMEOUT = READ_TIMEOUT = WRITE_TIMEOUT = 60
-    REQUEST_TIMEOUT = OPERATION_TIMEOUT = TOTAL_REQUEST_TIMEOUT = nil
     RESOLVER_TYPES = %i[memory file].freeze
     # default value used for "user-agent" header, when not overridden.
     USER_AGENT = "httpx.rb/#{VERSION}".freeze # rubocop:disable Style/RedundantFreeze
 
-    @options_names = []
+    @options_names = Set[]
 
     class << self
       attr_reader :options_names
@@ -199,14 +198,14 @@ module HTTPX
         # here, which is to make option objects shareable across ractors,
         # and in most cases debug should be nil, or one of the objects
         # which will eventually be shareable, like STDOUT or STDERR.
-        next if ivar == :debug
+        next if %i[debug io].include?(ivar)
 
-        instance_variable_get(:"@#{ivar}").freeze
+        send(ivar).freeze
       end
       super
     end
 
-    REQUEST_BODY_IVARS = %i[@headers].freeze
+    REQUEST_BODY_IVARS = %i[headers].freeze
 
     # checks whether +other+ matches the same connection-level options
     def connection_options_match?(other, ignore_ivars = nil)
@@ -214,20 +213,18 @@ module HTTPX
 
       # headers and other request options do not play a role, as they are
       # relevant only for the request.
-      ivars = instance_variables
-      ivars.reject! { |iv| REQUEST_BODY_IVARS.include?(iv) }
-      ivars.reject! { |iv| ignore_ivars.include?(iv) } if ignore_ivars
+      ivars = self.class.options_names.reject { |iv| REQUEST_BODY_IVARS.include?(iv) }
+      ivars = ivars.reject { |iv| ignore_ivars.include?(iv) } if ignore_ivars
 
-      other_ivars = other.instance_variables
-      other_ivars.reject! { |iv| REQUEST_BODY_IVARS.include?(iv) }
-      other_ivars.reject! { |iv| ignore_ivars.include?(iv) } if ignore_ivars
+      other_ivars = other.class.options_names.reject { |iv| REQUEST_BODY_IVARS.include?(iv) }
+      other_ivars = other_ivars.reject { |iv| ignore_ivars.include?(iv) } if ignore_ivars
 
       return false if ivars.size != other_ivars.size
 
-      return false if ivars.sort != other_ivars.sort
+      return false if ivars != other_ivars
 
       ivars.all? do |ivar|
-        instance_variable_get(ivar) == other.instance_variable_get(ivar)
+        send(ivar) == other.send(ivar)
       end
     end
 
@@ -296,6 +293,7 @@ module HTTPX
         opts.instance_variable_set(ivar, v)
       end
 
+      opts.freeze
       opts
     end
 
@@ -455,20 +453,22 @@ module HTTPX
     end
 
     def option_timeout(value)
+      return value if value.is_a?(Timeouts)
+
       timeout_hash = Hash[value]
 
-      default_timeouts = DEFAULT_OPTIONS[:timeout]
+      default_timeouts = Timeouts.members
 
       # Validate keys and values
       timeout_hash.each do |key, val|
-        raise TypeError, "invalid timeout: `:#{key}`" unless default_timeouts.key?(key)
+        raise TypeError, "invalid timeout: `:#{key}`" unless default_timeouts.include?(key)
 
         next if val.nil?
 
         raise TypeError, "`:#{key}` must be numeric" unless val.is_a?(Numeric)
       end
 
-      timeout_hash
+      Timeouts.new(**timeout_hash)
     end
 
     def option_supported_compression_formats(value)
@@ -483,7 +483,14 @@ module HTTPX
     end
 
     def option_addresses(value)
-      Array(value).map { |entry| Resolver::Entry.convert(entry) }
+      case value
+      when Hash
+        value.transform_values do |value|
+          Array(value).map { |entry| Resolver::Entry.convert(entry) }
+        end
+      else
+        Array(value).map { |entry| Resolver::Entry.convert(entry) }
+      end
     end
 
     def option_ip_families(value)
@@ -539,12 +546,67 @@ module HTTPX
       end
     end
 
-    def access_option(obj, k, ivar_map)
-      case obj
-      when Hash
-        obj[ivar_map[k]]
-      else
-        obj.instance_variable_get(k)
+    class Timeouts < Struct.new(
+      :connect_timeout, :settings_timeout,
+      :read_timeout, :write_timeout,
+      :request_timeout, :total_request_timeout,
+      :operation_timeout, :keep_alive_timeout, :ping_timeout, :close_handshake_timeout,
+      keyword_init: true
+    )
+      def initialize(
+        connect_timeout: nil,
+        settings_timeout: nil,
+        read_timeout: nil,
+        write_timeout: nil,
+        request_timeout: nil,
+        total_request_timeout: nil,
+        operation_timeout: nil,
+        close_handshake_timeout: nil,
+        keep_alive_timeout: nil,
+        ping_timeout: nil
+      )
+        super
+        freeze
+      end
+
+      def connect_timeout
+        self[:connect_timeout] || (CONNECT_TIMEOUT unless deadline_timeouts?)
+      end
+
+      def settings_timeout
+        self[:settings_timeout] || (SETTINGS_TIMEOUT unless deadline_timeouts?)
+      end
+
+      def read_timeout
+        self[:read_timeout] || (READ_TIMEOUT unless deadline_timeouts?)
+      end
+
+      def write_timeout
+        self[:write_timeout] || (WRITE_TIMEOUT unless deadline_timeouts?)
+      end
+
+      def close_handshake_timeout
+        self[:close_handshake_timeout] || CLOSE_HANDSHAKE_TIMEOUT
+      end
+
+      def keep_alive_timeout
+        self[:keep_alive_timeout] || KEEP_ALIVE_TIMEOUT
+      end
+
+      def ping_timeout
+        self[:ping_timeout] || PING_TIMEOUT
+      end
+
+      def merge(other)
+        other_h = other.to_h
+        other_h.compact!
+        Timeouts.new(**to_h, **other_h)
+      end
+
+      private
+
+      def deadline_timeouts?
+        request_timeout || total_request_timeout || operation_timeout
       end
     end
 
@@ -572,18 +634,7 @@ module HTTPX
       :max_response_headers => 1000,
       :max_response_header_value_size => nil,
       :max_response_body_size => Float::INFINITY,
-      :timeout => {
-        connect_timeout: CONNECT_TIMEOUT,
-        settings_timeout: SETTINGS_TIMEOUT,
-        close_handshake_timeout: CLOSE_HANDSHAKE_TIMEOUT,
-        operation_timeout: OPERATION_TIMEOUT,
-        keep_alive_timeout: KEEP_ALIVE_TIMEOUT,
-        ping_timeout: PING_TIMEOUT,
-        read_timeout: READ_TIMEOUT,
-        write_timeout: WRITE_TIMEOUT,
-        request_timeout: REQUEST_TIMEOUT,
-        total_request_timeout: TOTAL_REQUEST_TIMEOUT,
-      }.freeze,
+      :timeout => Timeouts.new,
       :headers_class => Class.new(Headers, &SET_TEMPORARY_NAME),
       :headers => EMPTY_HASH,
       :window_size => WINDOW_SIZE,

data/lib/httpx/parser/http1.rb

@@ -68,7 +68,7 @@ module HTTPX
         version, code, _ = m.captures
         raise(Error, "unsupported HTTP version (HTTP/#{version})") unless version && VERSIONS.include?(version)
 
-        @http_version = version.split(".").map(&:to_i)
+        @http_version = version
         @status_code = code.to_i
         raise(Error, "wrong status code (#{@status_code})") unless (100..599).cover?(@status_code)
 
@@ -78,9 +78,7 @@ module HTTPX
 
       def parse_headers
         headers = @headers
-        buffer = @buffer
-
-        #: @type var buffer: String
+        buffer = @buffer #: String
 
         while (idx = buffer.index("\n"))
           # @type var line: String
@@ -114,8 +112,7 @@ module HTTPX
           key = line.byteslice(0..(separator_index - 1))
 
           key.rstrip! # was lstripped previously!
-          # @type var value: String
-          value = line.byteslice((separator_index + 1)..-1)
+          value = line.byteslice((separator_index + 1)..-1) #: String
           value.strip!
           raise Error, "wrong header format" if value.nil?
 

data/lib/httpx/plugins/auth/digest.rb

@@ -14,7 +14,6 @@ module HTTPX
         def initialize(user, password, hashed: false, **)
           @user = user
           @password = password
-          @nonce = 0
           @hashed = hashed
         end
 
@@ -137,7 +136,7 @@ module HTTPX
         end
 
         def next_nonce
-          @nonce += 1
+          SecureRandom.random_number(2**32)
         end
 
         def raise_format_error

data/lib/httpx/plugins/auth.rb

@@ -69,14 +69,18 @@ module HTTPX
           @skip_auth_header_value = false
         end
 
+        # returns a new session where requests will be authorized using +token+ (or what the +blk+ yields),
+        # using +auth_header_type+ as header value prefix.
         def authorization(token = nil, auth_header_type: nil, &blk)
           with(auth_header_type: auth_header_type, auth_header_value: token || blk)
         end
 
+        # returns a new session which call #authorize while setting <tt>"Bearer"</tt> as the +auth_header_type+.
         def bearer_auth(token = nil, &blk)
           authorization(token, auth_header_type: "Bearer", &blk)
         end
 
+        # yields a block where authorization (if available) will be bypassed.
         def skip_auth_header
           @skip_auth_header_value = true
           yield
@@ -84,6 +88,7 @@ module HTTPX
           @skip_auth_header_value = false
         end
 
+        # resets authorization information.
         def reset_auth_header_value!
           @auth_header_value_mtx.synchronize do
             @auth_header_value = @auth_header_expires_at = nil
@@ -186,11 +191,15 @@ module HTTPX
             auth_error?(response, options) || super
           end
 
-          def prepare_to_retry(request, response)
+          def prepare_to_retry(request, response, retry_after)
             super
 
             return unless auth_error?(response, request.options) ||
-                          (@options.generate_auth_value_on_retry && @options.generate_auth_value_on_retry.call(response))
+                          (@options.generate_auth_value_on_retry && @options.generate_auth_value_on_retry.call(response)) ||
+                          (
+                            (expires_at = @auth_header_expires_at) &&
+                             expires_at < (Time.now.utc.to_i + (retry_after || 0))
+                          )
 
             # regenerate token before retry, but only if it's the first request from batch failing.
             # otherwise, it means that the first request already passed here, so this request should

data/lib/httpx/plugins/aws_sdk_authentication.rb

@@ -85,9 +85,13 @@ module HTTPX
 
       module InstanceMethods
         #
-        # aws_authentication
-        # aws_authentication(credentials: Aws::Credentials.new('akid', 'secret'))
-        # aws_authentication()
+        # returns a session which will use the AWS credentials provided by your AWS system configuration.
+        #
+        #     session = HTTPX.plugin(:aws_sdk_authentication)
+        #     auth_session = session.aws_authentication
+        #     # OR
+        #     auth_session = session.aws_authentication(credentials: Aws::Credentials.new('akid', 'secret'))
+        #     auth_session.get("http://...")
         #
         def aws_sdk_authentication(
           credentials: AwsSdkAuthentication.credentials(@options.aws_profile),

data/lib/httpx/plugins/aws_sigv4.rb

@@ -156,7 +156,6 @@ module HTTPX
 
       class << self
         def load_dependencies(*)
-          require "set"
           require "digest/sha2"
           require "cgi/escape"
         end
@@ -178,6 +177,7 @@ module HTTPX
       end
 
       module InstanceMethods
+        # returns a session which will authenticate requests using the AWS Sigv4 protocol.
         def aws_sigv4_authentication(**options)
           with(sigv4_signer: Signer.new(**options))
         end

data/lib/httpx/plugins/basic_auth.rb

@@ -19,6 +19,8 @@ module HTTPX
       end
 
       module InstanceMethods
+        # returns a new session which will authorize requests using HTTP Basic Auth, with
+        # +user+ and +password+ as credentials.
         def basic_auth(user, password)
           authorization(Authentication::Basic.new(user, password).authenticate)
         end

data/lib/httpx/plugins/cache.rb

@@ -106,7 +106,7 @@ module HTTPX
         def fetch_response(request, *)
           response = super
 
-          return unless response
+          return unless response && response.is_a?(Response)
 
           if cacheable_request?(request) && cacheable_response?(request, response) && !response.cached?
             log { "caching response for #{request.uri}..." }

data/lib/httpx/plugins/callbacks.rb

@@ -25,6 +25,7 @@ module HTTPX
 
         CALLBACKS.each do |meth|
           class_eval(<<-MOD, __FILE__, __LINE__ + 1)
+            # sets a <tt>:#{meth}</tt> callback on the session
             def on_#{meth}(&blk)   # def on_connection_opened(&blk)
               on(:#{meth}, &blk)   #   on(:connection_opened, &blk)
               self                 #   self
@@ -33,6 +34,7 @@ module HTTPX
         end
 
         def plugin(*args, &blk)
+          # ruby 2.7 requires explicit named splat
           super(*args).tap do |sess|
             CALLBACKS.each do |cb|
               next unless callbacks_for?(cb)
@@ -44,6 +46,12 @@ module HTTPX
           end
         end
 
+        def close(*)
+          super
+        rescue CallbackError => e
+          raise e.cause
+        end
+
         private
 
         def branch(options, &blk)
@@ -114,12 +122,6 @@ module HTTPX
         rescue CallbackError => e
           raise e.cause
         end
-
-        def close(*)
-          super
-        rescue CallbackError => e
-          raise e.cause
-        end
       end
 
       module RequestMethods

data/lib/httpx/plugins/circuit_breaker.rb

@@ -34,6 +34,7 @@ module HTTPX
 
         %i[circuit_open].each do |meth|
           class_eval(<<-MOD, __FILE__, __LINE__ + 1)
+        # sets a <tt>:#{meth}</tt> callback on the session.
         def on_#{meth}(&blk)   # def on_circuit_open(&blk)
           on(:#{meth}, &blk)   #   on(:circuit_open, &blk)
           self                 #   self

data/lib/httpx/plugins/content_digest.rb

@@ -81,15 +81,13 @@ module HTTPX
         end
 
         def write(chunk)
-          @content_digest_buffer.write(chunk) if @content_digest_buffer
+          @content_digest_buffer&.write(chunk)
           super
         end
 
         def close
-          if @content_digest_buffer
-            @content_digest_buffer.close
-            @content_digest_buffer = nil
-          end
+          @content_digest_buffer&.close
+          @content_digest_buffer = nil
           super
         end
       end

data/lib/httpx/plugins/digest_auth.rb

@@ -33,6 +33,8 @@ module HTTPX
       end
 
       module InstanceMethods
+        # returns a session which will authenticate requests using HTTP Digest Auth, with
+        # +user+ and +password+ as credentials.
         def digest_auth(user, password, hashed: false)
           with(digest: Authentication::Digest.new(user, password, hashed: hashed))
         end

data/lib/httpx/plugins/expect.rb

@@ -104,9 +104,15 @@ module HTTPX
       end
 
       module ConnectionMethods
-        def send_request_to_parser(request)
+        private
+
+        def set_request_timeouts(request)
           super
 
+          set_request_expect_timeout(request)
+        end
+
+        def set_request_expect_timeout(request)
           return unless request.headers["expect"] == "100-continue"
 
           expect_timeout = request.options.expect_timeout
@@ -118,6 +124,7 @@ module HTTPX
             if request.state == :expect && !request.expects?
               Expect.no_expect_store.add(request.origin)
               request.headers.delete("expect")
+              request.transition(:body)
               consume
             end
           end
@@ -125,6 +132,8 @@ module HTTPX
       end
 
       module InstanceMethods
+        private
+
         def fetch_response(request, selector, options)
           response = super