httpx 0.24.3 → 0.24.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 145a6e293b5ae6bfc1008765a755ab823d490edf7c50bdbf1de01194e3e1537b
-  data.tar.gz: 42d235a66fdb050a993b305d3aefed91893d26253cc62864ff2d0d088167ef0a
+  metadata.gz: b3d9a59fa9d9cc1908f3135c38d9a8cb132312c0539795d3a341d87da3755d23
+  data.tar.gz: 8e88ca75fceac380b8fbb31fa627f3a30a2b938e2527baffbc2db00db64b76e5
 SHA512:
-  metadata.gz: 7b10d4d91debc907d3445526ab864357a1e39cddfbe8b628f104241ed9015ae340e122dd49535be21bda7b20b95ebc047db4c476b7ff412e310d685f835e62c3
-  data.tar.gz: 3d82a8bc8bc3d5e6f94c2dde6f330ab52a3a42e0cd95e4b62328e4b6b005aac9946d1903d384f0944112e997ca26801dc342d77b44290dddb8e88ec0523e1338
+  metadata.gz: fe80712332209f6b188eca10aea16f0d1cb02b8af18682e6efda3844cd5d5de9e99cb0829cf162433fe85f7a070865808caa32a69df3ebe84b7dd483c105315f
+  data.tar.gz: 1ec1dd55a8ec80bceaab32a39dfa2e325f01f83c5e512be5c74feb8f4049ef9f34431ab6d33d00ad4b6fcf906d7883a0420e91a0663f2eb4fdd6ddb632b4d351

data/README.md

@@ -126,7 +126,7 @@ The plugin system is similar to the ones used by [sequel](https://github.com/jer
 
 ### Advanced DNS features
 
-`HTTPX` ships with custom DNS resolver implementations, including a native Happy Eyeballs resolver immplementation, and a DNS-over-HTTPS resolver.
+`HTTPX` ships with custom DNS resolver implementations, including a native Happy Eyeballs resolver implementation, and a DNS-over-HTTPS resolver.
 
 ## User-driven test suite
 

data/doc/release_notes/0_24_4.md

@@ -0,0 +1,18 @@
+# 0.24.4
+
+## Improvements
+
+* `digest_authentication` plugin now supports passing HA1hashed with password HA1s (common to store in htdigest files for example) when setting the`:hashed` kwarg to `true` in the `.digest_auth` call.
+  * ex: `http.digest_auth(user, get_hashed_passwd_from_htdigest(user), hashed: true)`
+* TLS session resumption is now supported
+  * whenever possible, `httpx` sessions will recycle used connections so that, in the case of TLS connections, the first session will keep being reusedd, thereby diminishing the overhead of subsequent TLS handshakes on the same host.
+  * TLS sessions are only reused in the scope of the same `httpx` session, unless the `:persistent` plugin is used, in which case, the persisted `httpx` session will always try to resume TLS sessions.
+
+## Bugfixess
+
+* When explicitly using IP addresses in the URL host, TLS handshake will now verify tif he IP address is included in the certificate.
+  * IP address will keep not be used for SNI, as per RFC 6066, section 3.
+  * ex: `http.get("https://10.12.0.12/get")`
+  * if you want the prior behavior, set `HTTPX.with(ssl: {verify_hostname: faalse})`
+* Turn TLS hostname verification on for `jruby` (it's turned off by default).
+  * if you want the prior behavior, set `HTTPX.with(ssl: {verify_hostname: faalse})`

data/doc/release_notes/0_24_5.md

@@ -0,0 +1,6 @@
+# 0.24.5
+
+## Bugfixes
+
+* fix for SSL handshake post connection SAN check using IPv6 address.
+* fix bug in DoH impl when the request returned no answer.

data/lib/httpx/connection.rb

@@ -42,7 +42,7 @@ module HTTPX
 
     def_delegator :@write_buffer, :empty?
 
-    attr_reader :type, :io, :origin, :origins, :state, :pending, :options
+    attr_reader :type, :io, :origin, :origins, :state, :pending, :options, :ssl_session
 
     attr_writer :timers
 
@@ -106,6 +106,12 @@ module HTTPX
       ) || (match_altsvcs?(uri) && match_altsvc_options?(uri, options))
     end
 
+    def expired?
+      return false unless @io
+
+      @io.expired?
+    end
+
     def mergeable?(connection)
       return false if @state == :closing || @state == :closed || !@io
 
@@ -138,6 +144,12 @@ module HTTPX
 
     def merge(connection)
       @origins |= connection.instance_variable_get(:@origins)
+      if connection.ssl_session
+        @ssl_session = connection.ssl_session
+        @io.session_new_cb do |sess|
+          @ssl_session = sess
+        end if @io
+      end
       connection.purge_pending do |req|
         send(req)
       end
@@ -280,6 +292,17 @@ module HTTPX
       @options.timeout[:operation_timeout]
     end
 
+    def idling
+      purge_after_closed
+      @write_buffer.clear
+      transition(:idle)
+      @parser = nil if @parser
+    end
+
+    def used?
+      @connected_at
+    end
+
     def deactivate
       transition(:inactive)
     end
@@ -310,6 +333,9 @@ module HTTPX
       catch(:called) do
         epiped = false
         loop do
+          # connection may have
+          return if @state == :idle
+
           parser.consume
 
           # we exit if there's no more requests to process
@@ -551,6 +577,7 @@ module HTTPX
       when :idle
         @timeout = @current_timeout = @options.timeout[:connect_timeout]
 
+        @connected_at = nil
       when :open
         return if @state == :closed
 
@@ -594,14 +621,23 @@ module HTTPX
     end
 
     def build_socket(addrs = nil)
-      transport_type = case @type
-                       when "tcp" then TCP
-                       when "ssl" then SSL
-                       when "unix" then UNIX
-                       else
-                         raise Error, "unsupported transport (#{@type})"
+      case @type
+      when "tcp"
+        TCP.new(@origin, addrs, @options)
+      when "ssl"
+        SSL.new(@origin, addrs, @options) do |sock|
+          sock.ssl_session = @ssl_session
+          sock.session_new_cb do |sess|
+            @ssl_session = sess
+
+            sock.ssl_session = sess
+          end
+        end
+      when "unix"
+        UNIX.new(@origin, addrs, @options)
+      else
+        raise Error, "unsupported transport (#{@type})"
       end
-      transport_type.new(@origin, addrs, @options)
     end
 
     def on_error(error)

data/lib/httpx/io/ssl.rb

@@ -4,26 +4,52 @@ require "openssl"
 
 module HTTPX
   TLSError = OpenSSL::SSL::SSLError
-  IPRegex = Regexp.union(Resolv::IPv4::Regex, Resolv::IPv6::Regex)
 
   class SSL < TCP
     using RegexpExtensions unless Regexp.method_defined?(:match?)
 
     TLS_OPTIONS = if OpenSSL::SSL::SSLContext.instance_methods.include?(:alpn_protocols)
-      { alpn_protocols: %w[h2 http/1.1].freeze }.freeze
+      { alpn_protocols: %w[h2 http/1.1].freeze }
     else
-      {}.freeze
+      {}
     end
+    # https://github.com/jruby/jruby-openssl/issues/284
+    TLS_OPTIONS[:verify_hostname] = true if RUBY_ENGINE == "jruby"
+    TLS_OPTIONS.freeze
+
+    attr_writer :ssl_session
 
     def initialize(_, _, options)
       super
-      @ctx = OpenSSL::SSL::SSLContext.new
+
       ctx_options = TLS_OPTIONS.merge(options.ssl)
       @sni_hostname = ctx_options.delete(:hostname) || @hostname
-      @ctx.set_params(ctx_options) unless ctx_options.empty?
-      @state = :negotiated if @keep_open
 
-      @hostname_is_ip = IPRegex.match?(@sni_hostname)
+      if @keep_open && @io.is_a?(OpenSSL::SSL::SSLSocket)
+        # externally initiated ssl socket
+        @ctx = @io.context
+        @state = :negotiated
+      else
+        @ctx = OpenSSL::SSL::SSLContext.new
+        @ctx.set_params(ctx_options) unless ctx_options.empty?
+        unless @ctx.session_cache_mode.nil? # a dummy method on JRuby
+          @ctx.session_cache_mode =
+            OpenSSL::SSL::SSLContext::SESSION_CACHE_CLIENT | OpenSSL::SSL::SSLContext::SESSION_CACHE_NO_INTERNAL_STORE
+        end
+
+        yield(self) if block_given?
+      end
+
+      @verify_hostname = @ctx.verify_hostname
+    end
+
+    if OpenSSL::SSL::SSLContext.method_defined?(:session_new_cb=)
+      def session_new_cb(&pr)
+        @ctx.session_new_cb = proc { |_, sess| pr.call(sess) }
+      end
+    else
+      # session_new_cb not implemented under JRuby
+      def session_new_cb; end
     end
 
     def protocol
@@ -43,25 +69,35 @@ module HTTPX
       OpenSSL::SSL.verify_certificate_identity(@io.peer_cert, host)
     end
 
-    def close
-      super
-      # allow reconnections
-      # connect only works if initial @io is a socket
-      @io = @io.io if @io.respond_to?(:io)
-    end
-
     def connected?
       @state == :negotiated
     end
 
+    def expired?
+      super || ssl_session_expired?
+    end
+
+    def ssl_session_expired?
+      @ssl_session.nil? || Process.clock_gettime(Process::CLOCK_REALTIME) >= (@ssl_session.time.to_f + @ssl_session.timeout)
+    end
+
     def connect
       super
       return if @state == :negotiated ||
                 @state != :connected
 
       unless @io.is_a?(OpenSSL::SSL::SSLSocket)
+        if (hostname_is_ip = (@ip == @sni_hostname))
+          # IPv6 address would be "[::1]", must turn to "0000:0000:0000:0000:0000:0000:0000:0001" for cert SAN check
+          @sni_hostname = @ip.to_string
+          # IP addresses in SNI is not valid per RFC 6066, section 3.
+          @ctx.verify_hostname = false
+        end
+
         @io = OpenSSL::SSL::SSLSocket.new(@io, @ctx)
-        @io.hostname = @sni_hostname unless @hostname_is_ip
+
+        @io.hostname = @sni_hostname unless hostname_is_ip
+        @io.session = @ssl_session unless ssl_session_expired?
         @io.sync_close = true
       end
       try_ssl_connect
@@ -71,7 +107,7 @@ module HTTPX
       # :nocov:
       def try_ssl_connect
         @io.connect_nonblock
-        @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE && !@hostname_is_ip
+        @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE && @verify_hostname
         transition(:negotiated)
         @interests = :w
       rescue ::IO::WaitReadable
@@ -103,7 +139,7 @@ module HTTPX
           @interests = :w
           return
         end
-        @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE && !@hostname_is_ip
+        @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE && @verify_hostname
         transition(:negotiated)
         @interests = :w
       end
@@ -130,6 +166,7 @@ module HTTPX
       case nextstate
       when :negotiated
         return unless @state == :connected
+
       when :closed
         return unless @state == :negotiated ||
                       @state == :connected

data/lib/httpx/io/tcp.rb

@@ -192,6 +192,17 @@ module HTTPX
       @state == :idle || @state == :closed
     end
 
+    def expired?
+      # do not mess with external sockets
+      return false if @options.io
+
+      return true unless @addresses
+
+      resolver_addresses = Resolver.nolookup_resolve(@hostname)
+
+      (Array(resolver_addresses) & @addresses).empty?
+    end
+
     # :nocov:
     def inspect
       "#<#{self.class}: #{@ip}:#{@port} (state: #{@state})>"

data/lib/httpx/io/unix.rb

@@ -56,6 +56,10 @@ module HTTPX
            ::IO::WaitReadable
     end
 
+    def expired?
+      false
+    end
+
     # :nocov:
     def inspect
       "#<#{self.class}(path: #{@path}): (state: #{@state})>"

data/lib/httpx/plugins/authentication/digest.rb

@@ -10,10 +10,11 @@ module HTTPX
       class Digest
         using RegexpExtensions unless Regexp.method_defined?(:match?)
 
-        def initialize(user, password, **)
+        def initialize(user, password, hashed: false, **)
           @user = user
           @password = password
           @nonce = 0
+          @hashed = hashed
         end
 
         def can_authenticate?(authenticate)
@@ -55,11 +56,13 @@ module HTTPX
           end
 
           a1 = if sess
-            [algorithm.hexdigest("#{@user}:#{params["realm"]}:#{@password}"),
-             nonce,
-             cnonce].join ":"
+            [
+              (@hashed ? @password : algorithm.hexdigest("#{@user}:#{params["realm"]}:#{@password}")),
+              nonce,
+              cnonce,
+            ].join ":"
           else
-            "#{@user}:#{params["realm"]}:#{@password}"
+            @hashed ? @password : "#{@user}:#{params["realm"]}:#{@password}"
           end
 
           ha1 = algorithm.hexdigest(a1)

data/lib/httpx/plugins/digest_authentication.rb

@@ -29,8 +29,8 @@ module HTTPX
       end
 
       module InstanceMethods
-        def digest_authentication(user, password)
-          with(digest: Authentication::Digest.new(user, password))
+        def digest_authentication(user, password, hashed: false)
+          with(digest: Authentication::Digest.new(user, password, hashed: hashed))
         end
 
         alias_method :digest_auth, :digest_authentication

data/lib/httpx/pool.rb

@@ -17,6 +17,7 @@ module HTTPX
       @timers = Timers.new
       @selector = Selector.new
       @connections = []
+      @eden_connections = []
       @connected_connections = 0
     end
 
@@ -52,6 +53,7 @@ module HTTPX
       return if connections.empty?
 
       @timers.cancel
+      @eden_connections.clear
       connections = connections.reject(&:inflight?)
       connections.each(&:close)
       next_tick until connections.none? { |c| c.state != :idle && @connections.include?(c) }
@@ -97,9 +99,24 @@ module HTTPX
     # maximize pipelining by opening as few connections as possible.
     #
     def find_connection(uri, options)
-      @connections.find do |connection|
+      conn = @connections.find do |connection|
         connection.match?(uri, options)
       end
+
+      unless conn
+        @eden_connections.delete_if do |connection|
+          is_expired = connection.expired?
+          conn = connection if conn.nil? && !is_expired && connection.match?(uri, options)
+          is_expired
+        end
+
+        if conn
+          @connections << conn
+          select_connection(conn)
+        end
+      end
+
+      conn
     end
 
     private
@@ -214,6 +231,10 @@ module HTTPX
 
     def unregister_connection(connection)
       @connections.delete(connection)
+      if connection.used? && !@eden_connections.include?(connection)
+        @eden_connections << connection
+        connection.idling
+      end
       @connected_connections -= 1 if deselect_connection(connection)
     end
 

data/lib/httpx/resolver/https.rb

@@ -132,7 +132,7 @@ module HTTPX
 
       case code
       when :ok
-        parse_addresses(result)
+        parse_addresses(result, request)
       when :no_domain_found
         # Indicates no such domain was found.
 
@@ -146,13 +146,13 @@ module HTTPX
 
         emit_resolve_error(connection)
       when :decode_error
-        host, connection = @queries.first
-        reset_hostname(host)
+        host = @requests.delete(request)
+        connection = reset_hostname(host)
         emit_resolve_error(connection, connection.origin.host, result)
       end
     end
 
-    def parse_addresses(answers)
+    def parse_addresses(answers, request)
       if answers.empty?
         # no address found, eliminate candidates
         host = @requests.delete(request)

data/lib/httpx/response.rb

@@ -301,7 +301,7 @@ module HTTPX
           end
         end
 
-        return unless %i[memory buffer].include?(@state)
+        nil unless %i[memory buffer].include?(@state)
       end
 
       def _with_same_buffer_pos

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "0.24.3"
+  VERSION = "0.24.5"
 end

data/sig/connection.rbs

@@ -48,6 +48,8 @@ module HTTPX
 
     def match?: (URI::Generic uri, Options options) -> bool
 
+    def expired?: () -> boolish
+
     def mergeable?: (Connection) -> bool
 
     def coalescable?: (Connection) -> bool
@@ -73,12 +75,17 @@ module HTTPX
     def call: () -> void
 
     def close: () -> void
+
     def reset: () -> void
 
     def send: (Request) -> void
 
     def timeout: () -> Numeric?
 
+    def idling: () -> void
+
+    def used?: () -> boolish
+
     def deactivate: () -> void
 
     def open?: () -> bool

data/sig/io/ssl.rbs

@@ -1,5 +1,4 @@
 module HTTPX
-  IPRegex: Regexp
 
   class TLSError < OpenSSL::SSL::SSLError
   end
@@ -7,10 +6,20 @@ module HTTPX
   class SSL < TCP
     TLS_OPTIONS: Hash[Symbol, untyped]
 
+    @ctx: OpenSSL::SSL::SSLContext
+    @verify_hostname: bool
+
+    attr_writer ssl_session: OpenSSL::SSL::Session?
+
+    # TODO: lift when https://github.com/ruby/rbs/issues/1497 fixed
+    # def initialize: (URI::Generic origin, Array[ipaddr]? addresses, options options) ?{ (self) -> void } -> void
+
     def can_verify_peer?: () -> bool
 
     def verify_hostname: (String host) -> bool
 
+    def ssl_session_expired?: () -> boolish
+
     # :nocov:
     def try_ssl_connect: () -> void
   end

data/sig/io/tcp.rbs

@@ -14,7 +14,8 @@ module HTTPX
 
     alias host ip
 
-    def initialize: (URI::Generic origin, Array[ipaddr]? addresses, options options) -> void
+    # TODO: lift when https://github.com/ruby/rbs/issues/1497 fixed
+    def initialize: (URI::Generic origin, Array[ipaddr]? addresses, options options) ?{ (self) -> void } -> void
 
     def add_addresses: (Array[ipaddr] addrs) -> void
 
@@ -39,6 +40,8 @@ module HTTPX
 
     def connected?: () -> bool
 
+    def expired?: () -> boolish
+
     def closed?: () -> bool
 
     # :nocov:

data/sig/plugins/authentication/digest.rbs

@@ -4,6 +4,7 @@ module HTTPX
       class Digest
         @user: String
         @password: String
+        @hashed: bool
 
         def can_authenticate?: (String? authenticate) -> boolish
 
@@ -13,7 +14,7 @@ module HTTPX
 
         def generate_header: (String meth, String uri, String authenticate) -> String
 
-        def initialize: (string user, string password) -> void
+        def initialize: (string user, string password, ?hashed: bool, **untyped) -> void
 
         def make_cnonce: () -> String
 

data/sig/plugins/digest_authentication.rbs

@@ -12,7 +12,7 @@ module HTTPX
       def self.load_dependencies: (*untyped) -> void
 
       module InstanceMethods
-        def digest_authentication: (string user, string password) -> instance
+        def digest_authentication: (string user, string password, ?hashed: bool) -> instance
       end
     end
 

data/sig/pool.rbs

@@ -6,6 +6,7 @@ module HTTPX
     @timers: Timers
     @selector: Selector
     @connections: Array[Connection]
+    @eden_connections: Array[Connection]
     @connected_connections: Integer
 
     def empty?: () -> void

data/sig/resolver/https.rbs

@@ -32,6 +32,8 @@ module HTTPX
 
       def parse: (Request request, Response response) -> void
 
+      def parse_addresses: (Array[dns_result] answers, Request request) -> void
+
       def build_request: (String hostname) -> Request
 
       def decode_response_body: (Response) -> dns_decoding_response

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: httpx
 version: !ruby/object:Gem::Version
-  version: 0.24.3
+  version: 0.24.5
 platform: ruby
 authors:
 - Tiago Cardoso
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-07-31 00:00:00.000000000 Z
+date: 2023-09-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: http-2-next
@@ -103,6 +103,8 @@ extra_rdoc_files:
 - doc/release_notes/0_24_1.md
 - doc/release_notes/0_24_2.md
 - doc/release_notes/0_24_3.md
+- doc/release_notes/0_24_4.md
+- doc/release_notes/0_24_5.md
 - doc/release_notes/0_2_0.md
 - doc/release_notes/0_2_1.md
 - doc/release_notes/0_3_0.md
@@ -198,6 +200,8 @@ files:
 - doc/release_notes/0_24_1.md
 - doc/release_notes/0_24_2.md
 - doc/release_notes/0_24_3.md
+- doc/release_notes/0_24_4.md
+- doc/release_notes/0_24_5.md
 - doc/release_notes/0_2_0.md
 - doc/release_notes/0_2_1.md
 - doc/release_notes/0_3_0.md