httpx 0.19.8 → 0.20.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 05a379db82c9c86f84a847680f4a784ed62a4cbcb8ce8d09a3d3c29abeb7d5d1
-  data.tar.gz: 5046f5d1e86f7d5c523a171b23fd5fb8126b63636a021c204a7c33a4ebf5e319
+  metadata.gz: b49337207a605c716ef9a1043660239ced7f5060758148f905be6c1d5546a8bc
+  data.tar.gz: 732ffb35912e5176827941034b74ce2777d744e828e1ed86351bdd221f010d19
 SHA512:
-  metadata.gz: d62d606f0688c485f79a38feed6b94841005fb6a70c5ab934c7b70cb7a7958066b85a496405959a7660da3f12f194d844f10660621ce6fb98c7aa898c38d855c
-  data.tar.gz: 42b46c44c4c864222f14235534be31db4eb5155716c542457178be010b227fdb391c095b009759e9c237d6320da7133123334cfc4b3d4dd6940ff58aa6df9f0e
+  metadata.gz: 5c1f12fa0d1675f8e8c478ea272456f910dc558b7836eaf0bfa6e916d29439e7dd093ffdde4903a43b474559909052df4d26872846e704c393045f10d496a53a
+  data.tar.gz: 9011bf6d1826ea2a44a3570f9dcbe9dc5bb319303809cf8b44247f9ac2bdb099ccc0895192155960df320dbdad4bebf009a7ec77dc1e37d1e8e27a59077efe5a

data/doc/release_notes/0_19_8.md

@@ -1,4 +1,4 @@
-# 0.19.7
+# 0.19.8
 
 ## Bugfixes
 

data/doc/release_notes/0_20_0.md

@@ -0,0 +1,36 @@
+# 0.20.0
+
+## Features
+
+### Sentry integration
+
+Documentation: https://gitlab.com/honeyryderchuck/httpx/-/wikis/Sentry-Adapter
+
+`httpx` ships with integration for `sentry-ruby` to provide HTTP request specific breadcrumbs and tracing. It can be enabled via:
+
+```ruby
+require "httpx/adapters/sentry"
+```
+
+### Proxy alternative auth schemes
+
+Besides the already previously supported (and still default) HTTP Basic Auth, the `:proxy` plugin supports HTTP Digest and NTLM auth as well. These are made available via the following APIs:
+
+```ruby
+http = HTTPX.plugin(:proxy)
+http.with_proxy_basic_auth(username: "user", password: "pass", uri: "http://proxy-uri:8126")
+http.with_proxy_digest_auth(username: "user", password: "pass", uri: "http://proxy-uri:8126")
+http.with_proxy_ntlm_auth(username: "user", password: "pass", uri: "http://proxy-uri:8126")
+
+# or alternatively
+http.with_proxy(proxy: "basic", username: "user", password: "pass", uri: "http://proxy-uri:8126")
+```
+
+## Bugfixes
+
+* HTTPS requests on an URL with an IP as a host, will now correctly not perform SNI during the TLS handshake, as per RFC;
+* `:follow_redirects` plugin will now halt redirections on 3xx responses with no `"location"` headers; this means it won't crash on 304 responses.
+    * If the `httpx` session has the `:proxy` plugin enabled, HTTP 305 responses will retry the request via the proxy exposed in the `"location"` header, as the RFC mandates.
+* `alt-svc` connection switch for HTTPS requests will be halted if the advertised alternative service "downgrades" to cleartext (example: `alt-svc` advertises `"h2c"`, but original connection was enabled via TLS).
+* A new connection to a TLS-enabled `alt-svc` advertised for a previous request, will now use that request's hostname as the SNI hostname, when performing the TLS handshake.
+* the `:response_cache` plugin will now correctly handle capitalized HTTP headers advertised in the `"vary"` header.

data/doc/release_notes/0_20_1.md

@@ -0,0 +1,5 @@
+# 0.20.1
+
+## Bugfixes
+
+* bugfix for unregistering connections when timing out on DNS resolving; this wasn't happening, leaving a few cases where requests to the same domain timing out on resolution would hang on the second request.

data/doc/release_notes/0_20_2.md

@@ -0,0 +1,7 @@
+# 0.20.2
+
+## Bugfixes
+
+* fix for selector timeout errors closing all connections and ignoring resolvers.
+
+Timeout errors on select were being propagated to all pooled connections, although not all of them were being selected on, and not all of them having timed out. plus, resolver timeouts were doing the same, making connections fail with connection timeout error, rather than resolve timeout error. A patch was implemented, where the selector now yields an error to the selected connections, rather than plain raising exception.

data/lib/httpx/adapters/sentry.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require "sentry-ruby"
+
+module HTTPX::Plugins
+  module Sentry
+    module Tracer
+      module_function
+
+      def call(request)
+        sentry_span = start_sentry_span
+
+        return unless sentry_span
+
+        set_sentry_trace_header(request, sentry_span)
+
+        request.on(:response, &method(:finish_sentry_span).curry(3)[sentry_span, request])
+      end
+
+      def start_sentry_span
+        return unless ::Sentry.initialized? && (span = ::Sentry.get_current_scope.get_span)
+        return if span.sampled == false
+
+        span.start_child(op: "httpx.client", start_timestamp: ::Sentry.utc_now.to_f)
+      end
+
+      def set_sentry_trace_header(request, sentry_span)
+        return unless sentry_span
+
+        trace = ::Sentry.get_current_client.generate_sentry_trace(sentry_span)
+        request.headers[::Sentry::SENTRY_TRACE_HEADER_NAME] = trace if trace
+      end
+
+      def finish_sentry_span(span, request, response)
+        return unless ::Sentry.initialized?
+
+        record_sentry_breadcrumb(request, response)
+        record_sentry_span(request, response, span)
+      end
+
+      def record_sentry_breadcrumb(req, res)
+        return unless ::Sentry.configuration.breadcrumbs_logger.include?(:http_logger)
+
+        request_info = extract_request_info(req)
+
+        data = if response.is_a?(HTTPX::ErrorResponse)
+          { error: res.message, **request_info }
+        else
+          { status: res.status, **request_info }
+        end
+
+        crumb = ::Sentry::Breadcrumb.new(
+          level: :info,
+          category: "httpx",
+          type: :info,
+          data: data
+        )
+        ::Sentry.add_breadcrumb(crumb)
+      end
+
+      def record_sentry_span(req, res, sentry_span)
+        return unless sentry_span
+
+        request_info = extract_request_info(req)
+        sentry_span.set_description("#{request_info[:method]} #{request_info[:url]}")
+        sentry_span.set_data(:status, res.status)
+        sentry_span.set_timestamp(::Sentry.utc_now.to_f)
+      end
+
+      def extract_request_info(req)
+        uri = req.uri
+
+        result = {
+          method: req.verb.to_s.upcase,
+        }
+
+        if ::Sentry.configuration.send_default_pii
+          uri += "?#{req.query}" unless req.query.empty?
+          result[:body] = req.body.to_s unless req.body.empty? || req.body.unbounded_body?
+        end
+
+        result[:url] = uri.to_s
+
+        result
+      end
+    end
+
+    module ConnectionMethods
+      def send(request)
+        Tracer.call(request)
+        super
+      end
+    end
+  end
+end
+
+Sentry.register_patch do
+  sentry_session = ::HTTPX.plugin(HTTPX::Plugins::Sentry)
+
+  HTTPX.send(:remove_const, :Session)
+  HTTPX.send(:const_set, :Session, sentry_session.class)
+end

data/lib/httpx/connection.rb

@@ -102,8 +102,8 @@ module HTTPX
           # origin came from an ORIGIN frame, we're going to verify the hostname with the
           # SSL certificate
           (@origins.size == 1 || @origin == uri.origin || (@io && @io.verify_hostname(uri.host)))
-        ) || match_altsvcs?(uri)
-      ) && @options == options
+        ) && @options == options
+      ) || (match_altsvcs?(uri) && match_altsvc_options?(uri, options))
     end
 
     def mergeable?(connection)
@@ -162,6 +162,14 @@ module HTTPX
         end
     end
 
+    def match_altsvc_options?(uri, options)
+      return @options == options unless @options.ssl[:hostname] == uri.host
+
+      dup_options = @options.merge(ssl: { hostname: nil })
+      dup_options.ssl.delete(:hostname)
+      dup_options == options
+    end
+
     def connecting?
       @state == :idle
     end
@@ -268,6 +276,12 @@ module HTTPX
       @state == :open || @state == :inactive
     end
 
+    def raise_timeout_error(interval)
+      error = HTTPX::TimeoutError.new(interval, "timed out while waiting on select")
+      error.set_backtrace(caller)
+      on_error(error)
+    end
+
     private
 
     def connect

data/lib/httpx/io/ssl.rb

@@ -4,8 +4,11 @@ require "openssl"
 
 module HTTPX
   TLSError = OpenSSL::SSL::SSLError
+  IPRegex = Regexp.union(Resolv::IPv4::Regex, Resolv::IPv6::Regex)
 
   class SSL < TCP
+    using RegexpExtensions unless Regexp.method_defined?(:match?)
+
     TLS_OPTIONS = if OpenSSL::SSL::SSLContext.instance_methods.include?(:alpn_protocols)
       { alpn_protocols: %w[h2 http/1.1].freeze }.freeze
     else
@@ -19,6 +22,8 @@ module HTTPX
       @sni_hostname = ctx_options.delete(:hostname) || @hostname
       @ctx.set_params(ctx_options) unless ctx_options.empty?
       @state = :negotiated if @keep_open
+
+      @hostname_is_ip = IPRegex.match?(@sni_hostname)
     end
 
     def protocol
@@ -56,7 +61,7 @@ module HTTPX
 
       unless @io.is_a?(OpenSSL::SSL::SSLSocket)
         @io = OpenSSL::SSL::SSLSocket.new(@io, @ctx)
-        @io.hostname = @sni_hostname
+        @io.hostname = @sni_hostname unless @hostname_is_ip
         @io.sync_close = true
       end
       try_ssl_connect
@@ -66,7 +71,7 @@ module HTTPX
       # :nocov:
       def try_ssl_connect
         @io.connect_nonblock
-        @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
+        @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE && !@hostname_is_ip
         transition(:negotiated)
         @interests = :w
       rescue ::IO::WaitReadable
@@ -98,7 +103,7 @@ module HTTPX
           @interests = :w
           return
         end
-        @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
+        @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE && !@hostname_is_ip
         transition(:negotiated)
         @interests = :w
       end

data/lib/httpx/options.rb

@@ -132,6 +132,7 @@ module HTTPX
     def freeze
       super
       @origin.freeze
+      @base_path.freeze
       @timeout.freeze
       @headers.freeze
       @addresses.freeze
@@ -141,6 +142,10 @@ module HTTPX
       URI(value)
     end
 
+    def option_base_path(value)
+      String(value)
+    end
+
     def option_headers(value)
       Headers.new(value)
     end

data/lib/httpx/plugins/authentication/basic.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require "base64"
+
+module HTTPX
+  module Plugins
+    module Authentication
+      class Basic
+        def initialize(user, password, **)
+          @user = user
+          @password = password
+        end
+
+        def can_authenticate?(authenticate)
+          authenticate && /Basic .*/.match?(authenticate)
+        end
+
+        def authenticate(*)
+          "Basic #{Base64.strict_encode64("#{@user}:#{@password}")}"
+        end
+      end
+    end
+  end
+end

data/lib/httpx/plugins/authentication/digest.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require "time"
+require "securerandom"
+require "digest"
+
+module HTTPX
+  module Plugins
+    module Authentication
+      class Digest
+        using RegexpExtensions unless Regexp.method_defined?(:match?)
+
+        def initialize(user, password, **)
+          @user = user
+          @password = password
+          @nonce = 0
+        end
+
+        def can_authenticate?(authenticate)
+          authenticate && /Digest .*/.match?(authenticate)
+        end
+
+        def authenticate(request, authenticate)
+          "Digest #{generate_header(request.verb.to_s.upcase, request.path, authenticate)}"
+        end
+
+        private
+
+        def generate_header(meth, uri, authenticate)
+          # discard first token, it's Digest
+          auth_info = authenticate[/^(\w+) (.*)/, 2]
+
+          params = Hash[auth_info.split(/ *, */)
+                                 .map { |val| val.split("=") }
+                                 .map { |k, v| [k, v.delete("\"")] }]
+          nonce = params["nonce"]
+          nc = next_nonce
+
+          # verify qop
+          qop = params["qop"]
+
+          if params["algorithm"] =~ /(.*?)(-sess)?$/
+            alg = Regexp.last_match(1)
+            algorithm = ::Digest.const_get(alg)
+            raise DigestError, "unknown algorithm \"#{alg}\"" unless algorithm
+
+            sess = Regexp.last_match(2)
+            params.delete("algorithm")
+          else
+            algorithm = ::Digest::MD5
+          end
+
+          if qop || sess
+            cnonce = make_cnonce
+            nc = format("%<nonce>08x", nonce: nc)
+          end
+
+          a1 = if sess
+            [algorithm.hexdigest("#{@user}:#{params["realm"]}:#{@password}"),
+             nonce,
+             cnonce].join ":"
+          else
+            "#{@user}:#{params["realm"]}:#{@password}"
+          end
+
+          ha1 = algorithm.hexdigest(a1)
+          ha2 = algorithm.hexdigest("#{meth}:#{uri}")
+          request_digest = [ha1, nonce]
+          request_digest.push(nc, cnonce, qop) if qop
+          request_digest << ha2
+          request_digest = request_digest.join(":")
+
+          header = [
+            %(username="#{@user}"),
+            %(nonce="#{nonce}"),
+            %(uri="#{uri}"),
+            %(response="#{algorithm.hexdigest(request_digest)}"),
+          ]
+          header << %(realm="#{params["realm"]}") if params.key?("realm")
+          header << %(algorithm=#{params["algorithm"]}") if params.key?("algorithm")
+          header << %(opaque="#{params["opaque"]}") if params.key?("opaque")
+          header << %(cnonce="#{cnonce}") if cnonce
+          header << %(nc=#{nc})
+          header << %(qop=#{qop}) if qop
+          header.join ", "
+        end
+
+        def make_cnonce
+          ::Digest::MD5.hexdigest [
+            Time.now.to_i,
+            Process.pid,
+            SecureRandom.random_number(2**32),
+          ].join ":"
+        end
+
+        def next_nonce
+          @nonce += 1
+        end
+      end
+    end
+  end
+end

data/lib/httpx/plugins/authentication/ntlm.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "base64"
+require "ntlm"
+
+module HTTPX
+  module Plugins
+    module Authentication
+      class Ntlm
+        using RegexpExtensions unless Regexp.method_defined?(:match?)
+
+        def initialize(user, password, domain: nil)
+          @user = user
+          @password = password
+          @domain = domain
+        end
+
+        def can_authenticate?(authenticate)
+          authenticate && /NTLM .*/.match?(authenticate)
+        end
+
+        def negotiate
+          "NTLM #{NTLM.negotiate(domain: @domain).to_base64}"
+        end
+
+        def authenticate(_req, www)
+          challenge = www[/NTLM (.*)/, 1]
+
+          challenge = Base64.decode64(challenge)
+          ntlm_challenge = NTLM.authenticate(challenge, @user, @domain, @password).to_base64
+
+          "NTLM #{ntlm_challenge}"
+        end
+      end
+    end
+  end
+end

data/lib/httpx/plugins/authentication/socks5.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require "base64"
+
+module HTTPX
+  module Plugins
+    module Authentication
+      class Socks5
+        def initialize(user, password, **)
+          @user = user
+          @password = password
+        end
+
+        def can_authenticate?(*)
+          @user && @password
+        end
+
+        def authenticate(*)
+          [0x01, @user.bytesize, @user, @password.bytesize, @password].pack("CCA*CA*")
+        end
+      end
+    end
+  end
+end

data/lib/httpx/plugins/basic_authentication.rb

@@ -7,10 +7,10 @@ module HTTPX
     #
     # https://gitlab.com/honeyryderchuck/httpx/wikis/Authentication#basic-authentication
     #
-    module BasicAuthentication
+    module BasicAuth
       class << self
         def load_dependencies(_klass)
-          require "base64"
+          require_relative "authentication/basic"
         end
 
         def configure(klass)
@@ -19,12 +19,12 @@ module HTTPX
       end
 
       module InstanceMethods
-        def basic_authentication(user, password)
-          authentication("Basic #{Base64.strict_encode64("#{user}:#{password}")}")
+        def basic_auth(user, password)
+          authentication(Authentication::Basic.new(user, password).authenticate)
         end
-        alias_method :basic_auth, :basic_authentication
+        alias_method :basic_authentication, :basic_auth
       end
     end
-    register_plugin :basic_authentication, BasicAuthentication
+    register_plugin :basic_authentication, BasicAuth
   end
 end

data/lib/httpx/plugins/digest_authentication.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "digest"
-
 module HTTPX
   module Plugins
     #
@@ -9,9 +7,7 @@ module HTTPX
     #
     # https://gitlab.com/honeyryderchuck/httpx/wikis/Authentication#authentication
     #
-    module DigestAuthentication
-      using RegexpExtensions unless Regexp.method_defined?(:match?)
-
+    module DigestAuth
       DigestError = Class.new(Error)
 
       class << self
@@ -20,14 +16,13 @@ module HTTPX
         end
 
         def load_dependencies(*)
-          require "securerandom"
-          require "digest"
+          require_relative "authentication/digest"
         end
       end
 
       module OptionsMethods
         def option_digest(value)
-          raise TypeError, ":digest must be a Digest" unless value.is_a?(Digest)
+          raise TypeError, ":digest must be a Digest" unless value.is_a?(Authentication::Digest)
 
           value
         end
@@ -35,7 +30,7 @@ module HTTPX
 
       module InstanceMethods
         def digest_authentication(user, password)
-          with(digest: Digest.new(user, password))
+          with(digest: Authentication::Digest.new(user, password))
         end
 
         alias_method :digest_auth, :digest_authentication
@@ -44,116 +39,25 @@ module HTTPX
           requests.flat_map do |request|
             digest = request.options.digest
 
-            if digest
-              probe_response = wrap { super(request).first }
-
-              if digest && !probe_response.is_a?(ErrorResponse) &&
-                 probe_response.status == 401 && probe_response.headers.key?("www-authenticate") &&
-                 /Digest .*/.match?(probe_response.headers["www-authenticate"])
-
-                request.transition(:idle)
-
-                token = digest.generate_header(request, probe_response)
-                request.headers["authorization"] = "Digest #{token}"
-
-                super(request)
-              else
-                probe_response
-              end
-            else
+            unless digest
               super(request)
+              next
             end
-          end
-        end
-      end
-
-      class Digest
-        def initialize(user, password)
-          @user = user
-          @password = password
-          @nonce = 0
-        end
-
-        def generate_header(request, response, _iis = false)
-          meth = request.verb.to_s.upcase
-          www = response.headers["www-authenticate"]
-
-          # discard first token, it's Digest
-          auth_info = www[/^(\w+) (.*)/, 2]
-
-          uri = request.path
-
-          params = Hash[auth_info.split(/ *, */)
-                                 .map { |val| val.split("=") }
-                                 .map { |k, v| [k, v.delete("\"")] }]
-          nonce = params["nonce"]
-          nc = next_nonce
 
-          # verify qop
-          qop = params["qop"]
+            probe_response = wrap { super(request).first }
 
-          if params["algorithm"] =~ /(.*?)(-sess)?$/
-            alg = Regexp.last_match(1)
-            algorithm = ::Digest.const_get(alg)
-            raise DigestError, "unknown algorithm \"#{alg}\"" unless algorithm
-
-            sess = Regexp.last_match(2)
-            params.delete("algorithm")
-          else
-            algorithm = ::Digest::MD5
-          end
-
-          if qop || sess
-            cnonce = make_cnonce
-            nc = format("%<nonce>08x", nonce: nc)
-          end
-
-          a1 = if sess
-            [algorithm.hexdigest("#{@user}:#{params["realm"]}:#{@password}"),
-             nonce,
-             cnonce].join ":"
-          else
-            "#{@user}:#{params["realm"]}:#{@password}"
+            if probe_response.status == 401 && digest.can_authenticate?(probe_response.headers["www-authenticate"])
+              request.transition(:idle)
+              request.headers["authorization"] = digest.authenticate(request, probe_response.headers["www-authenticate"])
+              super(request)
+            else
+              probe_response
+            end
           end
-
-          ha1 = algorithm.hexdigest(a1)
-          ha2 = algorithm.hexdigest("#{meth}:#{uri}")
-          request_digest = [ha1, nonce]
-          request_digest.push(nc, cnonce, qop) if qop
-          request_digest << ha2
-          request_digest = request_digest.join(":")
-
-          header = [
-            %(username="#{@user}"),
-            %(nonce="#{nonce}"),
-            %(uri="#{uri}"),
-            %(response="#{algorithm.hexdigest(request_digest)}"),
-          ]
-          header << %(realm="#{params["realm"]}") if params.key?("realm")
-          header << %(algorithm=#{params["algorithm"]}") if params.key?("algorithm")
-          header << %(opaque="#{params["opaque"]}") if params.key?("opaque")
-          header << %(cnonce="#{cnonce}") if cnonce
-          header << %(nc=#{nc})
-          header << %(qop=#{qop}) if qop
-          header.join ", "
-        end
-
-        private
-
-        def make_cnonce
-          ::Digest::MD5.hexdigest [
-            Time.now.to_i,
-            Process.pid,
-            SecureRandom.random_number(2**32),
-          ].join ":"
-        end
-
-        def next_nonce
-          @nonce += 1
         end
       end
     end
 
-    register_plugin :digest_authentication, DigestAuthentication
+    register_plugin :digest_authentication, DigestAuth
   end
 end