httpx 0.19.8 → 0.20.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 05a379db82c9c86f84a847680f4a784ed62a4cbcb8ce8d09a3d3c29abeb7d5d1
-  data.tar.gz: 5046f5d1e86f7d5c523a171b23fd5fb8126b63636a021c204a7c33a4ebf5e319
+  metadata.gz: a5a31aeffa012bd1eea19c1e892c8669624de0d21c9cd8911ae025375f0b93fb
+  data.tar.gz: 285707682eb2d1451106d28bc2d4a8ceb247d4237139c9e7d83bd5a0163b5a5c
 SHA512:
-  metadata.gz: d62d606f0688c485f79a38feed6b94841005fb6a70c5ab934c7b70cb7a7958066b85a496405959a7660da3f12f194d844f10660621ce6fb98c7aa898c38d855c
-  data.tar.gz: 42b46c44c4c864222f14235534be31db4eb5155716c542457178be010b227fdb391c095b009759e9c237d6320da7133123334cfc4b3d4dd6940ff58aa6df9f0e
+  metadata.gz: 6f2c50aa8895c6f07ed0acbca1b1171343da2b55bc838ae7f2a0f7a44300c101ea0020938946bf630b80ad6b81bed3cb54910b0734576b6e7c000e058497128c
+  data.tar.gz: 83e19a86d841056821d9198c9a8e3e7b06ec354445289a1fd7d612d070b02b0a01aa9f45f5acdb0b35e4da413c2c3955646e5f55c5421db95879092a0a3aa377

data/doc/release_notes/0_20_0.md

@@ -0,0 +1,36 @@
+# 0.19.0
+
+## Features
+
+### Sentry integration
+
+Documentation: https://gitlab.com/honeyryderchuck/httpx/-/wikis/Sentry-Adapter
+
+`httpx` ships with integration for `sentry-ruby` to provide HTTP request specific breadcrumbs and tracing. It can be enabled via:
+
+```ruby
+require "httpx/adapters/sentry"
+```
+
+### Proxy alternative auth schemes
+
+Besides the already previously supported (and still default) HTTP Basic Auth, the `:proxy` plugin supports HTTP Digest and NTLM auth as well. These are made available via the following APIs:
+
+```ruby
+http = HTTPX.plugin(:proxy)
+http.with_proxy_basic_auth(username: "user", password: "pass", uri: "http://proxy-uri:8126")
+http.with_proxy_digest_auth(username: "user", password: "pass", uri: "http://proxy-uri:8126")
+http.with_proxy_ntlm_auth(username: "user", password: "pass", uri: "http://proxy-uri:8126")
+
+# or alternatively
+http.with_proxy(proxy: "basic", username: "user", password: "pass", uri: "http://proxy-uri:8126")
+```
+
+## Bugfixes
+
+* HTTPS requests on an URL with an IP as a host, will now correctly not perform SNI during the TLS handshake, as per RFC;
+* `:follow_redirects` plugin will now halt redirections on 3xx responses with no `"location"` headers; this means it won't crash on 304 responses.
+    * If the `httpx` session has the `:proxy` plugin enabled, HTTP 305 responses will retry the request via the proxy exposed in the `"location"` header, as the RFC mandates.
+* `alt-svc` connection switch for HTTPS requests will be halted if the advertised alternative service "downgrades" to cleartext (example: `alt-svc` advertises `"h2c"`, but original connection was enabled via TLS).
+* A new connection to a TLS-enabled `alt-svc` advertised for a previous request, will now use that request's hostname as the SNI hostname, when performing the TLS handshake.
+* the `:response_cache` plugin will now correctly handle capitalized HTTP headers advertised in the `"vary"` header.

data/lib/httpx/adapters/sentry.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require "sentry-ruby"
+
+module HTTPX::Plugins
+  module Sentry
+    module Tracer
+      module_function
+
+      def call(request)
+        sentry_span = start_sentry_span
+
+        return unless sentry_span
+
+        set_sentry_trace_header(request, sentry_span)
+
+        request.on(:response, &method(:finish_sentry_span).curry(3)[sentry_span, request])
+      end
+
+      def start_sentry_span
+        return unless ::Sentry.initialized? && (span = ::Sentry.get_current_scope.get_span)
+        return if span.sampled == false
+
+        span.start_child(op: "httpx.client", start_timestamp: ::Sentry.utc_now.to_f)
+      end
+
+      def set_sentry_trace_header(request, sentry_span)
+        return unless sentry_span
+
+        trace = ::Sentry.get_current_client.generate_sentry_trace(sentry_span)
+        request.headers[::Sentry::SENTRY_TRACE_HEADER_NAME] = trace if trace
+      end
+
+      def finish_sentry_span(span, request, response)
+        return unless ::Sentry.initialized?
+
+        record_sentry_breadcrumb(request, response)
+        record_sentry_span(request, response, span)
+      end
+
+      def record_sentry_breadcrumb(req, res)
+        return unless ::Sentry.configuration.breadcrumbs_logger.include?(:http_logger)
+
+        request_info = extract_request_info(req)
+
+        data = if response.is_a?(HTTPX::ErrorResponse)
+          { error: res.message, **request_info }
+        else
+          { status: res.status, **request_info }
+        end
+
+        crumb = ::Sentry::Breadcrumb.new(
+          level: :info,
+          category: "httpx",
+          type: :info,
+          data: data
+        )
+        ::Sentry.add_breadcrumb(crumb)
+      end
+
+      def record_sentry_span(req, res, sentry_span)
+        return unless sentry_span
+
+        request_info = extract_request_info(req)
+        sentry_span.set_description("#{request_info[:method]} #{request_info[:url]}")
+        sentry_span.set_data(:status, res.status)
+        sentry_span.set_timestamp(::Sentry.utc_now.to_f)
+      end
+
+      def extract_request_info(req)
+        uri = req.uri
+
+        result = {
+          method: req.verb.to_s.upcase,
+        }
+
+        if ::Sentry.configuration.send_default_pii
+          uri += "?#{req.query}" unless req.query.empty?
+          result[:body] = req.body.to_s unless req.body.empty? || req.body.unbounded_body?
+        end
+
+        result[:url] = uri.to_s
+
+        result
+      end
+    end
+
+    module ConnectionMethods
+      def send(request)
+        Tracer.call(request)
+        super
+      end
+    end
+  end
+end
+
+Sentry.register_patch do
+  sentry_session = ::HTTPX.plugin(HTTPX::Plugins::Sentry)
+
+  HTTPX.send(:remove_const, :Session)
+  HTTPX.send(:const_set, :Session, sentry_session.class)
+end

data/lib/httpx/connection.rb

@@ -102,8 +102,8 @@ module HTTPX
           # origin came from an ORIGIN frame, we're going to verify the hostname with the
           # SSL certificate
           (@origins.size == 1 || @origin == uri.origin || (@io && @io.verify_hostname(uri.host)))
-        ) || match_altsvcs?(uri)
-      ) && @options == options
+        ) && @options == options
+      ) || (match_altsvcs?(uri) && match_altsvc_options?(uri, options))
     end
 
     def mergeable?(connection)
@@ -162,6 +162,14 @@ module HTTPX
         end
     end
 
+    def match_altsvc_options?(uri, options)
+      return @options == options unless @options.ssl[:hostname] == uri.host
+
+      dup_options = @options.merge(ssl: { hostname: nil })
+      dup_options.ssl.delete(:hostname)
+      dup_options == options
+    end
+
     def connecting?
       @state == :idle
     end

data/lib/httpx/io/ssl.rb

@@ -4,8 +4,11 @@ require "openssl"
 
 module HTTPX
   TLSError = OpenSSL::SSL::SSLError
+  IPRegex = Regexp.union(Resolv::IPv4::Regex, Resolv::IPv6::Regex)
 
   class SSL < TCP
+    using RegexpExtensions unless Regexp.method_defined?(:match?)
+
     TLS_OPTIONS = if OpenSSL::SSL::SSLContext.instance_methods.include?(:alpn_protocols)
       { alpn_protocols: %w[h2 http/1.1].freeze }.freeze
     else
@@ -19,6 +22,8 @@ module HTTPX
       @sni_hostname = ctx_options.delete(:hostname) || @hostname
       @ctx.set_params(ctx_options) unless ctx_options.empty?
       @state = :negotiated if @keep_open
+
+      @hostname_is_ip = IPRegex.match?(@sni_hostname)
     end
 
     def protocol
@@ -56,7 +61,7 @@ module HTTPX
 
       unless @io.is_a?(OpenSSL::SSL::SSLSocket)
         @io = OpenSSL::SSL::SSLSocket.new(@io, @ctx)
-        @io.hostname = @sni_hostname
+        @io.hostname = @sni_hostname unless @hostname_is_ip
         @io.sync_close = true
       end
       try_ssl_connect
@@ -66,7 +71,7 @@ module HTTPX
       # :nocov:
       def try_ssl_connect
         @io.connect_nonblock
-        @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
+        @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE && !@hostname_is_ip
         transition(:negotiated)
         @interests = :w
       rescue ::IO::WaitReadable
@@ -98,7 +103,7 @@ module HTTPX
           @interests = :w
           return
         end
-        @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
+        @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE && !@hostname_is_ip
         transition(:negotiated)
         @interests = :w
       end

data/lib/httpx/options.rb

@@ -132,6 +132,7 @@ module HTTPX
     def freeze
       super
       @origin.freeze
+      @base_path.freeze
       @timeout.freeze
       @headers.freeze
       @addresses.freeze
@@ -141,6 +142,10 @@ module HTTPX
       URI(value)
     end
 
+    def option_base_path(value)
+      String(value)
+    end
+
     def option_headers(value)
       Headers.new(value)
     end

data/lib/httpx/plugins/authentication/basic.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require "base64"
+
+module HTTPX
+  module Plugins
+    module Authentication
+      class Basic
+        def initialize(user, password, **)
+          @user = user
+          @password = password
+        end
+
+        def can_authenticate?(authenticate)
+          authenticate && /Basic .*/.match?(authenticate)
+        end
+
+        def authenticate(*)
+          "Basic #{Base64.strict_encode64("#{@user}:#{@password}")}"
+        end
+      end
+    end
+  end
+end

data/lib/httpx/plugins/authentication/digest.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require "time"
+require "securerandom"
+require "digest"
+
+module HTTPX
+  module Plugins
+    module Authentication
+      class Digest
+        using RegexpExtensions unless Regexp.method_defined?(:match?)
+
+        def initialize(user, password, **)
+          @user = user
+          @password = password
+          @nonce = 0
+        end
+
+        def can_authenticate?(authenticate)
+          authenticate && /Digest .*/.match?(authenticate)
+        end
+
+        def authenticate(request, authenticate)
+          "Digest #{generate_header(request.verb.to_s.upcase, request.path, authenticate)}"
+        end
+
+        private
+
+        def generate_header(meth, uri, authenticate)
+          # discard first token, it's Digest
+          auth_info = authenticate[/^(\w+) (.*)/, 2]
+
+          params = Hash[auth_info.split(/ *, */)
+                                 .map { |val| val.split("=") }
+                                 .map { |k, v| [k, v.delete("\"")] }]
+          nonce = params["nonce"]
+          nc = next_nonce
+
+          # verify qop
+          qop = params["qop"]
+
+          if params["algorithm"] =~ /(.*?)(-sess)?$/
+            alg = Regexp.last_match(1)
+            algorithm = ::Digest.const_get(alg)
+            raise DigestError, "unknown algorithm \"#{alg}\"" unless algorithm
+
+            sess = Regexp.last_match(2)
+            params.delete("algorithm")
+          else
+            algorithm = ::Digest::MD5
+          end
+
+          if qop || sess
+            cnonce = make_cnonce
+            nc = format("%<nonce>08x", nonce: nc)
+          end
+
+          a1 = if sess
+            [algorithm.hexdigest("#{@user}:#{params["realm"]}:#{@password}"),
+             nonce,
+             cnonce].join ":"
+          else
+            "#{@user}:#{params["realm"]}:#{@password}"
+          end
+
+          ha1 = algorithm.hexdigest(a1)
+          ha2 = algorithm.hexdigest("#{meth}:#{uri}")
+          request_digest = [ha1, nonce]
+          request_digest.push(nc, cnonce, qop) if qop
+          request_digest << ha2
+          request_digest = request_digest.join(":")
+
+          header = [
+            %(username="#{@user}"),
+            %(nonce="#{nonce}"),
+            %(uri="#{uri}"),
+            %(response="#{algorithm.hexdigest(request_digest)}"),
+          ]
+          header << %(realm="#{params["realm"]}") if params.key?("realm")
+          header << %(algorithm=#{params["algorithm"]}") if params.key?("algorithm")
+          header << %(opaque="#{params["opaque"]}") if params.key?("opaque")
+          header << %(cnonce="#{cnonce}") if cnonce
+          header << %(nc=#{nc})
+          header << %(qop=#{qop}) if qop
+          header.join ", "
+        end
+
+        def make_cnonce
+          ::Digest::MD5.hexdigest [
+            Time.now.to_i,
+            Process.pid,
+            SecureRandom.random_number(2**32),
+          ].join ":"
+        end
+
+        def next_nonce
+          @nonce += 1
+        end
+      end
+    end
+  end
+end

data/lib/httpx/plugins/authentication/ntlm.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "base64"
+require "ntlm"
+
+module HTTPX
+  module Plugins
+    module Authentication
+      class Ntlm
+        using RegexpExtensions unless Regexp.method_defined?(:match?)
+
+        def initialize(user, password, domain: nil)
+          @user = user
+          @password = password
+          @domain = domain
+        end
+
+        def can_authenticate?(authenticate)
+          authenticate && /NTLM .*/.match?(authenticate)
+        end
+
+        def negotiate
+          "NTLM #{NTLM.negotiate(domain: @domain).to_base64}"
+        end
+
+        def authenticate(_req, www)
+          challenge = www[/NTLM (.*)/, 1]
+
+          challenge = Base64.decode64(challenge)
+          ntlm_challenge = NTLM.authenticate(challenge, @user, @domain, @password).to_base64
+
+          "NTLM #{ntlm_challenge}"
+        end
+      end
+    end
+  end
+end

data/lib/httpx/plugins/authentication/socks5.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require "base64"
+
+module HTTPX
+  module Plugins
+    module Authentication
+      class Socks5
+        def initialize(user, password, **)
+          @user = user
+          @password = password
+        end
+
+        def can_authenticate?(*)
+          @user && @password
+        end
+
+        def authenticate(*)
+          [0x01, @user.bytesize, @user, @password.bytesize, @password].pack("CCA*CA*")
+        end
+      end
+    end
+  end
+end

data/lib/httpx/plugins/basic_authentication.rb

@@ -7,10 +7,10 @@ module HTTPX
     #
     # https://gitlab.com/honeyryderchuck/httpx/wikis/Authentication#basic-authentication
     #
-    module BasicAuthentication
+    module BasicAuth
       class << self
         def load_dependencies(_klass)
-          require "base64"
+          require_relative "authentication/basic"
         end
 
         def configure(klass)
@@ -19,12 +19,12 @@ module HTTPX
       end
 
       module InstanceMethods
-        def basic_authentication(user, password)
-          authentication("Basic #{Base64.strict_encode64("#{user}:#{password}")}")
+        def basic_auth(user, password)
+          authentication(Authentication::Basic.new(user, password).authenticate)
         end
-        alias_method :basic_auth, :basic_authentication
+        alias_method :basic_authentication, :basic_auth
       end
     end
-    register_plugin :basic_authentication, BasicAuthentication
+    register_plugin :basic_authentication, BasicAuth
   end
 end

data/lib/httpx/plugins/digest_authentication.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "digest"
-
 module HTTPX
   module Plugins
     #
@@ -9,9 +7,7 @@ module HTTPX
     #
     # https://gitlab.com/honeyryderchuck/httpx/wikis/Authentication#authentication
     #
-    module DigestAuthentication
-      using RegexpExtensions unless Regexp.method_defined?(:match?)
-
+    module DigestAuth
       DigestError = Class.new(Error)
 
       class << self
@@ -20,14 +16,13 @@ module HTTPX
         end
 
         def load_dependencies(*)
-          require "securerandom"
-          require "digest"
+          require_relative "authentication/digest"
         end
       end
 
       module OptionsMethods
         def option_digest(value)
-          raise TypeError, ":digest must be a Digest" unless value.is_a?(Digest)
+          raise TypeError, ":digest must be a Digest" unless value.is_a?(Authentication::Digest)
 
           value
         end
@@ -35,7 +30,7 @@ module HTTPX
 
       module InstanceMethods
         def digest_authentication(user, password)
-          with(digest: Digest.new(user, password))
+          with(digest: Authentication::Digest.new(user, password))
         end
 
         alias_method :digest_auth, :digest_authentication
@@ -44,116 +39,25 @@ module HTTPX
           requests.flat_map do |request|
             digest = request.options.digest
 
-            if digest
-              probe_response = wrap { super(request).first }
-
-              if digest && !probe_response.is_a?(ErrorResponse) &&
-                 probe_response.status == 401 && probe_response.headers.key?("www-authenticate") &&
-                 /Digest .*/.match?(probe_response.headers["www-authenticate"])
-
-                request.transition(:idle)
-
-                token = digest.generate_header(request, probe_response)
-                request.headers["authorization"] = "Digest #{token}"
-
-                super(request)
-              else
-                probe_response
-              end
-            else
+            unless digest
               super(request)
+              next
             end
-          end
-        end
-      end
-
-      class Digest
-        def initialize(user, password)
-          @user = user
-          @password = password
-          @nonce = 0
-        end
-
-        def generate_header(request, response, _iis = false)
-          meth = request.verb.to_s.upcase
-          www = response.headers["www-authenticate"]
-
-          # discard first token, it's Digest
-          auth_info = www[/^(\w+) (.*)/, 2]
-
-          uri = request.path
-
-          params = Hash[auth_info.split(/ *, */)
-                                 .map { |val| val.split("=") }
-                                 .map { |k, v| [k, v.delete("\"")] }]
-          nonce = params["nonce"]
-          nc = next_nonce
 
-          # verify qop
-          qop = params["qop"]
+            probe_response = wrap { super(request).first }
 
-          if params["algorithm"] =~ /(.*?)(-sess)?$/
-            alg = Regexp.last_match(1)
-            algorithm = ::Digest.const_get(alg)
-            raise DigestError, "unknown algorithm \"#{alg}\"" unless algorithm
-
-            sess = Regexp.last_match(2)
-            params.delete("algorithm")
-          else
-            algorithm = ::Digest::MD5
-          end
-
-          if qop || sess
-            cnonce = make_cnonce
-            nc = format("%<nonce>08x", nonce: nc)
-          end
-
-          a1 = if sess
-            [algorithm.hexdigest("#{@user}:#{params["realm"]}:#{@password}"),
-             nonce,
-             cnonce].join ":"
-          else
-            "#{@user}:#{params["realm"]}:#{@password}"
+            if probe_response.status == 401 && digest.can_authenticate?(probe_response.headers["www-authenticate"])
+              request.transition(:idle)
+              request.headers["authorization"] = digest.authenticate(request, probe_response.headers["www-authenticate"])
+              super(request)
+            else
+              probe_response
+            end
           end
-
-          ha1 = algorithm.hexdigest(a1)
-          ha2 = algorithm.hexdigest("#{meth}:#{uri}")
-          request_digest = [ha1, nonce]
-          request_digest.push(nc, cnonce, qop) if qop
-          request_digest << ha2
-          request_digest = request_digest.join(":")
-
-          header = [
-            %(username="#{@user}"),
-            %(nonce="#{nonce}"),
-            %(uri="#{uri}"),
-            %(response="#{algorithm.hexdigest(request_digest)}"),
-          ]
-          header << %(realm="#{params["realm"]}") if params.key?("realm")
-          header << %(algorithm=#{params["algorithm"]}") if params.key?("algorithm")
-          header << %(opaque="#{params["opaque"]}") if params.key?("opaque")
-          header << %(cnonce="#{cnonce}") if cnonce
-          header << %(nc=#{nc})
-          header << %(qop=#{qop}) if qop
-          header.join ", "
-        end
-
-        private
-
-        def make_cnonce
-          ::Digest::MD5.hexdigest [
-            Time.now.to_i,
-            Process.pid,
-            SecureRandom.random_number(2**32),
-          ].join ":"
-        end
-
-        def next_nonce
-          @nonce += 1
         end
       end
     end
 
-    register_plugin :digest_authentication, DigestAuthentication
+    register_plugin :digest_authentication, DigestAuth
   end
 end

data/lib/httpx/plugins/follow_redirects.rb

@@ -44,7 +44,7 @@ module HTTPX
 
           max_redirects = redirect_request.max_redirects
 
-          return response unless REDIRECT_STATUS.include?(response.status)
+          return response unless REDIRECT_STATUS.include?(response.status) && response.headers.key?("location")
           return response unless max_redirects.positive?
 
           retry_request = build_redirect_request(redirect_request, response, options)
@@ -86,10 +86,22 @@ module HTTPX
           redirect_uri = __get_location_from_response(response)
           max_redirects = request.max_redirects
 
-          # redirects are **ALWAYS** GET
-          retry_options = options.merge(headers: request.headers,
-                                        body: request.body,
-                                        max_redirects: max_redirects - 1)
+          if response.status == 305 && options.respond_to?(:proxy)
+            # The requested resource MUST be accessed through the proxy given by
+            # the Location field. The Location field gives the URI of the proxy.
+            retry_options = options.merge(headers: request.headers,
+                                          proxy: { uri: redirect_uri },
+                                          body: request.body,
+                                          max_redirects: max_redirects - 1)
+            redirect_uri = request.url
+          else
+
+            # redirects are **ALWAYS** GET
+            retry_options = options.merge(headers: request.headers,
+                                          body: request.body,
+                                          max_redirects: max_redirects - 1)
+          end
+
           build_request(:get, redirect_uri, retry_options)
         end