httpx 0.19.7 → 0.20.1

data/lib/httpx/plugins/authentication/socks5.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require "base64"
+
+module HTTPX
+  module Plugins
+    module Authentication
+      class Socks5
+        def initialize(user, password, **)
+          @user = user
+          @password = password
+        end
+
+        def can_authenticate?(*)
+          @user && @password
+        end
+
+        def authenticate(*)
+          [0x01, @user.bytesize, @user, @password.bytesize, @password].pack("CCA*CA*")
+        end
+      end
+    end
+  end
+end

data/lib/httpx/plugins/basic_authentication.rb

@@ -7,10 +7,10 @@ module HTTPX
     #
     # https://gitlab.com/honeyryderchuck/httpx/wikis/Authentication#basic-authentication
     #
-    module BasicAuthentication
+    module BasicAuth
       class << self
         def load_dependencies(_klass)
-          require "base64"
+          require_relative "authentication/basic"
         end
 
         def configure(klass)
@@ -19,12 +19,12 @@ module HTTPX
       end
 
       module InstanceMethods
-        def basic_authentication(user, password)
-          authentication("Basic #{Base64.strict_encode64("#{user}:#{password}")}")
+        def basic_auth(user, password)
+          authentication(Authentication::Basic.new(user, password).authenticate)
         end
-        alias_method :basic_auth, :basic_authentication
+        alias_method :basic_authentication, :basic_auth
       end
     end
-    register_plugin :basic_authentication, BasicAuthentication
+    register_plugin :basic_authentication, BasicAuth
   end
 end

data/lib/httpx/plugins/digest_authentication.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "digest"
-
 module HTTPX
   module Plugins
     #
@@ -9,9 +7,7 @@ module HTTPX
     #
     # https://gitlab.com/honeyryderchuck/httpx/wikis/Authentication#authentication
     #
-    module DigestAuthentication
-      using RegexpExtensions unless Regexp.method_defined?(:match?)
-
+    module DigestAuth
       DigestError = Class.new(Error)
 
       class << self
@@ -20,14 +16,13 @@ module HTTPX
         end
 
         def load_dependencies(*)
-          require "securerandom"
-          require "digest"
+          require_relative "authentication/digest"
         end
       end
 
       module OptionsMethods
         def option_digest(value)
-          raise TypeError, ":digest must be a Digest" unless value.is_a?(Digest)
+          raise TypeError, ":digest must be a Digest" unless value.is_a?(Authentication::Digest)
 
           value
         end
@@ -35,7 +30,7 @@ module HTTPX
 
       module InstanceMethods
         def digest_authentication(user, password)
-          with(digest: Digest.new(user, password))
+          with(digest: Authentication::Digest.new(user, password))
         end
 
         alias_method :digest_auth, :digest_authentication
@@ -44,116 +39,25 @@ module HTTPX
           requests.flat_map do |request|
             digest = request.options.digest
 
-            if digest
-              probe_response = wrap { super(request).first }
-
-              if digest && !probe_response.is_a?(ErrorResponse) &&
-                 probe_response.status == 401 && probe_response.headers.key?("www-authenticate") &&
-                 /Digest .*/.match?(probe_response.headers["www-authenticate"])
-
-                request.transition(:idle)
-
-                token = digest.generate_header(request, probe_response)
-                request.headers["authorization"] = "Digest #{token}"
-
-                super(request)
-              else
-                probe_response
-              end
-            else
+            unless digest
               super(request)
+              next
             end
-          end
-        end
-      end
-
-      class Digest
-        def initialize(user, password)
-          @user = user
-          @password = password
-          @nonce = 0
-        end
-
-        def generate_header(request, response, _iis = false)
-          meth = request.verb.to_s.upcase
-          www = response.headers["www-authenticate"]
-
-          # discard first token, it's Digest
-          auth_info = www[/^(\w+) (.*)/, 2]
-
-          uri = request.path
-
-          params = Hash[auth_info.split(/ *, */)
-                                 .map { |val| val.split("=") }
-                                 .map { |k, v| [k, v.delete("\"")] }]
-          nonce = params["nonce"]
-          nc = next_nonce
 
-          # verify qop
-          qop = params["qop"]
+            probe_response = wrap { super(request).first }
 
-          if params["algorithm"] =~ /(.*?)(-sess)?$/
-            alg = Regexp.last_match(1)
-            algorithm = ::Digest.const_get(alg)
-            raise DigestError, "unknown algorithm \"#{alg}\"" unless algorithm
-
-            sess = Regexp.last_match(2)
-            params.delete("algorithm")
-          else
-            algorithm = ::Digest::MD5
-          end
-
-          if qop || sess
-            cnonce = make_cnonce
-            nc = format("%<nonce>08x", nonce: nc)
-          end
-
-          a1 = if sess
-            [algorithm.hexdigest("#{@user}:#{params["realm"]}:#{@password}"),
-             nonce,
-             cnonce].join ":"
-          else
-            "#{@user}:#{params["realm"]}:#{@password}"
+            if probe_response.status == 401 && digest.can_authenticate?(probe_response.headers["www-authenticate"])
+              request.transition(:idle)
+              request.headers["authorization"] = digest.authenticate(request, probe_response.headers["www-authenticate"])
+              super(request)
+            else
+              probe_response
+            end
           end
-
-          ha1 = algorithm.hexdigest(a1)
-          ha2 = algorithm.hexdigest("#{meth}:#{uri}")
-          request_digest = [ha1, nonce]
-          request_digest.push(nc, cnonce, qop) if qop
-          request_digest << ha2
-          request_digest = request_digest.join(":")
-
-          header = [
-            %(username="#{@user}"),
-            %(nonce="#{nonce}"),
-            %(uri="#{uri}"),
-            %(response="#{algorithm.hexdigest(request_digest)}"),
-          ]
-          header << %(realm="#{params["realm"]}") if params.key?("realm")
-          header << %(algorithm=#{params["algorithm"]}") if params.key?("algorithm")
-          header << %(opaque="#{params["opaque"]}") if params.key?("opaque")
-          header << %(cnonce="#{cnonce}") if cnonce
-          header << %(nc=#{nc})
-          header << %(qop=#{qop}) if qop
-          header.join ", "
-        end
-
-        private
-
-        def make_cnonce
-          ::Digest::MD5.hexdigest [
-            Time.now.to_i,
-            Process.pid,
-            SecureRandom.random_number(2**32),
-          ].join ":"
-        end
-
-        def next_nonce
-          @nonce += 1
         end
       end
     end
 
-    register_plugin :digest_authentication, DigestAuthentication
+    register_plugin :digest_authentication, DigestAuth
   end
 end

data/lib/httpx/plugins/follow_redirects.rb

@@ -44,7 +44,7 @@ module HTTPX
 
           max_redirects = redirect_request.max_redirects
 
-          return response unless REDIRECT_STATUS.include?(response.status)
+          return response unless REDIRECT_STATUS.include?(response.status) && response.headers.key?("location")
           return response unless max_redirects.positive?
 
           retry_request = build_redirect_request(redirect_request, response, options)
@@ -86,10 +86,22 @@ module HTTPX
           redirect_uri = __get_location_from_response(response)
           max_redirects = request.max_redirects
 
-          # redirects are **ALWAYS** GET
-          retry_options = options.merge(headers: request.headers,
-                                        body: request.body,
-                                        max_redirects: max_redirects - 1)
+          if response.status == 305 && options.respond_to?(:proxy)
+            # The requested resource MUST be accessed through the proxy given by
+            # the Location field. The Location field gives the URI of the proxy.
+            retry_options = options.merge(headers: request.headers,
+                                          proxy: { uri: redirect_uri },
+                                          body: request.body,
+                                          max_redirects: max_redirects - 1)
+            redirect_uri = request.url
+          else
+
+            # redirects are **ALWAYS** GET
+            retry_options = options.merge(headers: request.headers,
+                                          body: request.body,
+                                          max_redirects: max_redirects - 1)
+          end
+
           build_request(:get, redirect_uri, retry_options)
         end
 

data/lib/httpx/plugins/ntlm_authentication.rb

@@ -5,13 +5,10 @@ module HTTPX
     #
     # https://gitlab.com/honeyryderchuck/httpx/wikis/Authentication#ntlm-authentication
     #
-    module NTLMAuthentication
-      NTLMParams = Struct.new(:user, :domain, :password)
-
+    module NTLMAuth
       class << self
         def load_dependencies(_klass)
-          require "base64"
-          require "ntlm"
+          require_relative "authentication/ntlm"
         end
 
         def extra_options(options)
@@ -21,7 +18,7 @@ module HTTPX
 
       module OptionsMethods
         def option_ntlm(value)
-          raise TypeError, ":ntlm must be a #{NTLMParams}" unless value.is_a?(NTLMParams)
+          raise TypeError, ":ntlm must be a #{Authentication::Ntlm}" unless value.is_a?(Authentication::Ntlm)
 
           value
         end
@@ -29,7 +26,7 @@ module HTTPX
 
       module InstanceMethods
         def ntlm_authentication(user, password, domain = nil)
-          with(ntlm: NTLMParams.new(user, domain, password))
+          with(ntlm: Authentication::Ntlm.new(user, password, domain: domain))
         end
 
         alias_method :ntlm_auth, :ntlm_authentication
@@ -39,19 +36,12 @@ module HTTPX
             ntlm = request.options.ntlm
 
             if ntlm
-              request.headers["authorization"] = "NTLM #{NTLM.negotiate(domain: ntlm.domain).to_base64}"
+              request.headers["authorization"] = ntlm.negotiate
               probe_response = wrap { super(request).first }
 
-              if !probe_response.is_a?(ErrorResponse) && probe_response.status == 401 &&
-                 probe_response.headers.key?("www-authenticate") &&
-                 (challenge = probe_response.headers["www-authenticate"][/NTLM (.*)/, 1])
-
-                challenge = Base64.decode64(challenge)
-                ntlm_challenge = NTLM.authenticate(challenge, ntlm.user, ntlm.domain, ntlm.password).to_base64
-
+              if probe_response.status == 401 && ntlm.can_authenticate?(probe_response.headers["www-authenticate"])
                 request.transition(:idle)
-
-                request.headers["authorization"] = "NTLM #{ntlm_challenge}"
+                request.headers["authorization"] = ntlm.authenticate(request, probe_response.headers["www-authenticate"])
                 super(request)
               else
                 probe_response
@@ -63,6 +53,6 @@ module HTTPX
         end
       end
     end
-    register_plugin :ntlm_authentication, NTLMAuthentication
+    register_plugin :ntlm_authentication, NTLMAuth
   end
 end

data/lib/httpx/plugins/proxy/http.rb

@@ -6,6 +6,42 @@ module HTTPX
   module Plugins
     module Proxy
       module HTTP
+        module InstanceMethods
+          def with_proxy_basic_auth(opts)
+            with(proxy: opts.merge(scheme: "basic"))
+          end
+
+          def with_proxy_digest_auth(opts)
+            with(proxy: opts.merge(scheme: "digest"))
+          end
+
+          def with_proxy_ntlm_auth(opts)
+            with(proxy: opts.merge(scheme: "ntlm"))
+          end
+
+          def fetch_response(request, connections, options)
+            response = super
+
+            if response &&
+               response.status == 407 &&
+               !request.headers.key?("proxy-authorization") &&
+               response.headers.key?("proxy-authenticate")
+
+              connection = find_connection(request, connections, options)
+
+              if connection.options.proxy.can_authenticate?(response.headers["proxy-authenticate"])
+                request.transition(:idle)
+                request.headers["proxy-authorization"] =
+                  connection.options.proxy.authenticate(request, response.headers["proxy-authenticate"])
+                connection.send(request)
+                return
+              end
+            end
+
+            response
+          end
+        end
+
         module ConnectionMethods
           def connecting?
             super || @state == :connecting || @state == :connected
@@ -23,11 +59,27 @@ module HTTPX
               @io.connect
               return unless @io.connected?
 
-              @parser = registry(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
-              @parser.extend(ProxyParser)
-              @parser.once(:response, &method(:__http_on_connect))
-              @parser.on(:close) { transition(:closing) }
-              __http_proxy_connect
+              @parser || begin
+                @parser = registry(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
+                parser = @parser
+                parser.extend(ProxyParser)
+                parser.on(:response, &method(:__http_on_connect))
+                parser.on(:close) { transition(:closing) }
+                parser.on(:reset) do
+                  if parser.empty?
+                    reset
+                  else
+                    transition(:closing)
+                    transition(:closed)
+                    emit(:reset)
+
+                    parser.reset if @parser
+                    transition(:idle)
+                    transition(:connecting)
+                  end
+                end
+                __http_proxy_connect(parser)
+              end
               return if @state == :connected
             when :connected
               return unless @state == :idle || @state == :connecting
@@ -44,13 +96,13 @@ module HTTPX
             super
           end
 
-          def __http_proxy_connect
+          def __http_proxy_connect(parser)
             req = @pending.first
-            # if the first request after CONNECT is to an https address, it is assumed that
-            # all requests in the queue are not only ALL HTTPS, but they also share the certificate,
-            # and therefore, will share the connection.
-            #
-            if req.uri.scheme == "https"
+            if req && req.uri.scheme == "https"
+              # if the first request after CONNECT is to an https address, it is assumed that
+              # all requests in the queue are not only ALL HTTPS, but they also share the certificate,
+              # and therefore, will share the connection.
+              #
               connect_request = ConnectRequest.new(req.uri, @options)
               @inflight += 1
               parser.send(connect_request)
@@ -59,7 +111,7 @@ module HTTPX
             end
           end
 
-          def __http_on_connect(_, response)
+          def __http_on_connect(request, response)
             @inflight -= 1
             if response.status == 200
               req = @pending.first
@@ -67,6 +119,14 @@ module HTTPX
               @io = ProxySSL.new(@io, request_uri, @options)
               transition(:connected)
               throw(:called)
+            elsif response.status == 407 &&
+                  !request.headers.key?("proxy-authorization") &&
+                  @options.proxy.can_authenticate?(response.headers["proxy-authenticate"])
+
+              request.transition(:idle)
+              request.headers["proxy-authorization"] = @options.proxy.authenticate(request, response.headers["proxy-authenticate"])
+              @parser.send(request)
+              @inflight += 1
             else
               pending = @pending + @parser.pending
               while (req = pending.shift)
@@ -88,7 +148,10 @@ module HTTPX
             extra_headers = super
 
             proxy_params = @options.proxy
-            extra_headers["proxy-authorization"] = "Basic #{proxy_params.token_authentication}" if proxy_params.authenticated?
+            if proxy_params.scheme == "basic"
+              # opt for basic auth
+              extra_headers["proxy-authorization"] = proxy_params.authenticate(extra_headers)
+            end
             extra_headers["proxy-connection"] = extra_headers.delete("connection") if extra_headers.key?("connection")
             extra_headers
           end

data/lib/httpx/plugins/proxy/socks5.rb

@@ -18,6 +18,10 @@ module HTTPX
 
         Error = Socks5Error
 
+        def self.load_dependencies(*)
+          require_relative "../authentication/socks5"
+        end
+
         module ConnectionMethods
           def call
             super
@@ -156,16 +160,14 @@ module HTTPX
 
           def negotiate(parameters)
             methods = [NOAUTH]
-            methods << PASSWD if parameters.authenticated?
+            methods << PASSWD if parameters.can_authenticate?
             methods.unshift(methods.size)
             methods.unshift(VERSION)
             methods.pack("C*")
           end
 
           def authenticate(parameters)
-            user = parameters.username
-            password = parameters.password
-            [0x01, user.bytesize, user, password.bytesize, password].pack("CCA*CA*")
+            parameters.authenticate
           end
 
           def connect(uri)

data/lib/httpx/plugins/proxy.rb

@@ -19,22 +19,43 @@ module HTTPX
       PROXY_ERRORS = [TimeoutError, IOError, SystemCallError, Error].freeze
 
       class Parameters
-        attr_reader :uri, :username, :password
+        attr_reader :uri, :username, :password, :scheme
 
-        def initialize(uri:, username: nil, password: nil)
+        def initialize(uri:, scheme: nil, username: nil, password: nil, **extra)
           @uri = uri.is_a?(URI::Generic) ? uri : URI(uri)
           @username = username || @uri.user
           @password = password || @uri.password
+
+          return unless @username && @password
+
+          scheme ||= case @uri.scheme
+                     when "socks5"
+                       @uri.scheme
+                     when "http", "https"
+                       "basic"
+                     else
+                       return
+          end
+
+          @scheme = scheme
+
+          auth_scheme = scheme.to_s.capitalize
+
+          require_relative "authentication/#{scheme}" unless defined?(Authentication) && Authentication.const_defined?(auth_scheme, false)
+
+          @authenticator = Authentication.const_get(auth_scheme).new(@username, @password, **extra)
         end
 
-        def authenticated?
-          @username && @password
+        def can_authenticate?(*args)
+          return false unless @authenticator
+
+          @authenticator.can_authenticate?(*args)
         end
 
-        def token_authentication
-          return unless authenticated?
+        def authenticate(*args)
+          return unless @authenticator
 
-          Base64.strict_encode64("#{@username}:#{@password}")
+          @authenticator.authenticate(*args)
         end
 
         def ==(other)
@@ -42,7 +63,8 @@ module HTTPX
           when Parameters
             @uri == other.uri &&
               @username == other.username &&
-              @password == other.password
+              @password == other.password &&
+              @scheme == other.scheme
           when URI::Generic, String
             proxy_uri = @uri.dup
             proxy_uri.user = @username

data/lib/httpx/plugins/response_cache/store.rb

@@ -37,6 +37,7 @@ module HTTPX::Plugins
             return unless request.headers.same_headers?(original_request.headers)
           else
             return unless vary.split(/ *, */).all? do |cache_field|
+              cache_field.downcase!
               !original_request.headers.key?(cache_field) || request.headers[cache_field] == original_request.headers[cache_field]
             end
           end

data/lib/httpx/pool.rb

@@ -80,6 +80,9 @@ module HTTPX
       connection.on(:activate) do
         select_connection(connection)
       end
+      connection.on(:close) do
+        unregister_connection(connection)
+      end
     end
 
     def deactivate(connections)
@@ -143,8 +146,6 @@ module HTTPX
 
     def on_resolver_error(connection, error)
       connection.emit(:error, error)
-      # must remove connection by hand, hasn't been started yet
-      unregister_connection(connection)
     end
 
     def on_resolver_close(resolver)
@@ -171,8 +172,7 @@ module HTTPX
 
     def unregister_connection(connection)
       @connections.delete(connection)
-      deselect_connection(connection)
-      @connected_connections -= 1
+      @connected_connections -= 1 if deselect_connection(connection)
     end
 
     def select_connection(connection)

data/lib/httpx/request.rb

@@ -49,7 +49,9 @@ module HTTPX
         origin = @options.origin
         raise(Error, "invalid URI: #{@uri}") unless origin
 
-        @uri = origin.merge(@uri)
+        base_path = @options.base_path
+
+        @uri = origin.merge("#{base_path}#{@uri}")
       end
 
       raise(Error, "unknown method: #{verb}") unless METHODS.include?(@verb)

data/lib/httpx/session.rb

@@ -106,16 +106,21 @@ module HTTPX
     end
 
     def build_altsvc_connection(existing_connection, connections, alt_origin, origin, alt_params, options)
+      # do not allow security downgrades on altsvc negotiation
+      return if existing_connection.origin.scheme == "https" && alt_origin.scheme != "https"
+
       altsvc = AltSvc.cached_altsvc_set(origin, alt_params.merge("origin" => alt_origin))
 
       # altsvc already exists, somehow it wasn't advertised, probably noop
       return unless altsvc
 
-      connection = pool.find_connection(alt_origin, options) || build_connection(alt_origin, options)
+      alt_options = options.merge(ssl: options.ssl.merge(hostname: URI(origin).host))
+
+      connection = pool.find_connection(alt_origin, alt_options) || build_connection(alt_origin, alt_options)
       # advertised altsvc is the same origin being used, ignore
       return if connection == existing_connection
 
-      set_connection_callbacks(connection, connections, options)
+      set_connection_callbacks(connection, connections, alt_options)
 
       log(level: 1) { "#{origin} alt-svc: #{alt_origin}" }
 

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "0.19.7"
+  VERSION = "0.20.1"
 end

data/sig/chainable.rbs

@@ -13,9 +13,9 @@ module HTTPX
             | (options) { (Session) -> void } -> void
 
     def plugin: (:authentication, ?options) -> Plugins::sessionAuthentication
-              | (:basic_authentication, ?options) -> Plugins::sessionBasicAuthentication
-              | (:digest_authentication, ?options) -> Plugins::sessionDigestAuthentication
-              | (:ntlm_authentication, ?options) -> Plugins::sessionNTLMAuthentication
+              | (:basic_authentication, ?options) -> Plugins::sessionBasicAuth
+              | (:digest_authentication, ?options) -> Plugins::sessionDigestAuth
+              | (:ntlm_authentication, ?options) -> Plugins::sessionNTLMAuth
               | (:aws_sdk_authentication, ?options) -> Plugins::sessionAwsSdkAuthentication
               | (:compression, ?options) -> Session
               | (:cookies, ?options) -> Plugins::sessionCookies
@@ -25,7 +25,7 @@ module HTTPX
               | (:h2c, ?options) -> Session
               | (:multipart, ?options) -> Session
               | (:persistent, ?options) -> Plugins::sessionPersistent
-              | (:proxy, ?options) -> Plugins::sessionProxy
+              | (:proxy, ?options) -> (Plugins::sessionProxy & Plugins::httpProxy)
               | (:push_promise, ?options) -> Plugins::sessionPushPromise
               | (:retries, ?options) -> Plugins::sessionRetries
               | (:rate_limiter, ?options) -> Session

data/sig/plugins/authentication/basic.rbs

@@ -0,0 +1,19 @@
+module HTTPX
+  module Plugins
+    module Authentication
+      class Basic
+        @user: String
+        @password: String
+
+        def can_authenticate?: (String? authenticate) -> boolish
+
+        def authenticate: (*untyped) -> String
+
+        private
+
+        def initialize: (string user, string password, *untyped) -> void
+
+      end
+    end
+  end
+end