httpx 0.19.8 → 0.20.2

data/lib/httpx/plugins/follow_redirects.rb

@@ -44,7 +44,7 @@ module HTTPX
 
           max_redirects = redirect_request.max_redirects
 
-          return response unless REDIRECT_STATUS.include?(response.status)
+          return response unless REDIRECT_STATUS.include?(response.status) && response.headers.key?("location")
           return response unless max_redirects.positive?
 
           retry_request = build_redirect_request(redirect_request, response, options)
@@ -86,10 +86,22 @@ module HTTPX
           redirect_uri = __get_location_from_response(response)
           max_redirects = request.max_redirects
 
-          # redirects are **ALWAYS** GET
-          retry_options = options.merge(headers: request.headers,
-                                        body: request.body,
-                                        max_redirects: max_redirects - 1)
+          if response.status == 305 && options.respond_to?(:proxy)
+            # The requested resource MUST be accessed through the proxy given by
+            # the Location field. The Location field gives the URI of the proxy.
+            retry_options = options.merge(headers: request.headers,
+                                          proxy: { uri: redirect_uri },
+                                          body: request.body,
+                                          max_redirects: max_redirects - 1)
+            redirect_uri = request.url
+          else
+
+            # redirects are **ALWAYS** GET
+            retry_options = options.merge(headers: request.headers,
+                                          body: request.body,
+                                          max_redirects: max_redirects - 1)
+          end
+
           build_request(:get, redirect_uri, retry_options)
         end
 

data/lib/httpx/plugins/ntlm_authentication.rb

@@ -5,13 +5,10 @@ module HTTPX
     #
     # https://gitlab.com/honeyryderchuck/httpx/wikis/Authentication#ntlm-authentication
     #
-    module NTLMAuthentication
-      NTLMParams = Struct.new(:user, :domain, :password)
-
+    module NTLMAuth
       class << self
         def load_dependencies(_klass)
-          require "base64"
-          require "ntlm"
+          require_relative "authentication/ntlm"
         end
 
         def extra_options(options)
@@ -21,7 +18,7 @@ module HTTPX
 
       module OptionsMethods
         def option_ntlm(value)
-          raise TypeError, ":ntlm must be a #{NTLMParams}" unless value.is_a?(NTLMParams)
+          raise TypeError, ":ntlm must be a #{Authentication::Ntlm}" unless value.is_a?(Authentication::Ntlm)
 
           value
         end
@@ -29,7 +26,7 @@ module HTTPX
 
       module InstanceMethods
         def ntlm_authentication(user, password, domain = nil)
-          with(ntlm: NTLMParams.new(user, domain, password))
+          with(ntlm: Authentication::Ntlm.new(user, password, domain: domain))
         end
 
         alias_method :ntlm_auth, :ntlm_authentication
@@ -39,19 +36,12 @@ module HTTPX
             ntlm = request.options.ntlm
 
             if ntlm
-              request.headers["authorization"] = "NTLM #{NTLM.negotiate(domain: ntlm.domain).to_base64}"
+              request.headers["authorization"] = ntlm.negotiate
               probe_response = wrap { super(request).first }
 
-              if !probe_response.is_a?(ErrorResponse) && probe_response.status == 401 &&
-                 probe_response.headers.key?("www-authenticate") &&
-                 (challenge = probe_response.headers["www-authenticate"][/NTLM (.*)/, 1])
-
-                challenge = Base64.decode64(challenge)
-                ntlm_challenge = NTLM.authenticate(challenge, ntlm.user, ntlm.domain, ntlm.password).to_base64
-
+              if probe_response.status == 401 && ntlm.can_authenticate?(probe_response.headers["www-authenticate"])
                 request.transition(:idle)
-
-                request.headers["authorization"] = "NTLM #{ntlm_challenge}"
+                request.headers["authorization"] = ntlm.authenticate(request, probe_response.headers["www-authenticate"])
                 super(request)
               else
                 probe_response
@@ -63,6 +53,6 @@ module HTTPX
         end
       end
     end
-    register_plugin :ntlm_authentication, NTLMAuthentication
+    register_plugin :ntlm_authentication, NTLMAuth
   end
 end

data/lib/httpx/plugins/proxy/http.rb

@@ -6,6 +6,42 @@ module HTTPX
   module Plugins
     module Proxy
       module HTTP
+        module InstanceMethods
+          def with_proxy_basic_auth(opts)
+            with(proxy: opts.merge(scheme: "basic"))
+          end
+
+          def with_proxy_digest_auth(opts)
+            with(proxy: opts.merge(scheme: "digest"))
+          end
+
+          def with_proxy_ntlm_auth(opts)
+            with(proxy: opts.merge(scheme: "ntlm"))
+          end
+
+          def fetch_response(request, connections, options)
+            response = super
+
+            if response &&
+               response.status == 407 &&
+               !request.headers.key?("proxy-authorization") &&
+               response.headers.key?("proxy-authenticate")
+
+              connection = find_connection(request, connections, options)
+
+              if connection.options.proxy.can_authenticate?(response.headers["proxy-authenticate"])
+                request.transition(:idle)
+                request.headers["proxy-authorization"] =
+                  connection.options.proxy.authenticate(request, response.headers["proxy-authenticate"])
+                connection.send(request)
+                return
+              end
+            end
+
+            response
+          end
+        end
+
         module ConnectionMethods
           def connecting?
             super || @state == :connecting || @state == :connected
@@ -23,11 +59,27 @@ module HTTPX
               @io.connect
               return unless @io.connected?
 
-              @parser = registry(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
-              @parser.extend(ProxyParser)
-              @parser.once(:response, &method(:__http_on_connect))
-              @parser.on(:close) { transition(:closing) }
-              __http_proxy_connect
+              @parser || begin
+                @parser = registry(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
+                parser = @parser
+                parser.extend(ProxyParser)
+                parser.on(:response, &method(:__http_on_connect))
+                parser.on(:close) { transition(:closing) }
+                parser.on(:reset) do
+                  if parser.empty?
+                    reset
+                  else
+                    transition(:closing)
+                    transition(:closed)
+                    emit(:reset)
+
+                    parser.reset if @parser
+                    transition(:idle)
+                    transition(:connecting)
+                  end
+                end
+                __http_proxy_connect(parser)
+              end
               return if @state == :connected
             when :connected
               return unless @state == :idle || @state == :connecting
@@ -44,13 +96,13 @@ module HTTPX
             super
           end
 
-          def __http_proxy_connect
+          def __http_proxy_connect(parser)
             req = @pending.first
-            # if the first request after CONNECT is to an https address, it is assumed that
-            # all requests in the queue are not only ALL HTTPS, but they also share the certificate,
-            # and therefore, will share the connection.
-            #
-            if req.uri.scheme == "https"
+            if req && req.uri.scheme == "https"
+              # if the first request after CONNECT is to an https address, it is assumed that
+              # all requests in the queue are not only ALL HTTPS, but they also share the certificate,
+              # and therefore, will share the connection.
+              #
               connect_request = ConnectRequest.new(req.uri, @options)
               @inflight += 1
               parser.send(connect_request)
@@ -59,7 +111,7 @@ module HTTPX
             end
           end
 
-          def __http_on_connect(_, response)
+          def __http_on_connect(request, response)
             @inflight -= 1
             if response.status == 200
               req = @pending.first
@@ -67,6 +119,14 @@ module HTTPX
               @io = ProxySSL.new(@io, request_uri, @options)
               transition(:connected)
               throw(:called)
+            elsif response.status == 407 &&
+                  !request.headers.key?("proxy-authorization") &&
+                  @options.proxy.can_authenticate?(response.headers["proxy-authenticate"])
+
+              request.transition(:idle)
+              request.headers["proxy-authorization"] = @options.proxy.authenticate(request, response.headers["proxy-authenticate"])
+              @parser.send(request)
+              @inflight += 1
             else
               pending = @pending + @parser.pending
               while (req = pending.shift)
@@ -88,7 +148,10 @@ module HTTPX
             extra_headers = super
 
             proxy_params = @options.proxy
-            extra_headers["proxy-authorization"] = "Basic #{proxy_params.token_authentication}" if proxy_params.authenticated?
+            if proxy_params.scheme == "basic"
+              # opt for basic auth
+              extra_headers["proxy-authorization"] = proxy_params.authenticate(extra_headers)
+            end
             extra_headers["proxy-connection"] = extra_headers.delete("connection") if extra_headers.key?("connection")
             extra_headers
           end

data/lib/httpx/plugins/proxy/socks5.rb

@@ -18,6 +18,10 @@ module HTTPX
 
         Error = Socks5Error
 
+        def self.load_dependencies(*)
+          require_relative "../authentication/socks5"
+        end
+
         module ConnectionMethods
           def call
             super
@@ -156,16 +160,14 @@ module HTTPX
 
           def negotiate(parameters)
             methods = [NOAUTH]
-            methods << PASSWD if parameters.authenticated?
+            methods << PASSWD if parameters.can_authenticate?
             methods.unshift(methods.size)
             methods.unshift(VERSION)
             methods.pack("C*")
           end
 
           def authenticate(parameters)
-            user = parameters.username
-            password = parameters.password
-            [0x01, user.bytesize, user, password.bytesize, password].pack("CCA*CA*")
+            parameters.authenticate
           end
 
           def connect(uri)

data/lib/httpx/plugins/proxy.rb

@@ -19,22 +19,43 @@ module HTTPX
       PROXY_ERRORS = [TimeoutError, IOError, SystemCallError, Error].freeze
 
       class Parameters
-        attr_reader :uri, :username, :password
+        attr_reader :uri, :username, :password, :scheme
 
-        def initialize(uri:, username: nil, password: nil)
+        def initialize(uri:, scheme: nil, username: nil, password: nil, **extra)
           @uri = uri.is_a?(URI::Generic) ? uri : URI(uri)
           @username = username || @uri.user
           @password = password || @uri.password
+
+          return unless @username && @password
+
+          scheme ||= case @uri.scheme
+                     when "socks5"
+                       @uri.scheme
+                     when "http", "https"
+                       "basic"
+                     else
+                       return
+          end
+
+          @scheme = scheme
+
+          auth_scheme = scheme.to_s.capitalize
+
+          require_relative "authentication/#{scheme}" unless defined?(Authentication) && Authentication.const_defined?(auth_scheme, false)
+
+          @authenticator = Authentication.const_get(auth_scheme).new(@username, @password, **extra)
         end
 
-        def authenticated?
-          @username && @password
+        def can_authenticate?(*args)
+          return false unless @authenticator
+
+          @authenticator.can_authenticate?(*args)
         end
 
-        def token_authentication
-          return unless authenticated?
+        def authenticate(*args)
+          return unless @authenticator
 
-          Base64.strict_encode64("#{@username}:#{@password}")
+          @authenticator.authenticate(*args)
         end
 
         def ==(other)
@@ -42,7 +63,8 @@ module HTTPX
           when Parameters
             @uri == other.uri &&
               @username == other.username &&
-              @password == other.password
+              @password == other.password &&
+              @scheme == other.scheme
           when URI::Generic, String
             proxy_uri = @uri.dup
             proxy_uri.user = @username

data/lib/httpx/plugins/response_cache/store.rb

@@ -37,6 +37,7 @@ module HTTPX::Plugins
             return unless request.headers.same_headers?(original_request.headers)
           else
             return unless vary.split(/ *, */).all? do |cache_field|
+              cache_field.downcase!
               !original_request.headers.key?(cache_field) || request.headers[cache_field] == original_request.headers[cache_field]
             end
           end

data/lib/httpx/pool.rb

@@ -80,6 +80,9 @@ module HTTPX
       connection.on(:activate) do
         select_connection(connection)
       end
+      connection.on(:close) do
+        unregister_connection(connection)
+      end
     end
 
     def deactivate(connections)
@@ -143,8 +146,6 @@ module HTTPX
 
     def on_resolver_error(connection, error)
       connection.emit(:error, error)
-      # must remove connection by hand, hasn't been started yet
-      unregister_connection(connection)
     end
 
     def on_resolver_close(resolver)
@@ -171,8 +172,7 @@ module HTTPX
 
     def unregister_connection(connection)
       @connections.delete(connection)
-      deselect_connection(connection)
-      @connected_connections -= 1
+      @connected_connections -= 1 if deselect_connection(connection)
     end
 
     def select_connection(connection)

data/lib/httpx/request.rb

@@ -49,7 +49,9 @@ module HTTPX
         origin = @options.origin
         raise(Error, "invalid URI: #{@uri}") unless origin
 
-        @uri = origin.merge(@uri)
+        base_path = @options.base_path
+
+        @uri = origin.merge("#{base_path}#{@uri}")
       end
 
       raise(Error, "unknown method: #{verb}") unless METHODS.include?(@verb)

data/lib/httpx/resolver/native.rb

@@ -118,6 +118,10 @@ module HTTPX
       @timeouts.values_at(*hosts).reject(&:empty?).map(&:first).min
     end
 
+    def raise_timeout_error(interval)
+      do_retry(interval)
+    end
+
     private
 
     def calculate_interests
@@ -134,10 +138,10 @@ module HTTPX
       dwrite if calculate_interests == :w
     end
 
-    def do_retry
+    def do_retry(loop_time = nil)
       return if @queries.empty? || !@start_timeout
 
-      loop_time = Utils.elapsed_time(@start_timeout)
+      loop_time ||= Utils.elapsed_time(@start_timeout)
 
       query = @queries.first
 

data/lib/httpx/selector.rb

@@ -74,7 +74,10 @@ class HTTPX::Selector
 
       readers, writers = IO.select(r, w, nil, interval)
 
-      raise HTTPX::TimeoutError.new(interval, "timed out while waiting on select") if readers.nil? && writers.nil? && interval
+      if readers.nil? && writers.nil? && interval
+        [*r, *w].each { |io| io.raise_timeout_error(interval) }
+        return
+      end
     rescue IOError, SystemCallError
       @selectables.reject!(&:closed?)
       retry
@@ -108,7 +111,11 @@ class HTTPX::Selector
              when nil then return
     end
 
-    raise HTTPX::TimeoutError.new(interval, "timed out while waiting on select") unless result || interval.nil?
+    unless result || interval.nil?
+      io.raise_timeout_error(interval)
+      return
+    end
+    # raise HTTPX::TimeoutError.new(interval, "timed out while waiting on select")
 
     yield io
   rescue IOError, SystemCallError

data/lib/httpx/session.rb

@@ -106,16 +106,21 @@ module HTTPX
     end
 
     def build_altsvc_connection(existing_connection, connections, alt_origin, origin, alt_params, options)
+      # do not allow security downgrades on altsvc negotiation
+      return if existing_connection.origin.scheme == "https" && alt_origin.scheme != "https"
+
       altsvc = AltSvc.cached_altsvc_set(origin, alt_params.merge("origin" => alt_origin))
 
       # altsvc already exists, somehow it wasn't advertised, probably noop
       return unless altsvc
 
-      connection = pool.find_connection(alt_origin, options) || build_connection(alt_origin, options)
+      alt_options = options.merge(ssl: options.ssl.merge(hostname: URI(origin).host))
+
+      connection = pool.find_connection(alt_origin, alt_options) || build_connection(alt_origin, alt_options)
       # advertised altsvc is the same origin being used, ignore
       return if connection == existing_connection
 
-      set_connection_callbacks(connection, connections, options)
+      set_connection_callbacks(connection, connections, alt_options)
 
       log(level: 1) { "#{origin} alt-svc: #{alt_origin}" }
 

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "0.19.8"
+  VERSION = "0.20.2"
 end

data/sig/chainable.rbs

@@ -13,9 +13,9 @@ module HTTPX
             | (options) { (Session) -> void } -> void
 
     def plugin: (:authentication, ?options) -> Plugins::sessionAuthentication
-              | (:basic_authentication, ?options) -> Plugins::sessionBasicAuthentication
-              | (:digest_authentication, ?options) -> Plugins::sessionDigestAuthentication
-              | (:ntlm_authentication, ?options) -> Plugins::sessionNTLMAuthentication
+              | (:basic_authentication, ?options) -> Plugins::sessionBasicAuth
+              | (:digest_authentication, ?options) -> Plugins::sessionDigestAuth
+              | (:ntlm_authentication, ?options) -> Plugins::sessionNTLMAuth
               | (:aws_sdk_authentication, ?options) -> Plugins::sessionAwsSdkAuthentication
               | (:compression, ?options) -> Session
               | (:cookies, ?options) -> Plugins::sessionCookies
@@ -25,7 +25,7 @@ module HTTPX
               | (:h2c, ?options) -> Session
               | (:multipart, ?options) -> Session
               | (:persistent, ?options) -> Plugins::sessionPersistent
-              | (:proxy, ?options) -> Plugins::sessionProxy
+              | (:proxy, ?options) -> (Plugins::sessionProxy & Plugins::httpProxy)
               | (:push_promise, ?options) -> Plugins::sessionPushPromise
               | (:retries, ?options) -> Plugins::sessionRetries
               | (:rate_limiter, ?options) -> Session

data/sig/connection.rbs

@@ -72,6 +72,9 @@ module HTTPX
     def timeout: () -> Numeric?
 
     def deactivate: () -> void
+
+    def raise_timeout_error: (Numeric interval) -> void
+
     private
 
     def initialize: (String, URI::Generic, options) -> untyped

data/sig/plugins/authentication/basic.rbs

@@ -0,0 +1,19 @@
+module HTTPX
+  module Plugins
+    module Authentication
+      class Basic
+        @user: String
+        @password: String
+
+        def can_authenticate?: (String? authenticate) -> boolish
+
+        def authenticate: (*untyped) -> String
+
+        private
+
+        def initialize: (string user, string password, *untyped) -> void
+
+      end
+    end
+  end
+end

data/sig/plugins/authentication/digest.rbs

@@ -0,0 +1,24 @@
+module HTTPX
+  module Plugins
+    module Authentication
+      class Digest
+        @user: String
+        @password: String
+
+        def can_authenticate?: (String? authenticate) -> boolish
+
+        def authenticate: (Request request, String authenticate) -> String
+
+        private
+
+        def generate_header: (String meth, String uri, String authenticate) -> String
+
+        def initialize: (string user, string password) -> void
+
+        def make_cnonce: () -> String
+
+        def next_nonce: () -> Integer
+      end
+    end
+  end
+end

data/sig/plugins/authentication/ntlm.rbs

@@ -0,0 +1,20 @@
+module HTTPX
+  module Plugins
+    module Authentication
+      class Ntlm
+        @user: String
+        @password: String
+        @domain: String?
+
+        def can_authenticate?: (String? authenticate) -> boolish
+
+        def authenticate: (Request request, String authenticate) -> String
+
+        private
+
+        def initialize: (string user, string password, ?domain: String?) -> void
+
+      end
+    end
+  end
+end

data/sig/plugins/authentication/socks5.rbs

@@ -0,0 +1,18 @@
+module HTTPX
+  module Plugins
+    module Authentication
+      class Socks5
+        @user: String
+        @password: String
+
+        def can_authenticate?: (*untyped) -> boolish
+
+        def authenticate: (*untyped) -> String
+
+        private
+
+        def initialize: (string user, string password) -> void
+      end
+    end
+  end
+end

data/sig/plugins/basic_authentication.rbs

@@ -1,6 +1,6 @@
 module HTTPX
   module Plugins
-    module BasicAuthentication
+    module BasicAuth
       def self.load_dependencies: (singleton(Session)) -> void
 
       def self.configure: (singleton(Session)) -> void
@@ -10,6 +10,6 @@ module HTTPX
       end
     end
 
-    type sessionBasicAuthentication = sessionAuthentication & Authentication::InstanceMethods & BasicAuthentication::InstanceMethods
+    type sessionBasicAuth = sessionAuthentication & Authentication::InstanceMethods & BasicAuth::InstanceMethods
   end
 end

data/sig/plugins/digest_authentication.rbs

@@ -1,10 +1,10 @@
 module HTTPX
   module Plugins
-    module DigestAuthentication
+    module DigestAuth
       DigestError: singleton(Error)
 
       interface _DigestOptions
-        def digest: () -> Digest?
+        def digest: () -> Authentication::Digest?
       end
 
       def self.extra_options: (Options) -> (Options & _DigestOptions)
@@ -14,18 +14,8 @@ module HTTPX
       module InstanceMethods
         def digest_authentication: (string user, string password) -> instance
       end
-
-      class Digest
-        def generate_header: (Request, Response, ?bool?) -> String
-
-        private
-
-        def initialize: (string user, string password) -> untyped
-        def make_cnonce: () -> String
-        def next_nonce: () -> Integer
-      end
     end
 
-    type sessionDigestAuthentication = sessionAuthentication & DigestAuthentication::InstanceMethods
+    type sessionDigestAuth = sessionAuthentication & DigestAuth::InstanceMethods
   end
 end

data/sig/plugins/ntlm_authentication.rbs

@@ -1,9 +1,9 @@
 module HTTPX
   module Plugins
-    module NTLMAuthentication
+    module NTLMAuth
 
       interface _NTLMOptions
-        def ntlm: () -> NTLMParams?
+        def ntlm: () -> Authentication::Ntlm?
       end
 
       def self.extra_options: (Options) -> (Options & _NTLMOptions)
@@ -14,13 +14,8 @@ module HTTPX
         def ntlm_authentication: (string user, string password, ?string? domain) -> instance
       end
 
-      class NTLMParams
-        attr_reader user: String
-        attr_reader password: String
-        attr_reader domain: String?
-      end
     end
 
-    type sessionNTLMAuthentication = sessionAuthentication & NTLMAuthentication::InstanceMethods
+    type sessionNTLMAuth = sessionAuthentication & NTLMAuth::InstanceMethods
   end
 end