httpx 0.21.0 → 1.2.1

data/lib/httpx/plugins/internal_telemetry.rb

@@ -32,6 +32,15 @@ module HTTPX
         end
       end
 
+      module NativeResolverMethods
+        def transition(nextstate)
+          state = @state
+          val = super
+          meter_elapsed_time("Resolver::Native: #{state} -> #{nextstate}")
+          val
+        end
+      end
+
       module InstanceMethods
         def self.included(klass)
           klass.prepend TrackTimeMethods
@@ -42,6 +51,13 @@ module HTTPX
           meter_elapsed_time("Session: initializing...")
           super
           meter_elapsed_time("Session: initialized!!!")
+          resolver_type = @options.resolver_class
+          resolver_type = Resolver.resolver_for(resolver_type)
+          return unless resolver_type <= Resolver::Native
+
+          resolver_type.prepend TrackTimeMethods
+          resolver_type.prepend NativeResolverMethods
+          @options = @options.merge(resolver_class: resolver_type)
         end
 
         def close(*)

data/lib/httpx/plugins/{ntlm_authentication.rb → ntlm_auth.rb}

@@ -3,12 +3,12 @@
 module HTTPX
   module Plugins
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Authentication#ntlm-authentication
+    # https://gitlab.com/os85/httpx/wikis/Auth#ntlm-auth
     #
     module NTLMAuth
       class << self
         def load_dependencies(_klass)
-          require_relative "authentication/ntlm"
+          require_relative "auth/ntlm"
         end
 
         def extra_options(options)
@@ -25,11 +25,11 @@ module HTTPX
       end
 
       module InstanceMethods
-        def ntlm_authentication(user, password, domain = nil)
+        def ntlm_auth(user, password, domain = nil)
           with(ntlm: Authentication::Ntlm.new(user, password, domain: domain))
         end
 
-        alias_method :ntlm_auth, :ntlm_authentication
+        private
 
         def send_requests(*requests)
           requests.flat_map do |request|
@@ -39,6 +39,8 @@ module HTTPX
               request.headers["authorization"] = ntlm.negotiate
               probe_response = wrap { super(request).first }
 
+              return probe_response unless probe_response.is_a?(Response)
+
               if probe_response.status == 401 && ntlm.can_authenticate?(probe_response.headers["www-authenticate"])
                 request.transition(:idle)
                 request.headers["authorization"] = ntlm.authenticate(request, probe_response.headers["www-authenticate"])
@@ -53,6 +55,6 @@ module HTTPX
         end
       end
     end
-    register_plugin :ntlm_authentication, NTLMAuth
+    register_plugin :ntlm_auth, NTLMAuth
   end
 end

data/lib/httpx/plugins/oauth.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # https://gitlab.com/os85/httpx/wikis/OAuth
+    #
+    module OAuth
+      class << self
+        def load_dependencies(_klass)
+          require_relative "auth/basic"
+        end
+      end
+
+      SUPPORTED_GRANT_TYPES = %w[client_credentials refresh_token].freeze
+      SUPPORTED_AUTH_METHODS = %w[client_secret_basic client_secret_post].freeze
+
+      class OAuthSession
+        attr_reader :token_endpoint_auth_method, :grant_type, :client_id, :client_secret, :access_token, :refresh_token, :scope
+
+        def initialize(
+          issuer:,
+          client_id:,
+          client_secret:,
+          access_token: nil,
+          refresh_token: nil,
+          scope: nil,
+          token_endpoint: nil,
+          response_type: nil,
+          grant_type: nil,
+          token_endpoint_auth_method: "client_secret_basic"
+        )
+          @issuer = URI(issuer)
+          @client_id = client_id
+          @client_secret = client_secret
+          @token_endpoint = URI(token_endpoint) if token_endpoint
+          @response_type = response_type
+          @scope = case scope
+                   when String
+                     scope.split
+                   when Array
+                     scope
+          end
+          @access_token = access_token
+          @refresh_token = refresh_token
+          @token_endpoint_auth_method = String(token_endpoint_auth_method)
+          @grant_type = grant_type || (@refresh_token ? "refresh_token" : "client_credentials")
+
+          unless SUPPORTED_AUTH_METHODS.include?(@token_endpoint_auth_method)
+            raise Error, "#{@token_endpoint_auth_method} is not a supported auth method"
+          end
+
+          return if SUPPORTED_GRANT_TYPES.include?(@grant_type)
+
+          raise Error, "#{@grant_type} is not a supported grant type"
+        end
+
+        def token_endpoint
+          @token_endpoint || "#{@issuer}/token"
+        end
+
+        def load(http)
+          return unless @token_endpoint && @token_endpoint_auth_method && @grant_type && @scope
+
+          metadata = http.get("#{issuer}/.well-known/oauth-authorization-server").raise_for_status.json
+
+          @token_endpoint = metadata["token_endpoint"]
+          @scope = metadata["scopes_supported"]
+          @grant_type = Array(metadata["grant_types_supported"]).find { |gr| SUPPORTED_GRANT_TYPES.include?(gr) }
+          @token_endpoint_auth_method = Array(metadata["token_endpoint_auth_methods_supported"]).find do |am|
+            SUPPORTED_AUTH_METHODS.include?(am)
+          end
+        end
+
+        def merge(other)
+          obj = dup
+
+          case other
+          when OAuthSession
+            other.instance_variables.each do |ivar|
+              val = other.instance_variable_get(ivar)
+              next unless val
+
+              obj.instance_variable_set(ivar, val)
+            end
+          when Hash
+            other.each do |k, v|
+              obj.instance_variable_set(:"@#{k}", v) if obj.instance_variable_defined?(:"@#{k}")
+            end
+          end
+          obj
+        end
+      end
+
+      module OptionsMethods
+        def option_oauth_session(value)
+          case value
+          when Hash
+            OAuthSession.new(**value)
+          when OAuthSession
+            value
+          else
+            raise TypeError, ":oauth_session must be a #{OAuthSession}"
+          end
+        end
+      end
+
+      module InstanceMethods
+        def oauth_auth(**args)
+          with(oauth_session: OAuthSession.new(**args))
+        end
+
+        def with_access_token
+          oauth_session = @options.oauth_session
+
+          oauth_session.load(self)
+
+          grant_type = oauth_session.grant_type
+
+          headers = {}
+          form_post = { "grant_type" => grant_type, "scope" => Array(oauth_session.scope).join(" ") }.compact
+
+          # auth
+          case oauth_session.token_endpoint_auth_method
+          when "client_secret_basic"
+            headers["authorization"] = Authentication::Basic.new(oauth_session.client_id, oauth_session.client_secret).authenticate
+          when "client_secret_post"
+            form_post["client_id"] = oauth_session.client_id
+            form_post["client_secret"] = oauth_session.client_secret
+          end
+
+          case grant_type
+          when "client_credentials"
+            # do nothing
+          when "refresh_token"
+            form_post["refresh_token"] = oauth_session.refresh_token
+          end
+
+          token_request = build_request("POST", oauth_session.token_endpoint, headers: headers, form: form_post)
+          token_request.headers.delete("authorization") unless oauth_session.token_endpoint_auth_method == "client_secret_basic"
+
+          token_response = request(token_request)
+          token_response.raise_for_status
+
+          payload = token_response.json
+
+          access_token = payload["access_token"]
+          refresh_token = payload["refresh_token"]
+
+          with(oauth_session: oauth_session.merge(access_token: access_token, refresh_token: refresh_token))
+        end
+
+        def build_request(*, _)
+          request = super
+
+          return request if request.headers.key?("authorization")
+
+          oauth_session = @options.oauth_session
+
+          return request unless oauth_session && oauth_session.access_token
+
+          request.headers["authorization"] = "Bearer #{oauth_session.access_token}"
+
+          request
+        end
+      end
+    end
+    register_plugin :oauth, OAuth
+  end
+end

data/lib/httpx/plugins/persistent.rb

@@ -15,7 +15,7 @@ module HTTPX
     # This plugin is also not recommendable when connecting to >9000 (like, a lot) different origins.
     # So when you use this, make sure that you don't fall into this trap.
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Persistent
+    # https://gitlab.com/os85/httpx/wikis/Persistent
     #
     module Persistent
       def self.load_dependencies(klass)

data/lib/httpx/plugins/proxy/http.rb

@@ -1,11 +1,15 @@
 # frozen_string_literal: true
 
-require "base64"
-
 module HTTPX
   module Plugins
     module Proxy
       module HTTP
+        class << self
+          def extra_options(options)
+            options.merge(supported_proxy_protocols: options.supported_proxy_protocols + %w[http])
+          end
+        end
+
         module InstanceMethods
           def with_proxy_basic_auth(opts)
             with(proxy: opts.merge(scheme: "basic"))
@@ -23,6 +27,7 @@ module HTTPX
             response = super
 
             if response &&
+               response.is_a?(Response) &&
                response.status == 407 &&
                !request.headers.key?("proxy-authorization") &&
                response.headers.key?("proxy-authenticate")
@@ -33,7 +38,7 @@ module HTTPX
                 request.transition(:idle)
                 request.headers["proxy-authorization"] =
                   connection.options.proxy.authenticate(request, response.headers["proxy-authenticate"])
-                connection.send(request)
+                send_request(request, connections)
                 return
               end
             end
@@ -60,7 +65,7 @@ module HTTPX
               return unless @io.connected?
 
               @parser || begin
-                @parser = registry(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
+                @parser = self.class.parser_type(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
                 parser = @parser
                 parser.extend(ProxyParser)
                 parser.on(:response, &method(:__http_on_connect))
@@ -71,7 +76,6 @@ module HTTPX
                   else
                     transition(:closing)
                     transition(:closed)
-                    emit(:reset)
 
                     parser.reset if @parser
                     transition(:idle)
@@ -113,13 +117,14 @@ module HTTPX
 
           def __http_on_connect(request, response)
             @inflight -= 1
-            if response.status == 200
+            if response.is_a?(Response) && response.status == 200
               req = @pending.first
               request_uri = req.uri
               @io = ProxySSL.new(@io, request_uri, @options)
               transition(:connected)
               throw(:called)
-            elsif response.status == 407 &&
+            elsif response.is_a?(Response) &&
+                  response.status == 407 &&
                   !request.headers.key?("proxy-authorization") &&
                   @options.proxy.can_authenticate?(response.headers["proxy-authenticate"])
 
@@ -139,9 +144,9 @@ module HTTPX
 
         module ProxyParser
           def join_headline(request)
-            return super if request.verb == :connect
+            return super if request.verb == "CONNECT"
 
-            "#{request.verb.to_s.upcase} #{request.uri} HTTP/#{@version.join(".")}"
+            "#{request.verb} #{request.uri} HTTP/#{@version.join(".")}"
           end
 
           def set_protocol_headers(request)
@@ -159,7 +164,7 @@ module HTTPX
 
         class ConnectRequest < Request
           def initialize(uri, _options)
-            super(:connect, uri, {})
+            super("CONNECT", uri, {})
             @headers.delete("accept")
           end
 

data/lib/httpx/plugins/proxy/socks4.rb

@@ -4,7 +4,7 @@ require "resolv"
 require "ipaddr"
 
 module HTTPX
-  class Socks4Error < Error; end
+  class Socks4Error < HTTPProxyError; end
 
   module Plugins
     module Proxy
@@ -16,6 +16,12 @@ module HTTPX
 
         Error = Socks4Error
 
+        class << self
+          def extra_options(options)
+            options.merge(supported_proxy_protocols: options.supported_proxy_protocols + PROTOCOLS)
+          end
+        end
+
         module ConnectionMethods
           def interests
             if @state == :connecting
@@ -79,17 +85,13 @@ module HTTPX
         end
 
         class SocksParser
-          include Callbacks
+          include HTTPX::Callbacks
 
           def initialize(buffer, options)
             @buffer = buffer
             @options = Options.new(options)
           end
 
-          def timeout
-            @options.timeout[:operation_timeout]
-          end
-
           def close; end
 
           def consume(*); end

data/lib/httpx/plugins/proxy/socks5.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module HTTPX
-  class Socks5Error < Error; end
+  class Socks5Error < HTTPProxyError; end
 
   module Plugins
     module Proxy
@@ -18,8 +18,14 @@ module HTTPX
 
         Error = Socks5Error
 
-        def self.load_dependencies(*)
-          require_relative "../authentication/socks5"
+        class << self
+          def load_dependencies(*)
+            require_relative "../auth/socks5"
+          end
+
+          def extra_options(options)
+            options.merge(supported_proxy_protocols: options.supported_proxy_protocols + %w[socks5])
+          end
         end
 
         module ConnectionMethods
@@ -131,17 +137,13 @@ module HTTPX
         end
 
         class SocksParser
-          include Callbacks
+          include HTTPX::Callbacks
 
           def initialize(buffer, options)
             @buffer = buffer
             @options = Options.new(options)
           end
 
-          def timeout
-            @options.timeout[:operation_timeout]
-          end
-
           def close; end
 
           def consume(*); end

data/lib/httpx/plugins/proxy.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module HTTPX
-  class HTTPProxyError < Error; end
+  class HTTPProxyError < ConnectionError; end
 
   module Plugins
     #
@@ -12,12 +12,24 @@ module HTTPX
     # * Socks4/4a proxies
     # * Socks5 proxies
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Proxy
+    # https://gitlab.com/os85/httpx/wikis/Proxy
     #
     module Proxy
       Error = HTTPProxyError
       PROXY_ERRORS = [TimeoutError, IOError, SystemCallError, Error].freeze
 
+      class << self
+        def configure(klass)
+          klass.plugin(:"proxy/http")
+          klass.plugin(:"proxy/socks4")
+          klass.plugin(:"proxy/socks5")
+        end
+
+        def extra_options(options)
+          options.merge(supported_proxy_protocols: [])
+        end
+      end
+
       class Parameters
         attr_reader :uri, :username, :password, :scheme
 
@@ -41,7 +53,7 @@ module HTTPX
 
           auth_scheme = scheme.to_s.capitalize
 
-          require_relative "authentication/#{scheme}" unless defined?(Authentication) && Authentication.const_defined?(auth_scheme, false)
+          require_relative "auth/#{scheme}" unless defined?(Authentication) && Authentication.const_defined?(auth_scheme, false)
 
           @authenticator = Authentication.const_get(auth_scheme).new(@username, @password, **extra)
         end
@@ -77,54 +89,61 @@ module HTTPX
         end
       end
 
-      class << self
-        def configure(klass)
-          klass.plugin(:"proxy/http")
-          klass.plugin(:"proxy/socks4")
-          klass.plugin(:"proxy/socks5")
-        end
-      end
-
       module OptionsMethods
         def option_proxy(value)
           value.is_a?(Parameters) ? value : Hash[value]
         end
+
+        def option_supported_proxy_protocols(value)
+          raise TypeError, ":supported_proxy_protocols must be an Array" unless value.is_a?(Array)
+
+          value.map(&:to_s)
+        end
       end
 
       module InstanceMethods
         private
 
-        def proxy_uris(uri, options)
-          @_proxy_uris ||= begin
-            uris = options.proxy ? Array(options.proxy[:uri]) : []
-            if uris.empty?
-              uri = URI(uri).find_proxy
-              uris << uri if uri
-            end
-            uris
-          end
-          return if @_proxy_uris.empty?
+        def find_connection(request, connections, options)
+          return super unless options.respond_to?(:proxy)
 
-          proxy = options.proxy
+          uri = URI(request.uri)
 
-          return { uri: uri.host } if proxy && proxy.key?(:no_proxy) && !Array(proxy[:no_proxy]).grep(uri.host).empty?
+          proxy_opts = if (next_proxy = uri.find_proxy)
+            { uri: next_proxy }
+          else
+            proxy = options.proxy
 
-          proxy_opts = { uri: @_proxy_uris.first }
-          proxy_opts = options.proxy.merge(proxy_opts) if options.proxy
-          proxy_opts
-        end
+            return super unless proxy
 
-        def find_connection(request, connections, options)
-          return super unless options.respond_to?(:proxy)
+            return super(request, connections, options.merge(proxy: nil)) unless proxy.key?(:uri)
 
-          uri = URI(request.uri)
-          next_proxy = proxy_uris(uri, options)
-          raise Error, "Failed to connect to proxy" unless next_proxy
+            @_proxy_uris ||= Array(proxy[:uri])
+
+            next_proxy = @_proxy_uris.first
+            raise Error, "Failed to connect to proxy" unless next_proxy
+
+            next_proxy = URI(next_proxy)
+
+            raise Error,
+                  "#{next_proxy.scheme}: unsupported proxy protocol" unless options.supported_proxy_protocols.include?(next_proxy.scheme)
 
-          proxy = Parameters.new(**next_proxy) unless next_proxy[:uri] == uri.host
+            if proxy.key?(:no_proxy)
+
+              no_proxy = proxy[:no_proxy]
+              no_proxy = no_proxy.join(",") if no_proxy.is_a?(Array)
+
+              return super(request, connections, options.merge(proxy: nil)) unless URI::Generic.use_proxy?(uri.host, next_proxy.host,
+                                                                                                           next_proxy.port, no_proxy)
+            end
+
+            proxy.merge(uri: next_proxy)
+          end
+
+          proxy = Parameters.new(**proxy_opts)
 
           proxy_options = options.merge(proxy: proxy)
-          connection = pool.find_connection(uri, proxy_options) || build_connection(uri, proxy_options)
+          connection = pool.find_connection(uri, proxy_options) || init_connection(uri, proxy_options)
           unless connections.nil? || connections.include?(connection)
             connections << connection
             set_connection_callbacks(connection, connections, options)
@@ -132,40 +151,24 @@ module HTTPX
           connection
         end
 
-        def build_connection(uri, options)
-          proxy = options.proxy
-          return super unless proxy
-
-          connection = options.connection_class.new("tcp", uri, options)
-          catch(:coalesced) do
-            pool.init_connection(connection, options)
-            connection
-          end
-        end
-
         def fetch_response(request, connections, options)
           response = super
 
-          if response.is_a?(ErrorResponse) &&
-             __proxy_error?(response) && !@_proxy_uris.empty?
+          if response.is_a?(ErrorResponse) && proxy_error?(request, response)
             @_proxy_uris.shift
+
+            # return last error response if no more proxies to try
+            return response if @_proxy_uris.empty?
+
             log { "failed connecting to proxy, trying next..." }
             request.transition(:idle)
-            connection = find_connection(request, connections, options)
-            connections << connection unless connections.include?(connection)
-            connection.send(request)
+            send_request(request, connections, options)
             return
           end
           response
         end
 
-        def build_altsvc_connection(_, _, _, _, _, options)
-          return if options.proxy
-
-          super
-        end
-
-        def __proxy_error?(response)
+        def proxy_error?(_request, response)
           error = response.error
           case error
           when NativeResolveError
@@ -222,13 +225,6 @@ module HTTPX
           end
         end
 
-        def send(request)
-          return super unless @options.proxy
-          return super unless connecting?
-
-          @pending << request
-        end
-
         def connecting?
           return super unless @options.proxy
 
@@ -250,13 +246,19 @@ module HTTPX
           return super unless @options.proxy
 
           @state = :open
-          transition(:closing)
-          transition(:closed)
+
+          super
           emit(:close)
         end
 
         private
 
+        def initialize_type(uri, options)
+          return super unless options.proxy
+
+          "tcp"
+        end
+
         def connect
           return super unless @options.proxy
 
@@ -284,7 +286,7 @@ module HTTPX
     register_plugin :proxy, Proxy
   end
 
-  class ProxySSL < IO.registry["ssl"]
+  class ProxySSL < SSL
     def initialize(tcp, request_uri, options)
       @io = tcp.to_io
       super(request_uri, tcp.addresses, options)

data/lib/httpx/plugins/push_promise.rb

@@ -8,7 +8,7 @@ module HTTPX
     # In order to benefit from this, requests are sent one at a time, so that
     # no push responses are received after corresponding request has been sent.
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Server-Push
+    # https://gitlab.com/os85/httpx/wikis/Server-Push
     #
     module PushPromise
       def self.extra_options(options)

data/lib/httpx/plugins/rate_limiter.rb

@@ -9,7 +9,7 @@ module HTTPX
     # * when the server is unavailable (503);
     # * when a 3xx request comes with a "retry-after" value
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/RateLimiter
+    # https://gitlab.com/os85/httpx/wikis/Rate-Limiter
     #
     module RateLimiter
       class << self
@@ -23,6 +23,8 @@ module HTTPX
         end
 
         def retry_on_rate_limited_response(response)
+          return false unless response.is_a?(Response)
+
           status = response.status
 
           RATE_LIMIT_CODES.include?(status)