httpx-patched 1.6.2.1

data/lib/httpx/plugins/oauth.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # https://gitlab.com/os85/httpx/wikis/OAuth
+    #
+    module OAuth
+      class << self
+        def load_dependencies(_klass)
+          require_relative "auth/basic"
+        end
+      end
+
+      SUPPORTED_GRANT_TYPES = %w[client_credentials refresh_token].freeze
+      SUPPORTED_AUTH_METHODS = %w[client_secret_basic client_secret_post].freeze
+
+      class OAuthSession
+        attr_reader :grant_type, :client_id, :client_secret, :access_token, :refresh_token, :scope, :audience
+
+        def initialize(
+          issuer:,
+          client_id:,
+          client_secret:,
+          access_token: nil,
+          refresh_token: nil,
+          scope: nil,
+          audience: nil,
+          token_endpoint: nil,
+          response_type: nil,
+          grant_type: nil,
+          token_endpoint_auth_method: nil
+        )
+          @issuer = URI(issuer)
+          @client_id = client_id
+          @client_secret = client_secret
+          @token_endpoint = URI(token_endpoint) if token_endpoint
+          @response_type = response_type
+          @scope = case scope
+                   when String
+                     scope.split
+                   when Array
+                     scope
+          end
+          @audience = audience
+          @access_token = access_token
+          @refresh_token = refresh_token
+          @token_endpoint_auth_method = String(token_endpoint_auth_method) if token_endpoint_auth_method
+          @grant_type = grant_type || (@refresh_token ? "refresh_token" : "client_credentials")
+
+          unless @token_endpoint_auth_method.nil? || SUPPORTED_AUTH_METHODS.include?(@token_endpoint_auth_method)
+            raise Error, "#{@token_endpoint_auth_method} is not a supported auth method"
+          end
+
+          return if SUPPORTED_GRANT_TYPES.include?(@grant_type)
+
+          raise Error, "#{@grant_type} is not a supported grant type"
+        end
+
+        def token_endpoint
+          @token_endpoint || "#{@issuer}/token"
+        end
+
+        def token_endpoint_auth_method
+          @token_endpoint_auth_method || "client_secret_basic"
+        end
+
+        def load(http)
+          return if @grant_type && @scope
+
+          metadata = http.get("#{@issuer}/.well-known/oauth-authorization-server").raise_for_status.json
+
+          @token_endpoint = metadata["token_endpoint"]
+          @scope = metadata["scopes_supported"]
+          @grant_type = Array(metadata["grant_types_supported"]).find { |gr| SUPPORTED_GRANT_TYPES.include?(gr) }
+          @token_endpoint_auth_method = Array(metadata["token_endpoint_auth_methods_supported"]).find do |am|
+            SUPPORTED_AUTH_METHODS.include?(am)
+          end
+          nil
+        end
+
+        def merge(other)
+          obj = dup
+
+          case other
+          when OAuthSession
+            other.instance_variables.each do |ivar|
+              val = other.instance_variable_get(ivar)
+              next unless val
+
+              obj.instance_variable_set(ivar, val)
+            end
+          when Hash
+            other.each do |k, v|
+              obj.instance_variable_set(:"@#{k}", v) if obj.instance_variable_defined?(:"@#{k}")
+            end
+          end
+          obj
+        end
+      end
+
+      module OptionsMethods
+        private
+
+        def option_oauth_session(value)
+          case value
+          when Hash
+            OAuthSession.new(**value)
+          when OAuthSession
+            value
+          else
+            raise TypeError, ":oauth_session must be a #{OAuthSession}"
+          end
+        end
+      end
+
+      module InstanceMethods
+        def oauth_auth(**args)
+          with(oauth_session: OAuthSession.new(**args))
+        end
+
+        def with_access_token
+          oauth_session = @options.oauth_session
+
+          oauth_session.load(self)
+
+          grant_type = oauth_session.grant_type
+
+          headers = {}
+          form_post = {
+            "grant_type" => grant_type,
+            "scope" => Array(oauth_session.scope).join(" "),
+            "audience" => oauth_session.audience,
+          }.compact
+
+          # auth
+          case oauth_session.token_endpoint_auth_method
+          when "client_secret_post"
+            form_post["client_id"] = oauth_session.client_id
+            form_post["client_secret"] = oauth_session.client_secret
+          when "client_secret_basic"
+            headers["authorization"] = Authentication::Basic.new(oauth_session.client_id, oauth_session.client_secret).authenticate
+          end
+
+          case grant_type
+          when "client_credentials"
+            # do nothing
+          when "refresh_token"
+            form_post["refresh_token"] = oauth_session.refresh_token
+          end
+
+          token_request = build_request("POST", oauth_session.token_endpoint, headers: headers, form: form_post)
+          token_request.headers.delete("authorization") unless oauth_session.token_endpoint_auth_method == "client_secret_basic"
+
+          token_response = request(token_request)
+          token_response.raise_for_status
+
+          payload = token_response.json
+
+          access_token = payload["access_token"]
+          refresh_token = payload["refresh_token"]
+
+          with(oauth_session: oauth_session.merge(access_token: access_token, refresh_token: refresh_token))
+        end
+
+        def build_request(*)
+          request = super
+
+          return request if request.headers.key?("authorization")
+
+          oauth_session = @options.oauth_session
+
+          return request unless oauth_session && oauth_session.access_token
+
+          request.headers["authorization"] = "Bearer #{oauth_session.access_token}"
+
+          request
+        end
+      end
+    end
+    register_plugin :oauth, OAuth
+  end
+end

data/lib/httpx/plugins/persistent.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    # This plugin implements a session that persists connections over the duration of the process.
+    #
+    # This will improve connection reuse in a long-running process.
+    #
+    # One important caveat to note is, although this session might not close connections,
+    # other sessions from the same process that don't have this plugin turned on might.
+    #
+    # This session will still be able to work with it, as if, when expecting a connection
+    # terminated by a different session, it will just retry on a new one and keep it open.
+    #
+    # This plugin is also not recommendable when connecting to >9000 (like, a lot) different origins.
+    # So when you use this, make sure that you don't fall into this trap.
+    #
+    # https://gitlab.com/os85/httpx/wikis/Persistent
+    #
+    module Persistent
+      class << self
+        def load_dependencies(klass)
+          klass.plugin(:fiber_concurrency)
+
+          max_retries = if klass.default_options.respond_to?(:max_retries)
+            [klass.default_options.max_retries, 1].max
+          else
+            1
+          end
+          klass.plugin(:retries, max_retries: max_retries)
+        end
+      end
+
+      def self.extra_options(options)
+        options.merge(persistent: true)
+      end
+
+      module InstanceMethods
+        def close(*)
+          super
+
+          # traverse other threads and unlink respective selector
+          # WARNING: this is not thread safe, make sure that the session isn't being
+          # used anymore, or all non-main threads are stopped.
+          Thread.list.each do |th|
+            store = thread_selector_store(th)
+
+            next unless store && store.key?(self)
+
+            selector = store.delete(self)
+
+            selector_close(selector)
+          end
+        end
+
+        private
+
+        def repeatable_request?(request, _)
+          super || begin
+            response = request.response
+
+            return false unless response && response.is_a?(ErrorResponse)
+
+            error = response.error
+
+            Retries::RECONNECTABLE_ERRORS.any? { |klass| error.is_a?(klass) }
+          end
+        end
+
+        def retryable_error?(ex)
+          super &&
+            # under the persistent plugin rules, requests are only retried for connection related errors,
+            # which do not include request timeout related errors. This only gets overriden if the end user
+            # manually changed +:max_retries+ to something else, which means it is aware of the
+            # consequences.
+            (!ex.is_a?(RequestTimeoutError) || @options.max_retries != 1)
+        end
+      end
+    end
+    register_plugin :persistent, Persistent
+  end
+end

data/lib/httpx/plugins/proxy/http.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    module Proxy
+      module HTTP
+        class << self
+          def extra_options(options)
+            options.merge(supported_proxy_protocols: options.supported_proxy_protocols + %w[http])
+          end
+        end
+
+        module InstanceMethods
+          def with_proxy_basic_auth(opts)
+            with(proxy: opts.merge(scheme: "basic"))
+          end
+
+          def with_proxy_digest_auth(opts)
+            with(proxy: opts.merge(scheme: "digest"))
+          end
+
+          def with_proxy_ntlm_auth(opts)
+            with(proxy: opts.merge(scheme: "ntlm"))
+          end
+
+          def fetch_response(request, selector, options)
+            response = super
+
+            if response &&
+               response.is_a?(Response) &&
+               response.status == 407 &&
+               !request.headers.key?("proxy-authorization") &&
+               response.headers.key?("proxy-authenticate") && options.proxy.can_authenticate?(response.headers["proxy-authenticate"])
+              request.transition(:idle)
+              request.headers["proxy-authorization"] =
+                options.proxy.authenticate(request, response.headers["proxy-authenticate"])
+              send_request(request, selector, options)
+              return
+            end
+
+            response
+          end
+        end
+
+        module ConnectionMethods
+          def connecting?
+            super || @state == :connecting || @state == :connected
+          end
+
+          private
+
+          def handle_transition(nextstate)
+            return super unless @options.proxy && @options.proxy.uri.scheme == "http"
+
+            case nextstate
+            when :connecting
+              return unless @state == :idle
+
+              @io.connect
+              return unless @io.connected?
+
+              @parser || begin
+                @parser = parser_type(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
+                parser = @parser
+                parser.extend(ProxyParser)
+                parser.on(:response, &method(:__http_on_connect))
+                parser.on(:close) do |force|
+                  next unless @parser
+
+                  if force
+                    reset
+                    emit(:terminate)
+                  end
+                end
+                parser.on(:reset) do
+                  if parser.empty?
+                    reset
+                  else
+                    transition(:closing)
+                    transition(:closed)
+
+                    parser.reset if @parser
+                    transition(:idle)
+                    transition(:connecting)
+                  end
+                end
+                __http_proxy_connect(parser)
+              end
+              return if @state == :connected
+            when :connected
+              return unless @state == :idle || @state == :connecting
+
+              case @state
+              when :connecting
+                parser = @parser
+                @parser = nil
+                parser.close
+              when :idle
+                @parser.callbacks.clear
+                set_parser_callbacks(@parser)
+              end
+            end
+            super
+          end
+
+          def __http_proxy_connect(parser)
+            req = @pending.first
+            if req && req.uri.scheme == "https"
+              # if the first request after CONNECT is to an https address, it is assumed that
+              # all requests in the queue are not only ALL HTTPS, but they also share the certificate,
+              # and therefore, will share the connection.
+              #
+              connect_request = ConnectRequest.new(req.uri, @options)
+              @inflight += 1
+              parser.send(connect_request)
+            else
+              handle_transition(:connected)
+            end
+          end
+
+          def __http_on_connect(request, response)
+            @inflight -= 1
+            if response.is_a?(Response) && response.status == 200
+              req = @pending.first
+              request_uri = req.uri
+              @io = ProxySSL.new(@io, request_uri, @options)
+              transition(:connected)
+              throw(:called)
+            elsif response.is_a?(Response) &&
+                  response.status == 407 &&
+                  !request.headers.key?("proxy-authorization") &&
+                  @options.proxy.can_authenticate?(response.headers["proxy-authenticate"])
+
+              request.transition(:idle)
+              request.headers["proxy-authorization"] = @options.proxy.authenticate(request, response.headers["proxy-authenticate"])
+              @parser.send(request)
+              @inflight += 1
+            else
+              pending = @pending + @parser.pending
+              while (req = pending.shift)
+                response.finish!
+                req.response = response
+                req.emit(:response, response)
+              end
+              reset
+            end
+          end
+        end
+
+        module ProxyParser
+          def join_headline(request)
+            return super if request.verb == "CONNECT"
+
+            "#{request.verb} #{request.uri} HTTP/#{@version.join(".")}"
+          end
+
+          def set_protocol_headers(request)
+            extra_headers = super
+
+            proxy_params = @options.proxy
+            if proxy_params.scheme == "basic"
+              # opt for basic auth
+              extra_headers["proxy-authorization"] = proxy_params.authenticate(extra_headers)
+            end
+            extra_headers["proxy-connection"] = extra_headers.delete("connection") if extra_headers.key?("connection")
+            extra_headers
+          end
+        end
+
+        class ConnectRequest < Request
+          def initialize(uri, options)
+            super("CONNECT", uri, options)
+            @headers.delete("accept")
+          end
+
+          def path
+            "#{@uri.hostname}:#{@uri.port}"
+          end
+        end
+      end
+    end
+    register_plugin :"proxy/http", Proxy::HTTP
+  end
+end

data/lib/httpx/plugins/proxy/socks4.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require "resolv"
+require "ipaddr"
+
+module HTTPX
+  class Socks4Error < ProxyError; end
+
+  module Plugins
+    module Proxy
+      module Socks4
+        VERSION = 4
+        CONNECT = 1
+        GRANTED = 0x5A
+        PROTOCOLS = %w[socks4 socks4a].freeze
+
+        Error = Socks4Error
+
+        class << self
+          def extra_options(options)
+            options.merge(supported_proxy_protocols: options.supported_proxy_protocols + PROTOCOLS)
+          end
+        end
+
+        module ConnectionMethods
+          def interests
+            if @state == :connecting
+              return @write_buffer.empty? ? :r : :w
+            end
+
+            super
+          end
+
+          private
+
+          def handle_transition(nextstate)
+            return super unless @options.proxy && PROTOCOLS.include?(@options.proxy.uri.scheme)
+
+            case nextstate
+            when :connecting
+              return unless @state == :idle
+
+              @io.connect
+              return unless @io.connected?
+
+              req = @pending.first
+              return unless req
+
+              request_uri = req.uri
+              @write_buffer << Packet.connect(@options.proxy, request_uri)
+              __socks4_proxy_connect
+            when :connected
+              return unless @state == :connecting
+
+              @parser = nil
+            end
+            log(level: 1) { "SOCKS4: #{nextstate}: #{@write_buffer.to_s.inspect}" } unless nextstate == :open
+            super
+          end
+
+          def __socks4_proxy_connect
+            @parser = SocksParser.new(@write_buffer, @options)
+            @parser.once(:packet, &method(:__socks4_on_packet))
+          end
+
+          def __socks4_on_packet(packet)
+            _version, status, _port, _ip = packet.unpack("CCnN")
+            if status == GRANTED
+              req = @pending.first
+              request_uri = req.uri
+              @io = ProxySSL.new(@io, request_uri, @options) if request_uri.scheme == "https"
+              transition(:connected)
+              throw(:called)
+            else
+              on_socks4_error("socks error: #{status}")
+            end
+          end
+
+          def on_socks4_error(message)
+            ex = Error.new(message)
+            ex.set_backtrace(caller)
+            on_error(ex)
+            throw(:called)
+          end
+        end
+
+        class SocksParser
+          include HTTPX::Callbacks
+
+          def initialize(buffer, options)
+            @buffer = buffer
+            @options = options
+          end
+
+          def close; end
+
+          def consume(*); end
+
+          def empty?
+            true
+          end
+
+          def <<(packet)
+            emit(:packet, packet)
+          end
+        end
+
+        module Packet
+          module_function
+
+          def connect(parameters, uri)
+            packet = [VERSION, CONNECT, uri.port].pack("CCn")
+
+            case parameters.uri.scheme
+            when "socks4"
+              socks_host = uri.host
+              begin
+                ip = IPAddr.new(socks_host)
+                packet << ip.hton
+              rescue IPAddr::InvalidAddressError
+                socks_host = Resolv.getaddress(socks_host)
+                retry
+              end
+              packet << [parameters.username].pack("Z*")
+            when "socks4a"
+              packet << "\x0\x0\x0\x1" << [parameters.username].pack("Z*") << uri.host << "\x0"
+            end
+            packet
+          end
+        end
+      end
+    end
+    register_plugin :"proxy/socks4", Proxy::Socks4
+  end
+end