httpx-patched 1.6.2.1

data/lib/httpx/plugins/cookies/cookie.rb

@@ -0,0 +1,174 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins::Cookies
+    # The HTTP Cookie.
+    #
+    # Contains the single cookie info: name, value and attributes.
+    class Cookie
+      include Comparable
+      # Maximum number of bytes per cookie (RFC 6265 6.1 requires 4096 at
+      # least)
+      MAX_LENGTH = 4096
+
+      attr_reader :domain, :path, :name, :value, :created_at
+
+      def path=(path)
+        path = String(path)
+        @path = path.start_with?("/") ? path : "/"
+      end
+
+      # See #domain.
+      def domain=(domain)
+        domain = String(domain)
+
+        if domain.start_with?(".")
+          @for_domain = true
+          domain = domain[1..-1]
+        end
+
+        return if domain.empty?
+
+        @domain_name = DomainName.new(domain)
+        # RFC 6265 5.3 5.
+        @for_domain = false if @domain_name.domain.nil? # a public suffix or IP address
+
+        @domain = @domain_name.hostname
+      end
+
+      # Compares the cookie with another.  When there are many cookies with
+      # the same name for a URL, the value of the smallest must be used.
+      def <=>(other)
+        # RFC 6265 5.4
+        # Precedence: 1. longer path  2. older creation
+        (@name <=> other.name).nonzero? ||
+          (other.path.length <=> @path.length).nonzero? ||
+          (@created_at <=> other.created_at).nonzero? || 0
+      end
+
+      class << self
+        def new(cookie, *args)
+          return cookie if cookie.is_a?(self)
+
+          super
+        end
+
+        # Tests if +target_path+ is under +base_path+ as described in RFC
+        # 6265 5.1.4.  +base_path+ must be an absolute path.
+        # +target_path+ may be empty, in which case it is treated as the
+        # root path.
+        #
+        # e.g.
+        #
+        #         path_match?('/admin/', '/admin/index') == true
+        #         path_match?('/admin/', '/Admin/index') == false
+        #         path_match?('/admin/', '/admin/') == true
+        #         path_match?('/admin/', '/admin') == false
+        #
+        #         path_match?('/admin', '/admin') == true
+        #         path_match?('/admin', '/Admin') == false
+        #         path_match?('/admin', '/admins') == false
+        #         path_match?('/admin', '/admin/') == true
+        #         path_match?('/admin', '/admin/index') == true
+        def path_match?(base_path, target_path)
+          base_path.start_with?("/") || (return false)
+          # RFC 6265 5.1.4
+          bsize = base_path.size
+          tsize = target_path.size
+          return bsize == 1 if tsize.zero? # treat empty target_path as "/"
+          return false unless target_path.start_with?(base_path)
+          return true if bsize == tsize || base_path.end_with?("/")
+
+          target_path[bsize] == "/"
+        end
+      end
+
+      def initialize(arg, *attrs)
+        @created_at = Time.now
+
+        if attrs.empty?
+          attr_hash = Hash.try_convert(arg)
+        else
+          @name = arg
+          @value, attr_hash = attrs
+          attr_hash = Hash.try_convert(attr_hash)
+        end
+
+        attr_hash.each do |key, val|
+          key = key.downcase.tr("-", "_").to_sym unless key.is_a?(Symbol)
+
+          case key
+          when :domain, :path
+            __send__(:"#{key}=", val)
+          else
+            instance_variable_set(:"@#{key}", val)
+          end
+        end if attr_hash
+
+        @path ||= "/"
+        raise ArgumentError, "name must be specified" if @name.nil?
+
+        @name = @name.to_s
+      end
+
+      def expires
+        @expires || (@created_at && @max_age ? @created_at + @max_age : nil)
+      end
+
+      def expired?(time = Time.now)
+        return false unless expires
+
+        expires <= time
+      end
+
+      # Returns a string for use in the Cookie header, i.e. `name=value`
+      # or `name="value"`.
+      def cookie_value
+        "#{@name}=#{Scanner.quote(@value.to_s)}"
+      end
+      alias_method :to_s, :cookie_value
+
+      # Tests if it is OK to send this cookie to a given `uri`.  A
+      # RuntimeError is raised if the cookie's domain is unknown.
+      def valid_for_uri?(uri)
+        uri = URI(uri)
+        # RFC 6265 5.4
+
+        return false if @secure && uri.scheme != "https"
+
+        acceptable_from_uri?(uri) && Cookie.path_match?(@path, uri.path)
+      end
+
+      private
+
+      # Tests if it is OK to accept this cookie if it is sent from a given
+      # URI/URL, `uri`.
+      def acceptable_from_uri?(uri)
+        uri = URI(uri)
+
+        host = DomainName.new(uri.host)
+
+        # RFC 6265 5.3
+        if host.hostname == @domain
+          true
+        elsif @for_domain # !host-only-flag
+          host.cookie_domain?(@domain_name)
+        else
+          @domain.nil?
+        end
+      end
+
+      module Scanner
+        RE_BAD_CHAR = /([\x00-\x20\x7F",;\\])/.freeze
+
+        module_function
+
+        def quote(s)
+          return s unless s.match(RE_BAD_CHAR)
+
+          "\"#{s.gsub(/([\\"])/, "\\\\\\1")}\""
+        end
+      end
+    end
+  end
+end

data/lib/httpx/plugins/cookies/jar.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins::Cookies
+    # The Cookie Jar
+    #
+    # It holds a bunch of cookies.
+    class Jar
+      using URIExtensions
+
+      include Enumerable
+
+      def initialize_dup(orig)
+        super
+        @cookies = orig.instance_variable_get(:@cookies).dup
+      end
+
+      def initialize(cookies = nil)
+        @cookies = []
+
+        cookies.each do |elem|
+          cookie = case elem
+                   when Cookie
+                     elem
+                   when Array
+                     Cookie.new(*elem)
+                   else
+                     Cookie.new(elem)
+          end
+
+          @cookies << cookie
+        end if cookies
+      end
+
+      def parse(set_cookie)
+        SetCookieParser.call(set_cookie) do |name, value, attrs|
+          add(Cookie.new(name, value, attrs))
+        end
+      end
+
+      def add(cookie, path = nil)
+        c = cookie.dup
+
+        c.path = path if path && c.path == "/"
+
+        # If the user agent receives a new cookie with the same cookie-name, domain-value, and path-value
+        # as a cookie that it has already stored, the existing cookie is evicted and replaced with the new cookie.
+        @cookies.delete_if { |ck| ck.name == c.name && ck.domain == c.domain && ck.path == c.path }
+
+        @cookies << c
+      end
+
+      def [](uri)
+        each(uri).sort
+      end
+
+      def each(uri = nil, &blk)
+        return enum_for(__method__, uri) unless blk
+
+        return @cookies.each(&blk) unless uri
+
+        now = Time.now
+        tpath = uri.path
+
+        @cookies.delete_if do |cookie|
+          if cookie.expired?(now)
+            true
+          else
+            yield cookie if cookie.valid_for_uri?(uri) && Cookie.path_match?(cookie.path, tpath)
+            false
+          end
+        end
+      end
+
+      def merge(other)
+        cookies_dup = dup
+
+        other.each do |elem|
+          cookie = case elem
+                   when Cookie
+                     elem
+                   when Array
+                     Cookie.new(*elem)
+                   else
+                     Cookie.new(elem)
+          end
+
+          cookies_dup.add(cookie)
+        end
+
+        cookies_dup
+      end
+    end
+  end
+end

data/lib/httpx/plugins/cookies/set_cookie_parser.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+require "strscan"
+require "time"
+
+module HTTPX
+  module Plugins::Cookies
+    module SetCookieParser
+      # Whitespace.
+      RE_WSP = /[ \t]+/.freeze
+
+      # A pattern that matches a cookie name or attribute name which may
+      # be empty, capturing trailing whitespace.
+      RE_NAME = /(?!#{RE_WSP})[^,;\\"=]*/.freeze
+
+      RE_BAD_CHAR = /([\x00-\x20\x7F",;\\])/.freeze
+
+      # A pattern that matches the comma in a (typically date) value.
+      RE_COOKIE_COMMA = /,(?=#{RE_WSP}?#{RE_NAME}=)/.freeze
+
+      module_function
+
+      def scan_dquoted(scanner)
+        s = +""
+
+        until scanner.eos?
+          break if scanner.skip(/"/)
+
+          if scanner.skip(/\\/)
+            s << scanner.getch
+          elsif scanner.scan(/[^"\\]+/)
+            s << scanner.matched
+          end
+        end
+
+        s
+      end
+
+      def scan_value(scanner, comma_as_separator = false)
+        value = +""
+
+        until scanner.eos?
+          if scanner.scan(/[^,;"]+/)
+            value << scanner.matched
+          elsif scanner.skip(/"/)
+            # RFC 6265 2.2
+            # A cookie-value may be DQUOTE'd.
+            value << scan_dquoted(scanner)
+          elsif scanner.check(/;/)
+            break
+          elsif comma_as_separator && scanner.check(RE_COOKIE_COMMA)
+            break
+          else
+            value << scanner.getch
+          end
+        end
+
+        value.rstrip!
+        value
+      end
+
+      def scan_name_value(scanner, comma_as_separator = false)
+        name = scanner.scan(RE_NAME)
+        name.rstrip! if name
+
+        if scanner.skip(/=/)
+          value = scan_value(scanner, comma_as_separator)
+        else
+          scan_value(scanner, comma_as_separator)
+          value = nil
+        end
+        [name, value]
+      end
+
+      def call(set_cookie)
+        scanner = StringScanner.new(set_cookie)
+
+        # RFC 6265 4.1.1 & 5.2
+        until scanner.eos?
+          start = scanner.pos
+          len = nil
+
+          scanner.skip(RE_WSP)
+
+          name, value = scan_name_value(scanner, true)
+          value = nil if name && name.empty?
+
+          attrs = {}
+
+          until scanner.eos?
+            if scanner.skip(/,/)
+              # The comma is used as separator for concatenating multiple
+              # values of a header.
+              len = (scanner.pos - 1) - start
+              break
+            elsif scanner.skip(/;/)
+              scanner.skip(RE_WSP)
+
+              aname, avalue = scan_name_value(scanner, true)
+
+              next if (aname.nil? || aname.empty?) || value.nil?
+
+              aname.downcase!
+
+              case aname
+              when "expires"
+                next unless avalue
+
+                # RFC 6265 5.2.1
+                (avalue = Time.parse(avalue)) || next
+              when "max-age"
+                next unless avalue
+                # RFC 6265 5.2.2
+                next unless /\A-?\d+\z/.match?(avalue)
+
+                avalue = Integer(avalue)
+              when "domain"
+                # RFC 6265 5.2.3
+                # An empty value SHOULD be ignored.
+                next if avalue.nil? || avalue.empty?
+              when "path"
+                # RFC 6265 5.2.4
+                # A relative path must be ignored rather than normalizing it
+                # to "/".
+                next unless avalue && avalue.start_with?("/")
+              when "secure", "httponly"
+                # RFC 6265 5.2.5, 5.2.6
+                avalue = true
+              end
+              attrs[aname] = avalue
+            end
+          end
+
+          len ||= scanner.pos - start
+
+          next if len > Cookie::MAX_LENGTH
+
+          yield(name, value, attrs) if name && !name.empty? && value
+        end
+      end
+    end
+  end
+end

data/lib/httpx/plugins/cookies.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require "forwardable"
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin implements a persistent cookie jar for the duration of a session.
+    #
+    # It also adds a *#cookies* helper, so that you can pre-fill the cookies of a session.
+    #
+    # https://gitlab.com/os85/httpx/wikis/Cookies
+    #
+    module Cookies
+      def self.load_dependencies(*)
+        require "httpx/plugins/cookies/jar"
+        require "httpx/plugins/cookies/cookie"
+        require "httpx/plugins/cookies/set_cookie_parser"
+      end
+
+      module InstanceMethods
+        extend Forwardable
+
+        def_delegator :@options, :cookies
+
+        def initialize(options = {}, &blk)
+          super({ cookies: Jar.new }.merge(options), &blk)
+        end
+
+        def wrap
+          return super unless block_given?
+
+          super do |session|
+            old_cookies_jar = @options.cookies.dup
+            begin
+              yield session
+            ensure
+              @options = @options.merge(cookies: old_cookies_jar)
+            end
+          end
+        end
+
+        def build_request(*)
+          request = super
+          request.headers.set_cookie(request.options.cookies[request.uri])
+          request
+        end
+
+        private
+
+        def set_request_callbacks(request)
+          super
+          request.on(:response) do |response|
+            next unless response && response.respond_to?(:headers) && (set_cookie = response.headers["set-cookie"])
+
+            log { "cookies: set-cookie is over #{Cookie::MAX_LENGTH}" } if set_cookie.bytesize > Cookie::MAX_LENGTH
+
+            @options.cookies.parse(set_cookie)
+          end
+        end
+      end
+
+      module HeadersMethods
+        def set_cookie(cookies)
+          return if cookies.empty?
+
+          header_value = cookies.sort.join("; ")
+
+          add("cookie", header_value)
+        end
+      end
+
+      # adds support for the following options:
+      #
+      # :cookies :: cookie jar for the session (can be a Hash, an Array, an instance of HTTPX::Plugins::Cookies::CookieJar)
+      module OptionsMethods
+        private
+
+        def option_headers(*)
+          value = super
+
+          merge_cookie_in_jar(value.delete("cookie"), @cookies) if defined?(@cookies) && value.key?("cookie")
+
+          value
+        end
+
+        def option_cookies(value)
+          jar = value.is_a?(Jar) ? value : Jar.new(value)
+
+          merge_cookie_in_jar(@headers.delete("cookie"), jar) if defined?(@headers) && @headers.key?("cookie")
+
+          jar
+        end
+
+        def merge_cookie_in_jar(cookies, jar)
+          cookies.each do |ck|
+            ck.split(/ *; */).each do |cookie|
+              name, value = cookie.split("=", 2)
+              jar.add(Cookie.new(name, value))
+            end
+          end
+        end
+      end
+    end
+    register_plugin :cookies, Cookies
+  end
+end

data/lib/httpx/plugins/digest_auth.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds helper methods to implement HTTP Digest Auth (https://datatracker.ietf.org/doc/html/rfc7616)
+    #
+    # https://gitlab.com/os85/httpx/wikis/Auth#digest-auth
+    #
+    module DigestAuth
+      DigestError = Class.new(Error)
+
+      class << self
+        def extra_options(options)
+          options.merge(max_concurrent_requests: 1)
+        end
+
+        def load_dependencies(*)
+          require_relative "auth/digest"
+        end
+      end
+
+      # adds support for the following options:
+      #
+      # :digest :: instance of HTTPX::Plugins::Authentication::Digest, used to authenticate requests in the session.
+      module OptionsMethods
+        private
+
+        def option_digest(value)
+          raise TypeError, ":digest must be a #{Authentication::Digest}" unless value.is_a?(Authentication::Digest)
+
+          value
+        end
+      end
+
+      module InstanceMethods
+        def digest_auth(user, password, hashed: false)
+          with(digest: Authentication::Digest.new(user, password, hashed: hashed))
+        end
+
+        private
+
+        def send_requests(*requests)
+          requests.flat_map do |request|
+            digest = request.options.digest
+
+            next super(request) unless digest
+
+            probe_response = wrap { super(request).first }
+
+            return probe_response unless probe_response.is_a?(Response)
+
+            if probe_response.status == 401 && digest.can_authenticate?(probe_response.headers["www-authenticate"])
+              request.transition(:idle)
+              request.headers["authorization"] = digest.authenticate(request, probe_response.headers["www-authenticate"])
+              super(request)
+            else
+              probe_response
+            end
+          end
+        end
+      end
+    end
+
+    register_plugin :digest_auth, DigestAuth
+  end
+end

data/lib/httpx/plugins/expect.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin makes all HTTP/1.1 requests with a body send the "Expect: 100-continue".
+    #
+    # https://gitlab.com/os85/httpx/wikis/Expect#expect
+    #
+    module Expect
+      EXPECT_TIMEOUT = 2
+
+      class << self
+        def no_expect_store
+          @no_expect_store ||= []
+        end
+
+        def extra_options(options)
+          options.merge(expect_timeout: EXPECT_TIMEOUT)
+        end
+      end
+
+      # adds support for the following options:
+      #
+      # :expect_timeout :: time (in seconds) to wait for a 100-expect response,
+      #                    before retrying without the Expect header (defaults to <tt>2</tt>).
+      # :expect_threshold_size :: min threshold (in bytes) of the request payload to enable the 100-continue negotiation on.
+      module OptionsMethods
+        private
+
+        def option_expect_timeout(value)
+          seconds = Float(value)
+          raise TypeError, ":expect_timeout must be positive" unless seconds.positive?
+
+          seconds
+        end
+
+        def option_expect_threshold_size(value)
+          bytes = Integer(value)
+          raise TypeError, ":expect_threshold_size must be positive" unless bytes.positive?
+
+          bytes
+        end
+      end
+
+      module RequestMethods
+        def initialize(*)
+          super
+          return if @body.empty?
+
+          threshold = @options.expect_threshold_size
+          return if threshold && !@body.unbounded_body? && @body.bytesize < threshold
+
+          return if Expect.no_expect_store.include?(origin)
+
+          @headers["expect"] = "100-continue"
+        end
+
+        def response=(response)
+          if response.is_a?(Response) &&
+             response.status == 100 &&
+             !@headers.key?("expect") &&
+             (@state == :body || @state == :done)
+
+            # if we're past this point, this means that we just received a 100-Continue response,
+            # but the request doesn't have the expect flag, and is already flushing (or flushed) the body.
+            #
+            # this means that expect was deactivated for this request too soon, i.e. response took longer.
+            #
+            # so we have to reactivate it again.
+            @headers["expect"] = "100-continue"
+            @informational_status = 100
+            Expect.no_expect_store.delete(origin)
+          end
+          super
+        end
+      end
+
+      module ConnectionMethods
+        def send_request_to_parser(request)
+          super
+
+          return unless request.headers["expect"] == "100-continue"
+
+          expect_timeout = request.options.expect_timeout
+
+          return if expect_timeout.nil? || expect_timeout.infinite?
+
+          set_request_timeout(:expect_timeout, request, expect_timeout, :expect, %i[body response]) do
+            # expect timeout expired
+            if request.state == :expect && !request.expects?
+              Expect.no_expect_store << request.origin
+              request.headers.delete("expect")
+              consume
+            end
+          end
+        end
+      end
+
+      module InstanceMethods
+        def fetch_response(request, selector, options)
+          response = super
+
+          return unless response
+
+          if response.is_a?(Response) && response.status == 417 && request.headers.key?("expect")
+            response.close
+            request.headers.delete("expect")
+            request.transition(:idle)
+            send_request(request, selector, options)
+            return
+          end
+
+          response
+        end
+      end
+    end
+    register_plugin :expect, Expect
+  end
+end