httpx 1.7.2 → 1.7.6

data/lib/httpx/parser/http1.rb

@@ -14,6 +14,8 @@ module HTTPX
         @state = :idle
         @buffer = "".b
         @headers = {}
+        @content_length = nil
+        @_has_trailers = @upgrade = false
       end
 
       def <<(chunk)
@@ -25,7 +27,8 @@ module HTTPX
         @state = :idle
         @headers = {}
         @content_length = nil
-        @_has_trailers = nil
+        @_has_trailers = @upgrade = false
+        @buffer = @buffer.to_s
         @buffer.clear
       end
 
@@ -34,7 +37,7 @@ module HTTPX
       end
 
       def upgrade_data
-        @buffer
+        @buffer.to_s
       end
 
       private
@@ -55,6 +58,7 @@ module HTTPX
       end
 
       def parse_headline
+        #: @type ivar @buffer: String
         idx = @buffer.index("\n")
         return unless idx
 
@@ -75,6 +79,8 @@ module HTTPX
         headers = @headers
         buffer = @buffer
 
+        #: @type var buffer: String
+
         while (idx = buffer.index("\n"))
           # @type var line: String
           line = buffer.byteslice(0..idx)
@@ -118,17 +124,20 @@ module HTTPX
 
       def parse_data
         if @buffer.respond_to?(:each)
+          # @type ivar @buffer: Transcoder::Chunker::Decoder
           @buffer.each do |chunk|
             @observer.on_data(chunk)
           end
         elsif @content_length
-          # @type var data: String
+          # @type ivar @buffer: String
           data = @buffer.byteslice(0, @content_length)
+          # @type var data: String
           @buffer = @buffer.byteslice(@content_length..-1) || "".b
           @content_length -= data.bytesize
           @observer.on_data(data)
           data.clear
         else
+          # @type ivar @buffer: String
           @observer.on_data(@buffer)
           @buffer.clear
         end
@@ -152,7 +161,7 @@ module HTTPX
             tr_encoding.split(/ *, */).each do |encoding|
               case encoding
               when "chunked"
-                @buffer = Transcoder::Chunker::Decoder.new(@buffer, @_has_trailers)
+                @buffer = Transcoder::Chunker::Decoder.new(@buffer.to_s, @_has_trailers)
               end
             end
           end
@@ -165,6 +174,7 @@ module HTTPX
         if @content_length
           @content_length <= 0
         elsif @buffer.respond_to?(:finished?)
+          # @type ivar @buffer: Transcoder::Chunker::Decoder
           @buffer.finished?
         else
           false

data/lib/httpx/plugins/auth/digest.rb

@@ -8,7 +8,8 @@ module HTTPX
   module Plugins
     module Authentication
       class Digest
-        Error = Class.new(Error)
+        class Error < Error
+        end
 
         def initialize(user, password, hashed: false, **)
           @user = user

data/lib/httpx/plugins/auth.rb

@@ -44,6 +44,7 @@ module HTTPX
           super
 
           @auth_header_value = nil
+          @auth_header_value_mtx = Thread::Mutex.new
           @skip_auth_header_value = false
         end
 
@@ -63,7 +64,9 @@ module HTTPX
         end
 
         def reset_auth_header_value!
-          @auth_header_value = nil
+          @auth_header_value_mtx.synchronize do
+            @auth_header_value = nil
+          end
         end
 
         private
@@ -71,9 +74,11 @@ module HTTPX
         def send_request(request, *)
           return super if @skip_auth_header_value || request.authorized?
 
-          @auth_header_value ||= generate_auth_token
+          auth_header_value = @auth_header_value_mtx.synchronize do
+            @auth_header_value ||= generate_auth_token
+          end
 
-          request.authorize(@auth_header_value) if @auth_header_value
+          request.authorize(auth_header_value) if auth_header_value
 
           super
         end
@@ -92,9 +97,11 @@ module HTTPX
       end
 
       module RequestMethods
+        attr_reader :auth_token_value
+
         def initialize(*)
           super
-          @auth_token_value = nil
+          @auth_token_value = @auth_header_value = nil
         end
 
         def authorized?
@@ -102,19 +109,20 @@ module HTTPX
         end
 
         def unauthorize!
-          return unless (auth_value = @auth_token_value)
+          return unless (auth_value = @auth_header_value)
 
           @headers.get("authorization").delete(auth_value)
 
-          @auth_token_value = nil
+          @auth_token_value = @auth_header_value = nil
         end
 
         def authorize(auth_value)
+          @auth_header_value = auth_value
           if (auth_type = @options.auth_header_type)
-            auth_value = "#{auth_type} #{auth_value}"
+            @auth_header_value = "#{auth_type} #{@auth_header_value}"
           end
 
-          @headers.add("authorization", auth_value)
+          @headers.add("authorization", @auth_header_value)
 
           @auth_token_value = auth_value
         end
@@ -138,8 +146,14 @@ module HTTPX
             return unless auth_error?(response, request.options) ||
                           (@options.generate_auth_value_on_retry && @options.generate_auth_value_on_retry.call(response))
 
+            # regenerate token before retry, but only if it's the first request from batch failing.
+            # otherwise, it means that the first request already passed here, so this request should
+            # use whatever was generated for it.
+            @auth_header_value_mtx.synchronize do
+              @auth_header_value = generate_auth_token if request.auth_token_value == @auth_header_value
+            end
+
             request.unauthorize!
-            @auth_header_value = generate_auth_token
           end
 
           def auth_error?(response, options)

data/lib/httpx/plugins/brotli.rb

@@ -3,11 +3,33 @@
 module HTTPX
   module Plugins
     module Brotli
+      class Error < HTTPX::Error; end
+
       class Deflater < Transcoder::Deflater
+        def initialize(body)
+          @compressor = ::Brotli::Compressor.new
+          super
+        end
+
         def deflate(chunk)
-          return unless chunk
+          return @compressor.process(chunk) << @compressor.flush if chunk
+
+          @compressor.finish
+        end
+      end
+
+      class Inflater
+        def initialize(bytesize)
+          @inflater = ::Brotli::Decompressor.new
+          @bytesize = bytesize
+        end
+
+        def call(chunk)
+          buffer = @inflater.process(chunk)
+          @bytesize -= chunk.bytesize
+          raise Error, "Unexpected end of compressed stream" if @bytesize <= 0 && !@inflater.finished?
 
-          ::Brotli.deflate(chunk)
+          buffer
         end
       end
 
@@ -30,19 +52,25 @@ module HTTPX
       module_function
 
       def load_dependencies(*)
+        gem "brotli", ">= 0.8.0"
         require "brotli"
       end
 
       def self.extra_options(options)
-        options.merge(supported_compression_formats: %w[br] + options.supported_compression_formats)
+        supported_compression_formats = (%w[br] + options.supported_compression_formats).freeze
+        options.merge(
+          supported_compression_formats: supported_compression_formats,
+          headers: options.headers_class.new(options.headers.merge("accept-encoding" => supported_compression_formats))
+        )
       end
 
       def encode(body)
         Deflater.new(body)
       end
 
-      def decode(_response, **)
-        ::Brotli.method(:inflate)
+      def decode(response, bytesize: nil)
+        bytesize ||= response.headers.key?("content-length") ? response.headers["content-length"].to_i : Float::INFINITY
+        Inflater.new(bytesize)
       end
     end
     register_plugin :brotli, Brotli

data/lib/httpx/plugins/cookies/cookie.rb

@@ -14,12 +14,14 @@ module HTTPX
 
       attr_reader :domain, :path, :name, :value, :created_at
 
+      # assigns a new +path+ to this cookie.
       def path=(path)
         path = String(path)
+        @for_domain = false
         @path = path.start_with?("/") ? path : "/"
       end
 
-      # See #domain.
+      # assigns a new +domain+ to this cookie.
       def domain=(domain)
         domain = String(domain)
 
@@ -37,6 +39,13 @@ module HTTPX
         @domain = @domain_name.hostname
       end
 
+      # checks whether +other+ is the same cookie, i.e. name, value, domain and path are
+      # the same.
+      def ==(other)
+        @name == other.name && @value == other.value &&
+          @path == other.path && @domain == other.domain
+      end
+
       # Compares the cookie with another.  When there are many cookies with
       # the same name for a URL, the value of the smallest must be used.
       def <=>(other)
@@ -47,11 +56,29 @@ module HTTPX
           (@created_at <=> other.created_at).nonzero? || 0
       end
 
+      def match?(name_or_options)
+        case name_or_options
+        when String
+          @name == name_or_options
+        when Hash, Array
+          name_or_options.all? { |k, v| respond_to?(k) && send(k) == v }
+        else
+          false
+        end
+      end
+
       class << self
         def new(cookie, *args)
-          return cookie if cookie.is_a?(self)
+          case cookie
+          when self
+            cookie
+          when Array, Hash
+            options = Hash[cookie] #: cookie_attributes
+            super(options[:name], options[:value], options)
+          else
 
-          super
+            super
+          end
         end
 
         # Tests if +target_path+ is under +base_path+ as described in RFC
@@ -84,16 +111,12 @@ module HTTPX
         end
       end
 
-      def initialize(arg, *attrs)
+      def initialize(arg, value, attrs = nil)
         @created_at = Time.now
 
-        if attrs.empty?
-          attr_hash = Hash.try_convert(arg)
-        else
-          @name = arg
-          @value, attr_hash = attrs
-          attr_hash = Hash.try_convert(attr_hash)
-        end
+        @name = arg
+        @value = value
+        attr_hash = Hash.try_convert(attrs)
 
         attr_hash.each do |key, val|
           key = key.downcase.tr("-", "_").to_sym unless key.is_a?(Symbol)

data/lib/httpx/plugins/cookies/jar.rb

@@ -4,7 +4,12 @@ module HTTPX
   module Plugins::Cookies
     # The Cookie Jar
     #
-    # It holds a bunch of cookies.
+    # It stores and manages cookies for a session, such as i.e. evicting when expired, access methods, or
+    # initialization from parsing `Set-Cookie` HTTP header values.
+    #
+    # It closely follows the [CookieStore API](https://developer.mozilla.org/en-US/docs/Web/API/CookieStore),
+    # by implementing the same methods, with a few specific conveniences for this non-browser manipulation use-case.
+    #
     class Jar
       using URIExtensions
 
@@ -12,10 +17,14 @@ module HTTPX
 
       def initialize_dup(orig)
         super
+        @mtx = orig.instance_variable_get(:@mtx).dup
         @cookies = orig.instance_variable_get(:@cookies).dup
       end
 
+      # initializes the cookie store, either empty, or with whatever is passed as +cookies+, which
+      # can be an array of HTTPX::Plugins::Cookies::Cookie objects or hashes-or-tuples of cookie attributes.
       def initialize(cookies = nil)
+        @mtx = Thread::Mutex.new
         @cookies = []
 
         cookies.each do |elem|
@@ -32,48 +41,106 @@ module HTTPX
         end if cookies
       end
 
+      # parses the `Set-Cookie` header value as +set_cookie+ and does the corresponding updates.
       def parse(set_cookie)
         SetCookieParser.call(set_cookie) do |name, value, attrs|
-          add(Cookie.new(name, value, attrs))
+          set(Cookie.new(name, value, attrs))
+        end
+      end
+
+      # returns the first HTTPX::Plugins::Cookie::Cookie instance in the store which matches either the name
+      # (when String) or all attributes (when a Hash or array of tuples) passed to +name_or_options+
+      def get(name_or_options)
+        each.find { |ck| ck.match?(name_or_options) }
+      end
+
+      # returns all HTTPX::Plugins::Cookie::Cookie instances in the store which match either the name
+      # (when String) or all attributes (when a Hash or array of tuples) passed to +name_or_options+
+      def get_all(name_or_options)
+        each.select { |ck| ck.match?(name_or_options) } # rubocop:disable Style/SelectByRegexp
+      end
+
+      # when +name+ is a HTTPX::Plugins::Cookie::Cookie, it stores it internally; when +name+ is a String,
+      # it creates a cookie with it and the value-or-attributes passed to +value_or_options+.
+
+      # optionally, +name+ can also be the attributes hash-or-array as long it contains a <tt>:name</tt> field).
+      def set(name, value_or_options = nil)
+        cookie = case name
+                 when Cookie
+                   raise ArgumentError, "there should not be a second argument" if value_or_options
+
+                   name
+                 when Array, Hash
+                   raise ArgumentError, "there should not be a second argument" if value_or_options
+
+                   Cookie.new(name)
+                 else
+                   raise ArgumentError, "the second argument is required" unless value_or_options
+
+                   Cookie.new(name, value_or_options)
+        end
+
+        synchronize do
+          # If the user agent receives a new cookie with the same cookie-name, domain-value, and path-value
+          # as a cookie that it has already stored, the existing cookie is evicted and replaced with the new cookie.
+          @cookies.delete_if { |ck| ck.name == cookie.name && ck.domain == cookie.domain && ck.path == cookie.path }
+
+          @cookies << cookie
         end
       end
 
+      # @deprecated
       def add(cookie, path = nil)
+        warn "DEPRECATION WARNING: calling `##{__method__}` is deprecated. Use `#set` instead."
         c = cookie.dup
-
         c.path = path if path && c.path == "/"
+        set(c)
+      end
 
-        # If the user agent receives a new cookie with the same cookie-name, domain-value, and path-value
-        # as a cookie that it has already stored, the existing cookie is evicted and replaced with the new cookie.
-        @cookies.delete_if { |ck| ck.name == c.name && ck.domain == c.domain && ck.path == c.path }
-
-        @cookies << c
+      # deletes all cookies  in the store which match either the name (when String) or all attributes (when a Hash
+      # or array of tuples) passed to +name_or_options+.
+      #
+      # alternatively, of +name_or_options+ is an instance of HTTPX::Plugins::Cookies::Cookiem, it deletes it from the store.
+      def delete(name_or_options)
+        synchronize do
+          case name_or_options
+          when Cookie
+            @cookies.delete(name_or_options)
+          else
+            @cookies.delete_if { |ck| ck.match?(name_or_options) }
+          end
+        end
       end
 
+      # returns the list of valid cookies which matdh the domain and path from the URI object passed to +uri+.
       def [](uri)
         each(uri).sort
       end
 
+      # enumerates over all stored cookies. if +uri+ is passed, it'll filter out expired cookies and
+      # only yield cookies which match its domain and path.
       def each(uri = nil, &blk)
         return enum_for(__method__, uri) unless blk
 
-        return @cookies.each(&blk) unless uri
+        return synchronize { @cookies.each(&blk) } unless uri
 
         now = Time.now
         tpath = uri.path
 
-        @cookies.delete_if do |cookie|
-          if cookie.expired?(now)
-            true
-          else
-            yield cookie if cookie.valid_for_uri?(uri) && Cookie.path_match?(cookie.path, tpath)
-            false
+        synchronize do
+          @cookies.delete_if do |cookie|
+            if cookie.expired?(now)
+              true
+            else
+              yield cookie if cookie.valid_for_uri?(uri) && Cookie.path_match?(cookie.path, tpath)
+              false
+            end
           end
         end
       end
 
       def merge(other)
-        cookies_dup = dup
+        jar_dup = dup
 
         other.each do |elem|
           cookie = case elem
@@ -85,10 +152,18 @@ module HTTPX
                      Cookie.new(elem)
           end
 
-          cookies_dup.add(cookie)
+          jar_dup.set(cookie)
         end
 
-        cookies_dup
+        jar_dup
+      end
+
+      private
+
+      def synchronize(&block)
+        return yield if @mtx.owned?
+
+        @mtx.synchronize(&block)
       end
     end
   end

data/lib/httpx/plugins/cookies.rb

@@ -7,8 +7,6 @@ module HTTPX
     #
     # This plugin implements a persistent cookie jar for the duration of a session.
     #
-    # It also adds a *#cookies* helper, so that you can pre-fill the cookies of a session.
-    #
     # https://gitlab.com/os85/httpx/wikis/Cookies
     #
     module Cookies
@@ -46,6 +44,12 @@ module HTTPX
           request
         end
 
+        # factory method to return a Jar to the user, which can then manipulate
+        # externally to the session.
+        def make_jar(*args)
+          Jar.new(*args)
+        end
+
         private
 
         def set_request_callbacks(request)
@@ -96,7 +100,7 @@ module HTTPX
           cookies.each do |ck|
             ck.split(/ *; */).each do |cookie|
               name, value = cookie.split("=", 2)
-              jar.add(Cookie.new(name, value))
+              jar.set(name, value)
             end
           end
         end

data/lib/httpx/plugins/expect.rb

@@ -9,10 +9,34 @@ module HTTPX
     #
     module Expect
       EXPECT_TIMEOUT = 2
+      NOEXPECT_STORE_MUTEX = Thread::Mutex.new
+
+      class Store
+        def initialize
+          @store = []
+          @mutex = Thread::Mutex.new
+        end
+
+        def include?(host)
+          @mutex.synchronize { @store.include?(host) }
+        end
+
+        def add(host)
+          @mutex.synchronize { @store << host }
+        end
+
+        def delete(host)
+          @mutex.synchronize { @store.delete(host) }
+        end
+      end
 
       class << self
         def no_expect_store
-          @no_expect_store ||= []
+          return Ractor.store_if_absent(:httpx_no_expect_store) { Store.new } if Utils.in_ractor?
+
+          @no_expect_store ||= NOEXPECT_STORE_MUTEX.synchronize do
+            @no_expect_store || Store.new
+          end
         end
 
         def extra_options(options)
@@ -46,6 +70,9 @@ module HTTPX
       module RequestMethods
         def initialize(*)
           super
+
+          @informational_status = nil
+
           return if @body.empty?
 
           threshold = @options.expect_threshold_size
@@ -89,7 +116,7 @@ module HTTPX
           set_request_timeout(:expect_timeout, request, expect_timeout, :expect, %i[body response]) do
             # expect timeout expired
             if request.state == :expect && !request.expects?
-              Expect.no_expect_store << request.origin
+              Expect.no_expect_store.add(request.origin)
               request.headers.delete("expect")
               consume
             end
@@ -108,7 +135,10 @@ module HTTPX
             request.headers.delete("expect")
             request.transition(:idle)
             send_request(request, selector, options)
-            return
+
+            # recalling itself, in case an error was triggered by the above, and we can
+            # verify retriability again.
+            return fetch_response(request, selector, options)
           end
 
           response

data/lib/httpx/plugins/fiber_concurrency.rb

@@ -160,9 +160,7 @@ module HTTPX
         end
       end
 
-      module NativeResolverMethods
-        private
-
+      module ResolverNativeMethods
         def calculate_interests
           return if @queries.empty?
 
@@ -172,7 +170,7 @@ module HTTPX
         end
       end
 
-      module SystemResolverMethods
+      module ResolverSystemMethods
         def interests
           return unless @queries.any? { |_, conn| conn.current_context? }
 

data/lib/httpx/plugins/follow_redirects.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
 module HTTPX
-  InsecureRedirectError = Class.new(Error)
+  class InsecureRedirectError < Error
+  end
+
   module Plugins
     #
     # This plugin adds support for automatically following redirect (status 30X) responses.
@@ -163,6 +165,10 @@ module HTTPX
             end
           else
             send_request(retry_request, selector, options)
+
+            # recalling itself, in case an error was triggered by the above, and we can
+            # verify retriability again.
+            return fetch_response(request, selector, options)
           end
           nil
         end

data/lib/httpx/plugins/h2c.rb

@@ -81,7 +81,7 @@ module HTTPX
           @parser.upgrade(request, response)
           @upgrade_protocol = "h2c"
 
-          prev_parser.requests.each do |req|
+          prev_parser.pending.each do |req|
             req.transition(:idle)
             send(req)
           end