httpx 1.7.2 → 1.7.4

data/lib/httpx/plugins/cookies/cookie.rb

@@ -14,12 +14,14 @@ module HTTPX
 
       attr_reader :domain, :path, :name, :value, :created_at
 
+      # assigns a new +path+ to this cookie.
       def path=(path)
         path = String(path)
+        @for_domain = false
         @path = path.start_with?("/") ? path : "/"
       end
 
-      # See #domain.
+      # assigns a new +domain+ to this cookie.
       def domain=(domain)
         domain = String(domain)
 
@@ -37,6 +39,13 @@ module HTTPX
         @domain = @domain_name.hostname
       end
 
+      # checks whether +other+ is the same cookie, i.e. name, value, domain and path are
+      # the same.
+      def ==(other)
+        @name == other.name && @value == other.value &&
+          @path == other.path && @domain == other.domain
+      end
+
       # Compares the cookie with another.  When there are many cookies with
       # the same name for a URL, the value of the smallest must be used.
       def <=>(other)
@@ -47,11 +56,29 @@ module HTTPX
           (@created_at <=> other.created_at).nonzero? || 0
       end
 
+      def match?(name_or_options)
+        case name_or_options
+        when String
+          @name == name_or_options
+        when Hash, Array
+          name_or_options.all? { |k, v| respond_to?(k) && send(k) == v }
+        else
+          false
+        end
+      end
+
       class << self
         def new(cookie, *args)
-          return cookie if cookie.is_a?(self)
+          case cookie
+          when self
+            cookie
+          when Array, Hash
+            options = Hash[cookie] #: cookie_attributes
+            super(options[:name], options[:value], options)
+          else
 
-          super
+            super
+          end
         end
 
         # Tests if +target_path+ is under +base_path+ as described in RFC
@@ -84,16 +111,12 @@ module HTTPX
         end
       end
 
-      def initialize(arg, *attrs)
+      def initialize(arg, value, attrs = nil)
         @created_at = Time.now
 
-        if attrs.empty?
-          attr_hash = Hash.try_convert(arg)
-        else
-          @name = arg
-          @value, attr_hash = attrs
-          attr_hash = Hash.try_convert(attr_hash)
-        end
+        @name = arg
+        @value = value
+        attr_hash = Hash.try_convert(attrs)
 
         attr_hash.each do |key, val|
           key = key.downcase.tr("-", "_").to_sym unless key.is_a?(Symbol)

data/lib/httpx/plugins/cookies/jar.rb

@@ -4,7 +4,12 @@ module HTTPX
   module Plugins::Cookies
     # The Cookie Jar
     #
-    # It holds a bunch of cookies.
+    # It stores and manages cookies for a session, such as i.e. evicting when expired, access methods, or
+    # initialization from parsing `Set-Cookie` HTTP header values.
+    #
+    # It closely follows the [CookieStore API](https://developer.mozilla.org/en-US/docs/Web/API/CookieStore),
+    # by implementing the same methods, with a few specific conveniences for this non-browser manipulation use-case.
+    #
     class Jar
       using URIExtensions
 
@@ -12,10 +17,14 @@ module HTTPX
 
       def initialize_dup(orig)
         super
+        @mtx = orig.instance_variable_get(:@mtx).dup
         @cookies = orig.instance_variable_get(:@cookies).dup
       end
 
+      # initializes the cookie store, either empty, or with whatever is passed as +cookies+, which
+      # can be an array of HTTPX::Plugins::Cookies::Cookie objects or hashes-or-tuples of cookie attributes.
       def initialize(cookies = nil)
+        @mtx = Thread::Mutex.new
         @cookies = []
 
         cookies.each do |elem|
@@ -32,48 +41,106 @@ module HTTPX
         end if cookies
       end
 
+      # parses the `Set-Cookie` header value as +set_cookie+ and does the corresponding updates.
       def parse(set_cookie)
         SetCookieParser.call(set_cookie) do |name, value, attrs|
-          add(Cookie.new(name, value, attrs))
+          set(Cookie.new(name, value, attrs))
+        end
+      end
+
+      # returns the first HTTPX::Plugins::Cookie::Cookie instance in the store which matches either the name
+      # (when String) or all attributes (when a Hash or array of tuples) passed to +name_or_options+
+      def get(name_or_options)
+        each.find { |ck| ck.match?(name_or_options) }
+      end
+
+      # returns all HTTPX::Plugins::Cookie::Cookie instances in the store which match either the name
+      # (when String) or all attributes (when a Hash or array of tuples) passed to +name_or_options+
+      def get_all(name_or_options)
+        each.select { |ck| ck.match?(name_or_options) } # rubocop:disable Style/SelectByRegexp
+      end
+
+      # when +name+ is a HTTPX::Plugins::Cookie::Cookie, it stores it internally; when +name+ is a String,
+      # it creates a cookie with it and the value-or-attributes passed to +value_or_options+.
+
+      # optionally, +name+ can also be the attributes hash-or-array as long it contains a <tt>:name</tt> field).
+      def set(name, value_or_options = nil)
+        cookie = case name
+                 when Cookie
+                   raise ArgumentError, "there should not be a second argument" if value_or_options
+
+                   name
+                 when Array, Hash
+                   raise ArgumentError, "there should not be a second argument" if value_or_options
+
+                   Cookie.new(name)
+                 else
+                   raise ArgumentError, "the second argument is required" unless value_or_options
+
+                   Cookie.new(name, value_or_options)
+        end
+
+        synchronize do
+          # If the user agent receives a new cookie with the same cookie-name, domain-value, and path-value
+          # as a cookie that it has already stored, the existing cookie is evicted and replaced with the new cookie.
+          @cookies.delete_if { |ck| ck.name == cookie.name && ck.domain == cookie.domain && ck.path == cookie.path }
+
+          @cookies << cookie
         end
       end
 
+      # @deprecated
       def add(cookie, path = nil)
+        warn "DEPRECATION WARNING: calling `##{__method__}` is deprecated. Use `#set` instead."
         c = cookie.dup
-
         c.path = path if path && c.path == "/"
+        set(c)
+      end
 
-        # If the user agent receives a new cookie with the same cookie-name, domain-value, and path-value
-        # as a cookie that it has already stored, the existing cookie is evicted and replaced with the new cookie.
-        @cookies.delete_if { |ck| ck.name == c.name && ck.domain == c.domain && ck.path == c.path }
-
-        @cookies << c
+      # deletes all cookies  in the store which match either the name (when String) or all attributes (when a Hash
+      # or array of tuples) passed to +name_or_options+.
+      #
+      # alternatively, of +name_or_options+ is an instance of HTTPX::Plugins::Cookies::Cookiem, it deletes it from the store.
+      def delete(name_or_options)
+        synchronize do
+          case name_or_options
+          when Cookie
+            @cookies.delete(name_or_options)
+          else
+            @cookies.delete_if { |ck| ck.match?(name_or_options) }
+          end
+        end
       end
 
+      # returns the list of valid cookies which matdh the domain and path from the URI object passed to +uri+.
       def [](uri)
         each(uri).sort
       end
 
+      # enumerates over all stored cookies. if +uri+ is passed, it'll filter out expired cookies and
+      # only yield cookies which match its domain and path.
       def each(uri = nil, &blk)
         return enum_for(__method__, uri) unless blk
 
-        return @cookies.each(&blk) unless uri
+        return synchronize { @cookies.each(&blk) } unless uri
 
         now = Time.now
         tpath = uri.path
 
-        @cookies.delete_if do |cookie|
-          if cookie.expired?(now)
-            true
-          else
-            yield cookie if cookie.valid_for_uri?(uri) && Cookie.path_match?(cookie.path, tpath)
-            false
+        synchronize do
+          @cookies.delete_if do |cookie|
+            if cookie.expired?(now)
+              true
+            else
+              yield cookie if cookie.valid_for_uri?(uri) && Cookie.path_match?(cookie.path, tpath)
+              false
+            end
           end
         end
       end
 
       def merge(other)
-        cookies_dup = dup
+        jar_dup = dup
 
         other.each do |elem|
           cookie = case elem
@@ -85,10 +152,18 @@ module HTTPX
                      Cookie.new(elem)
           end
 
-          cookies_dup.add(cookie)
+          jar_dup.set(cookie)
         end
 
-        cookies_dup
+        jar_dup
+      end
+
+      private
+
+      def synchronize(&block)
+        return yield if @mtx.owned?
+
+        @mtx.synchronize(&block)
       end
     end
   end

data/lib/httpx/plugins/cookies.rb

@@ -7,8 +7,6 @@ module HTTPX
     #
     # This plugin implements a persistent cookie jar for the duration of a session.
     #
-    # It also adds a *#cookies* helper, so that you can pre-fill the cookies of a session.
-    #
     # https://gitlab.com/os85/httpx/wikis/Cookies
     #
     module Cookies
@@ -46,6 +44,12 @@ module HTTPX
           request
         end
 
+        # factory method to return a Jar to the user, which can then manipulate
+        # externally to the session.
+        def make_jar(*args)
+          Jar.new(*args)
+        end
+
         private
 
         def set_request_callbacks(request)
@@ -96,7 +100,7 @@ module HTTPX
           cookies.each do |ck|
             ck.split(/ *; */).each do |cookie|
               name, value = cookie.split("=", 2)
-              jar.add(Cookie.new(name, value))
+              jar.set(name, value)
             end
           end
         end

data/lib/httpx/plugins/expect.rb

@@ -9,10 +9,34 @@ module HTTPX
     #
     module Expect
       EXPECT_TIMEOUT = 2
+      NOEXPECT_STORE_MUTEX = Thread::Mutex.new
+
+      class Store
+        def initialize
+          @store = []
+          @mutex = Thread::Mutex.new
+        end
+
+        def include?(host)
+          @mutex.synchronize { @store.include?(host) }
+        end
+
+        def add(host)
+          @mutex.synchronize { @store << host }
+        end
+
+        def delete(host)
+          @mutex.synchronize { @store.delete(host) }
+        end
+      end
 
       class << self
         def no_expect_store
-          @no_expect_store ||= []
+          return Ractor.store_if_absent(:httpx_no_expect_store) { Store.new } if Utils.in_ractor?
+
+          @no_expect_store ||= NOEXPECT_STORE_MUTEX.synchronize do
+            @no_expect_store || Store.new
+          end
         end
 
         def extra_options(options)
@@ -89,7 +113,7 @@ module HTTPX
           set_request_timeout(:expect_timeout, request, expect_timeout, :expect, %i[body response]) do
             # expect timeout expired
             if request.state == :expect && !request.expects?
-              Expect.no_expect_store << request.origin
+              Expect.no_expect_store.add(request.origin)
               request.headers.delete("expect")
               consume
             end
@@ -108,7 +132,10 @@ module HTTPX
             request.headers.delete("expect")
             request.transition(:idle)
             send_request(request, selector, options)
-            return
+
+            # recalling itself, in case an error was triggered by the above, and we can
+            # verify retriability again.
+            return fetch_response(request, selector, options)
           end
 
           response

data/lib/httpx/plugins/fiber_concurrency.rb

@@ -160,9 +160,7 @@ module HTTPX
         end
       end
 
-      module NativeResolverMethods
-        private
-
+      module ResolverNativeMethods
         def calculate_interests
           return if @queries.empty?
 
@@ -172,7 +170,7 @@ module HTTPX
         end
       end
 
-      module SystemResolverMethods
+      module ResolverSystemMethods
         def interests
           return unless @queries.any? { |_, conn| conn.current_context? }
 

data/lib/httpx/plugins/follow_redirects.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
 module HTTPX
-  InsecureRedirectError = Class.new(Error)
+  class InsecureRedirectError < Error
+  end
+
   module Plugins
     #
     # This plugin adds support for automatically following redirect (status 30X) responses.
@@ -163,6 +165,10 @@ module HTTPX
             end
           else
             send_request(retry_request, selector, options)
+
+            # recalling itself, in case an error was triggered by the above, and we can
+            # verify retriability again.
+            return fetch_response(request, selector, options)
           end
           nil
         end

data/lib/httpx/plugins/h2c.rb

@@ -81,7 +81,7 @@ module HTTPX
           @parser.upgrade(request, response)
           @upgrade_protocol = "h2c"
 
-          prev_parser.requests.each do |req|
+          prev_parser.pending.each do |req|
             req.transition(:idle)
             send(req)
           end

data/lib/httpx/plugins/proxy/http.rb

@@ -35,7 +35,10 @@ module HTTPX
               request.headers["proxy-authorization"] =
                 options.proxy.authenticate(request, response.headers["proxy-authenticate"])
               send_request(request, selector, options)
-              return
+
+              # recalling itself, in case an error was triggered by the above, and we can
+              # verify retriability again.
+              return fetch_response(request, selector, options)
             end
 
             response
@@ -46,7 +49,7 @@ module HTTPX
           def force_close(*)
             if @state == :connecting
               # proxy connect related requests should not be reenqueed
-              @parser.reset!
+              @parser.reset
               @inflight -= @parser.pending.size
               @parser.pending.clear
             end
@@ -67,18 +70,16 @@ module HTTPX
               return unless @io.connected?
 
               @parser || begin
-                @parser = parser_type(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
-                parser = @parser
+                @parser = parser = parser_type(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
                 parser.extend(ProxyParser)
                 parser.on(:response, &method(:__http_on_connect))
                 parser.on(:close) do
                   next unless @parser
 
                   reset
-                  disconnect
                 end
                 parser.on(:reset) do
-                  if parser.empty?
+                  if parser.pending.empty? && parser.empty?
                     reset
                   else
                     enqueue_pending_requests_from_parser(parser)
@@ -94,17 +95,23 @@ module HTTPX
                     # keep parser state around due to proxy auth protocol;
                     # intermediate authenticated request is already inside
                     # the parser
-                    parser = nil
+                    connect_request = parser = nil
 
                     if initial_state == :connecting
                       parser = @parser
                       @parser.reset
+                      if @pending.first.is_a?(ConnectRequest)
+                        connect_request = @pending.shift # this happened when reenqueing
+                      end
                     end
 
                     idling
 
                     @parser = parser
-
+                    if connect_request
+                      @inflight += 1
+                      parser.send(connect_request)
+                    end
                     transition(:connecting)
                   end
                 end

data/lib/httpx/plugins/proxy.rb

@@ -202,7 +202,10 @@ module HTTPX
               log { "failed connecting to proxy, trying next..." }
               request.transition(:idle)
               send_request(request, selector, options)
-              return
+
+              # recalling itself, in case an error was triggered by the above, and we can
+              # verify retriability again.
+              return fetch_response(request, selector, options)
             end
             response
           rescue ProxyError
@@ -320,7 +323,12 @@ module HTTPX
 
         def purge_after_closed
           super
-          @io = @io.proxy_io if @io.respond_to?(:proxy_io)
+
+          while @io.respond_to?(:proxy_io)
+            @io = @io.proxy_io
+
+            super
+          end
         end
       end
 

data/lib/httpx/plugins/rate_limiter.rb

@@ -16,25 +16,7 @@ module HTTPX
 
       class << self
         def load_dependencies(klass)
-          klass.plugin(:retries, retry_after: method(:retry_after_rate_limit))
-        end
-
-        # Servers send the "Retry-After" header field to indicate how long the
-        # user agent ought to wait before making a follow-up request.  When
-        # sent with a 503 (Service Unavailable) response, Retry-After indicates
-        # how long the service is expected to be unavailable to the client.
-        # When sent with any 3xx (Redirection) response, Retry-After indicates
-        # the minimum time that the user agent is asked to wait before issuing
-        # the redirected request.
-        #
-        def retry_after_rate_limit(_, response)
-          return unless response.is_a?(Response)
-
-          retry_after = response.headers["retry-after"]
-
-          return unless retry_after
-
-          Utils.parse_retry_after(retry_after)
+          klass.plugin(:retries)
         end
       end
 
@@ -52,6 +34,24 @@ module HTTPX
         def rate_limit_error?(response)
           response.is_a?(Response) && RATE_LIMIT_CODES.include?(response.status)
         end
+
+        # Servers send the "Retry-After" header field to indicate how long the
+        # user agent ought to wait before making a follow-up request.  When
+        # sent with a 503 (Service Unavailable) response, Retry-After indicates
+        # how long the service is expected to be unavailable to the client.
+        # When sent with any 3xx (Redirection) response, Retry-After indicates
+        # the minimum time that the user agent is asked to wait before issuing
+        # the redirected request.
+        #
+        def when_to_retry(_, response, options)
+          return super unless response.is_a?(Response)
+
+          retry_after = response.headers["retry-after"]
+
+          return super unless retry_after
+
+          Utils.parse_retry_after(retry_after)
+        end
       end
     end
 

data/lib/httpx/plugins/retries.rb

@@ -148,10 +148,7 @@ module HTTPX
             log { "failed to get response, #{request.retries} tries to go..." }
             prepare_to_retry(request, response)
 
-            retry_after = options.retry_after
-            retry_after = retry_after.call(request, response) if retry_after.respond_to?(:call)
-
-            if retry_after
+            if (retry_after = when_to_retry(request, response, options))
               # apply jitter
               if (jitter = request.options.retry_jitter)
                 retry_after = jitter.call(retry_after)
@@ -169,11 +166,15 @@ module HTTPX
                   send_request(request, selector, options)
                 end
               end
+
+              return
             else
               send_request(request, selector, options)
-            end
 
-            return
+              # recalling itself, in case an error was triggered by the above, and we can
+              # verify retriability again.
+              return fetch_response(request, selector, options)
+            end
           end
           response
         end
@@ -201,6 +202,12 @@ module HTTPX
           request.transition(:idle)
         end
 
+        def when_to_retry(request, response, options)
+          retry_after = options.retry_after
+          retry_after = retry_after.call(request, response) if retry_after.respond_to?(:call)
+          retry_after
+        end
+
         #
         # Attempt to set the request to perform a partial range request.
         # This happens if the peer server accepts byte-range requests, and
@@ -237,14 +244,15 @@ module HTTPX
         def initialize(*args)
           super
           @retries = @options.max_retries
+          @partial_response = nil
         end
 
         def response=(response)
-          if @partial_response
+          if (partial_response = @partial_response)
             if response.is_a?(Response) && response.status == 206
-              response.from_partial_response(@partial_response)
+              response.from_partial_response(partial_response)
             else
-              @partial_response.close
+              partial_response.close
             end
             @partial_response = nil
           end

data/lib/httpx/plugins/ssrf_filter.rb

@@ -106,6 +106,7 @@ module HTTPX
             error = ServerSideRequestForgeryError.new("#{request.uri} URI scheme not allowed")
             error.set_backtrace(caller)
             response = ErrorResponse.new(request, error)
+            request.response = response
             request.emit(:response, response)
             response
           end

data/lib/httpx/plugins/stream_bidi.rb

@@ -189,6 +189,10 @@ module HTTPX
           !@closed
         end
 
+        def force_close(*)
+          terminate
+        end
+
         def terminate
           return if @closed
 
@@ -202,6 +206,8 @@ module HTTPX
           terminate
         end
 
+        alias_method :on_io_error, :on_error
+
         # noop (the owner connection will take of it)
         def handle_socket_timeout(interval); end
       end