httpx 1.7.1 → 1.7.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a9c30e22a2d406a61ef87a58fddd607bb2b1fad7b50d837d8db062d4a538a2d2
-  data.tar.gz: 9c5e1997b9c03434071c59b2deb2b004f7b1a1077c2ccaed03b9e3a1db7aafe5
+  metadata.gz: 54c87d9d8b2be0d12570204fd3c60d37f82127b624dde3a233085d5bcb43778c
+  data.tar.gz: 8bfec9fadfe697d083d37a9317e9ec9d6e23016f174bd62cabf24275f18fee79
 SHA512:
-  metadata.gz: 87d99c4971b99f086f7811be81a8453e71bd2535120b5e560a2fc837d50e8a040e70ab1b2b191beee4b481fcea5063576d8895d318a0089183c21cb59fdd0f24
-  data.tar.gz: 85469cf5b5822990367f1e5591798a1512de90a60226853da4b4e6b00dde8816d6218dd2aacc9b5d85f84f6cb1637dd4f7b37836863f44144f05329e4fbdf7f5
+  metadata.gz: 2c425e8714c36bdca0ab5d068157caa94ca591829466ccdb888e903487aaab50c9a8756eaa53f1cd692a109160deca10ec9ce4405a7690f38149a9e95ec9fb17
+  data.tar.gz: 20c6ae3067f1fe187402c743fb1741a131ba09e5039ed86e5c5d2eb8c94d590c84644ca3e6ab5fd3948b635d328bba4149df3a1d94dde3dfbf257bd694730275

data/README.md

@@ -46,7 +46,9 @@ And that's the simplest one there is. But you can also do:
 HTTPX.post("http://example.com", form: { user: "john", password: "pass" })
 
 http = HTTPX.with(headers: { "x-my-name" => "joe" })
-http.patch("http://example.com/file", body: File.open("path/to/file")) # request body is streamed
+File.open("path/to/file") do |file|
+  http.patch("http://example.com/file", body: file) # request body is streamed
+end
 ```
 
 If you want to do some more things with the response, you can get an `HTTPX::Response`:

data/doc/release_notes/1_7_2.md

@@ -0,0 +1,6 @@
+# 1.7.2
+
+## Bugfixes
+
+* `:stream_bidi` plugin: when used with the `:retries` plugin, it will skip calling callbacks referencing state from the connection/stream the request was moved from.
+* `:auth` plugin: fix issue causing tokens to be concatenated on non-auth errors.

data/doc/release_notes/1_7_3.md

@@ -0,0 +1,29 @@
+# 1.7.3
+
+## Improvements
+
+### cookies plugin: Jar as CookieStore
+
+While previously an implementation detail, the cookie jar from a `:cookie` plugin-enabled session can now be manipulated by the end user:
+
+```ruby
+cookies_sess = HTTPX.plugin(:cookies)
+
+jar = cookies.make_jar
+
+sess = cookies_ses.with(cookies: jar)
+
+# perform requests using sess, get/set/delete cookies in jar
+```
+
+The jar API now closely follows the [Web Cookie Store API](https://developer.mozilla.org/en-US/docs/Web/API/CookieStore), by providing the same set of functions.
+
+Some API backwards compatibility is maintained, however since this was an internal implementation detail, this effort isn't meant to be thorough.
+
+## Bugfixes
+
+* `http-2`: clear buffered data chunks when receiving a `GOAWAY` stream frame; without this, the client kept sending the corresponding `DATA` frames, despite the peer server making it known that it wouldn't process it. While this is valid HTTP/2, this could increase the connection window until a point where it'd go over the max frame size. this issue was observed during large file uploads where the first request could fail and make the client renegotiate.
+* `webmock` adapter: fixed response body length accounting which was making `response.body.empty?` return true for responses with payload.
+* `:rate_limiter` plugin relies on an internal refactoring to be able to wait for the time suggested by the peer server instead of the potentially relying on custom user logic via own `:retry_after`.
+* `:fiber_concurrency`: fix wrong names for native/system resolver overrides.
+* connection: fix for race condition when closing the connection, where the state only transitions to `closed` after checking the connection back in to the pool, potentially corrupting it if another session meanwhile has picked it up and manipulated it.

data/lib/httpx/adapters/webmock.rb

@@ -82,6 +82,7 @@ module WebMock
 
         def mock!
           @mocked = true
+          @body.mock!
         end
 
         def mocked?
@@ -90,10 +91,8 @@ module WebMock
       end
 
       module ResponseBodyMethods
-        def decode_chunk(chunk)
-          return chunk if @response.mocked?
-
-          super
+        def mock!
+          @inflaters = nil
         end
       end
 

data/lib/httpx/connection/http1.rb

@@ -280,7 +280,6 @@ module HTTPX
     end
 
     def disable_pipelining
-      return if @requests.empty?
       # do not disable pipelining if already set to 1 request at a time
       return if @max_concurrent_requests == 1
 

data/lib/httpx/connection/http2.rb

@@ -3,8 +3,6 @@
 require "securerandom"
 require "http/2"
 
-HTTP2::Connection.__send__(:public, :send_buffer) if HTTP2::VERSION < "1.1.1"
-
 module HTTPX
   class Connection::HTTP2
     include Callbacks
@@ -215,9 +213,7 @@ module HTTPX
     def handle_stream(stream, request)
       request.on(:refuse, &method(:on_stream_refuse).curry(3)[stream, request])
       stream.on(:close, &method(:on_stream_close).curry(3)[stream, request])
-      stream.on(:half_close) do
-        log(level: 2) { "#{stream.id}: waiting for response..." }
-      end
+      stream.on(:half_close) { on_stream_half_close(stream, request) }
       stream.on(:altsvc, &method(:on_altsvc).curry(2)[request.origin])
       stream.on(:headers, &method(:on_stream_headers).curry(3)[stream, request])
       stream.on(:data, &method(:on_stream_data).curry(3)[stream, request])
@@ -302,7 +298,7 @@ module HTTPX
       end
 
       log(color: :yellow) do
-        h.map { |k, v| "#{stream.id}: <- HEADER: #{k}: #{log_redact_headers(v)}" }.join("\n")
+        h.map { |k, v| "#{stream.id}: <- HEADER: #{k}: #{k == ":status" ? v : log_redact_headers(v)}" }.join("\n")
       end
       _, status = h.shift
       headers = request.options.headers_class.new(h)
@@ -331,6 +327,16 @@ module HTTPX
       stream.close
     end
 
+    def on_stream_half_close(stream, _request)
+      unless stream.send_buffer.empty?
+        stream.send_buffer.clear
+        stream.data("", end_stream: true)
+      end
+
+      # TODO: omit log line if response already here
+      log(level: 2) { "#{stream.id}: waiting for response..." }
+    end
+
     def on_stream_close(stream, request, error)
       return if error == :stream_closed && !@streams.key?(request)
 
@@ -404,34 +410,39 @@ module HTTPX
 
     def on_frame_sent(frame)
       log(level: 2) { "#{frame[:stream]}: frame was sent!" }
-      log(level: 2, color: :blue) do
-        payload =
-          case frame[:type]
-          when :data
-            frame.merge(payload: frame[:payload].bytesize)
-          when :headers, :ping
-            frame.merge(payload: log_redact_headers(frame[:payload]))
-          else
-            frame
-          end
-        "#{frame[:stream]}: #{payload}"
-      end
+      log(level: 2, color: :blue) { "#{frame[:stream]}: #{frame_with_extra_info(frame)}" }
     end
 
     def on_frame_received(frame)
       log(level: 2) { "#{frame[:stream]}: frame was received!" }
-      log(level: 2, color: :magenta) do
-        payload =
-          case frame[:type]
-          when :data
-            frame.merge(payload: frame[:payload].bytesize)
-          when :headers, :ping
-            frame.merge(payload: log_redact_headers(frame[:payload]))
-          else
-            frame
-          end
-        "#{frame[:stream]}: #{payload}"
-      end
+      log(level: 2, color: :magenta) { "#{frame[:stream]}: #{frame_with_extra_info(frame)}" }
+    end
+
+    def frame_with_extra_info(frame)
+      case frame[:type]
+      when :data
+        frame.merge(payload: frame[:payload].bytesize)
+      when :headers, :ping
+        frame.merge(payload: log_redact_headers(frame[:payload]))
+      when :window_update
+        connection_or_stream = if (id = frame[:stream]).zero?
+          @connection
+        else
+          @streams.each_value.find { |s| s.id == id }
+        end
+        if connection_or_stream
+          frame.merge(
+            local_window: connection_or_stream.local_window,
+            remote_window: connection_or_stream.remote_window,
+            buffered_amount: connection_or_stream.buffered_amount,
+            stream_state: connection_or_stream.state,
+          )
+        else
+          frame
+        end
+      else
+        frame
+      end.merge(connection_state: @connection.state)
     end
 
     def on_altsvc(origin, frame)

data/lib/httpx/connection.rb

@@ -227,6 +227,9 @@ module HTTPX
         consume
       end
       nil
+    rescue IOError => e
+      @write_buffer.clear
+      on_io_error(e)
     rescue StandardError => e
       @write_buffer.clear
       on_error(e)
@@ -375,6 +378,11 @@ module HTTPX
       current_session.deselect_connection(self, current_selector, @cloned)
     end
 
+    def on_io_error(e)
+      on_error(e)
+      force_close(true)
+    end
+
     def on_error(error, request = nil)
       if error.is_a?(OperationTimeoutError)
 
@@ -493,7 +501,7 @@ module HTTPX
           # flush as many bytes as the sockets allow.
           #
           loop do
-            # buffer has been drainned, mark and exit the write loop.
+            # buffer has been drained, mark and exit the write loop.
             if @write_buffer.empty?
               # we only mark as drained on the first loop
               write_drained = write_drained.nil? && @inflight.positive?
@@ -586,6 +594,8 @@ module HTTPX
       # back to the pending list before the parser is reset.
       @inflight -= parser_pending_requests.size
       @pending.unshift(*parser_pending_requests)
+
+      parser.pending.clear
     end
 
     def build_parser(protocol = @io.protocol)
@@ -721,8 +731,6 @@ module HTTPX
 
         # do not deactivate connection in use
         return if @inflight.positive? || @parser.waiting_for_ping?
-
-        disconnect
       when :closing
         return unless connecting? || @state == :open
 
@@ -740,8 +748,9 @@ module HTTPX
         return unless @write_buffer.empty?
 
         purge_after_closed
-        disconnect if @pending.empty?
 
+        # TODO: should this raise an error instead?
+        return unless @pending.empty?
       when :already_open
         nextstate = :open
         # the first check for given io readiness must still use a timeout.
@@ -758,6 +767,11 @@ module HTTPX
       end
       log(level: 3) { "#{@state} -> #{nextstate}" }
       @state = nextstate
+      # post state change
+      case nextstate
+      when :closed, :inactive
+        disconnect
+      end
     end
 
     def close_sibling

data/lib/httpx/plugins/auth/digest.rb

@@ -8,7 +8,8 @@ module HTTPX
   module Plugins
     module Authentication
       class Digest
-        Error = Class.new(Error)
+        class Error < Error
+        end
 
         def initialize(user, password, hashed: false, **)
           @user = user

data/lib/httpx/plugins/auth.rb

@@ -69,7 +69,7 @@ module HTTPX
         private
 
         def send_request(request, *)
-          return super if @skip_auth_header_value
+          return super if @skip_auth_header_value || request.authorized?
 
           @auth_header_value ||= generate_auth_token
 
@@ -92,12 +92,31 @@ module HTTPX
       end
 
       module RequestMethods
+        def initialize(*)
+          super
+          @auth_token_value = nil
+        end
+
+        def authorized?
+          !@auth_token_value.nil?
+        end
+
+        def unauthorize!
+          return unless (auth_value = @auth_token_value)
+
+          @headers.get("authorization").delete(auth_value)
+
+          @auth_token_value = nil
+        end
+
         def authorize(auth_value)
           if (auth_type = @options.auth_header_type)
             auth_value = "#{auth_type} #{auth_value}"
           end
 
           @headers.add("authorization", auth_value)
+
+          @auth_token_value = auth_value
         end
       end
 
@@ -119,7 +138,7 @@ module HTTPX
             return unless auth_error?(response, request.options) ||
                           (@options.generate_auth_value_on_retry && @options.generate_auth_value_on_retry.call(response))
 
-            request.headers.get("authorization").pop
+            request.unauthorize!
             @auth_header_value = generate_auth_token
           end
 

data/lib/httpx/plugins/cookies/cookie.rb

@@ -14,12 +14,14 @@ module HTTPX
 
       attr_reader :domain, :path, :name, :value, :created_at
 
+      # assigns a new +path+ to this cookie.
       def path=(path)
         path = String(path)
+        @for_domain = false
         @path = path.start_with?("/") ? path : "/"
       end
 
-      # See #domain.
+      # assigns a new +domain+ to this cookie.
       def domain=(domain)
         domain = String(domain)
 
@@ -37,6 +39,13 @@ module HTTPX
         @domain = @domain_name.hostname
       end
 
+      # checks whether +other+ is the same cookie, i.e. name, value, domain and path are
+      # the same.
+      def ==(other)
+        @name == other.name && @value == other.value &&
+          @path == other.path && @domain == other.domain
+      end
+
       # Compares the cookie with another.  When there are many cookies with
       # the same name for a URL, the value of the smallest must be used.
       def <=>(other)
@@ -47,11 +56,29 @@ module HTTPX
           (@created_at <=> other.created_at).nonzero? || 0
       end
 
+      def match?(name_or_options)
+        case name_or_options
+        when String
+          @name == name_or_options
+        when Hash, Array
+          name_or_options.all? { |k, v| respond_to?(k) && send(k) == v }
+        else
+          false
+        end
+      end
+
       class << self
         def new(cookie, *args)
-          return cookie if cookie.is_a?(self)
+          case cookie
+          when self
+            cookie
+          when Array, Hash
+            options = Hash[cookie] #: cookie_attributes
+            super(options[:name], options[:value], options)
+          else
 
-          super
+            super
+          end
         end
 
         # Tests if +target_path+ is under +base_path+ as described in RFC
@@ -84,16 +111,12 @@ module HTTPX
         end
       end
 
-      def initialize(arg, *attrs)
+      def initialize(arg, value, attrs = nil)
         @created_at = Time.now
 
-        if attrs.empty?
-          attr_hash = Hash.try_convert(arg)
-        else
-          @name = arg
-          @value, attr_hash = attrs
-          attr_hash = Hash.try_convert(attr_hash)
-        end
+        @name = arg
+        @value = value
+        attr_hash = Hash.try_convert(attrs)
 
         attr_hash.each do |key, val|
           key = key.downcase.tr("-", "_").to_sym unless key.is_a?(Symbol)

data/lib/httpx/plugins/cookies/jar.rb

@@ -4,7 +4,12 @@ module HTTPX
   module Plugins::Cookies
     # The Cookie Jar
     #
-    # It holds a bunch of cookies.
+    # It stores and manages cookies for a session, such as i.e. evicting when expired, access methods, or
+    # initialization from parsing `Set-Cookie` HTTP header values.
+    #
+    # It closely follows the [CookieStore API](https://developer.mozilla.org/en-US/docs/Web/API/CookieStore),
+    # by implementing the same methods, with a few specific conveniences for this non-browser manipulation use-case.
+    #
     class Jar
       using URIExtensions
 
@@ -12,10 +17,14 @@ module HTTPX
 
       def initialize_dup(orig)
         super
+        @mtx = orig.instance_variable_get(:@mtx).dup
         @cookies = orig.instance_variable_get(:@cookies).dup
       end
 
+      # initializes the cookie store, either empty, or with whatever is passed as +cookies+, which
+      # can be an array of HTTPX::Plugins::Cookies::Cookie objects or hashes-or-tuples of cookie attributes.
       def initialize(cookies = nil)
+        @mtx = Thread::Mutex.new
         @cookies = []
 
         cookies.each do |elem|
@@ -32,48 +41,106 @@ module HTTPX
         end if cookies
       end
 
+      # parses the `Set-Cookie` header value as +set_cookie+ and does the corresponding updates.
       def parse(set_cookie)
         SetCookieParser.call(set_cookie) do |name, value, attrs|
-          add(Cookie.new(name, value, attrs))
+          set(Cookie.new(name, value, attrs))
+        end
+      end
+
+      # returns the first HTTPX::Plugins::Cookie::Cookie instance in the store which matches either the name
+      # (when String) or all attributes (when a Hash or array of tuples) passed to +name_or_options+
+      def get(name_or_options)
+        each.find { |ck| ck.match?(name_or_options) }
+      end
+
+      # returns all HTTPX::Plugins::Cookie::Cookie instances in the store which match either the name
+      # (when String) or all attributes (when a Hash or array of tuples) passed to +name_or_options+
+      def get_all(name_or_options)
+        each.select { |ck| ck.match?(name_or_options) } # rubocop:disable Style/SelectByRegexp
+      end
+
+      # when +name+ is a HTTPX::Plugins::Cookie::Cookie, it stores it internally; when +name+ is a String,
+      # it creates a cookie with it and the value-or-attributes passed to +value_or_options+.
+
+      # optionally, +name+ can also be the attributes hash-or-array as long it contains a <tt>:name</tt> field).
+      def set(name, value_or_options = nil)
+        cookie = case name
+                 when Cookie
+                   raise ArgumentError, "there should not be a second argument" if value_or_options
+
+                   name
+                 when Array, Hash
+                   raise ArgumentError, "there should not be a second argument" if value_or_options
+
+                   Cookie.new(name)
+                 else
+                   raise ArgumentError, "the second argument is required" unless value_or_options
+
+                   Cookie.new(name, value_or_options)
+        end
+
+        synchronize do
+          # If the user agent receives a new cookie with the same cookie-name, domain-value, and path-value
+          # as a cookie that it has already stored, the existing cookie is evicted and replaced with the new cookie.
+          @cookies.delete_if { |ck| ck.name == cookie.name && ck.domain == cookie.domain && ck.path == cookie.path }
+
+          @cookies << cookie
         end
       end
 
+      # @deprecated
       def add(cookie, path = nil)
+        warn "DEPRECATION WARNING: calling `##{__method__}` is deprecated. Use `#set` instead."
         c = cookie.dup
-
         c.path = path if path && c.path == "/"
+        set(c)
+      end
 
-        # If the user agent receives a new cookie with the same cookie-name, domain-value, and path-value
-        # as a cookie that it has already stored, the existing cookie is evicted and replaced with the new cookie.
-        @cookies.delete_if { |ck| ck.name == c.name && ck.domain == c.domain && ck.path == c.path }
-
-        @cookies << c
+      # deletes all cookies  in the store which match either the name (when String) or all attributes (when a Hash
+      # or array of tuples) passed to +name_or_options+.
+      #
+      # alternatively, of +name_or_options+ is an instance of HTTPX::Plugins::Cookies::Cookiem, it deletes it from the store.
+      def delete(name_or_options)
+        synchronize do
+          case name_or_options
+          when Cookie
+            @cookies.delete(name_or_options)
+          else
+            @cookies.delete_if { |ck| ck.match?(name_or_options) }
+          end
+        end
       end
 
+      # returns the list of valid cookies which matdh the domain and path from the URI object passed to +uri+.
       def [](uri)
         each(uri).sort
       end
 
+      # enumerates over all stored cookies. if +uri+ is passed, it'll filter out expired cookies and
+      # only yield cookies which match its domain and path.
       def each(uri = nil, &blk)
         return enum_for(__method__, uri) unless blk
 
-        return @cookies.each(&blk) unless uri
+        return synchronize { @cookies.each(&blk) } unless uri
 
         now = Time.now
         tpath = uri.path
 
-        @cookies.delete_if do |cookie|
-          if cookie.expired?(now)
-            true
-          else
-            yield cookie if cookie.valid_for_uri?(uri) && Cookie.path_match?(cookie.path, tpath)
-            false
+        synchronize do
+          @cookies.delete_if do |cookie|
+            if cookie.expired?(now)
+              true
+            else
+              yield cookie if cookie.valid_for_uri?(uri) && Cookie.path_match?(cookie.path, tpath)
+              false
+            end
           end
         end
       end
 
       def merge(other)
-        cookies_dup = dup
+        jar_dup = dup
 
         other.each do |elem|
           cookie = case elem
@@ -85,10 +152,18 @@ module HTTPX
                      Cookie.new(elem)
           end
 
-          cookies_dup.add(cookie)
+          jar_dup.set(cookie)
         end
 
-        cookies_dup
+        jar_dup
+      end
+
+      private
+
+      def synchronize(&block)
+        return yield if @mtx.owned?
+
+        @mtx.synchronize(&block)
       end
     end
   end

data/lib/httpx/plugins/cookies.rb

@@ -7,8 +7,6 @@ module HTTPX
     #
     # This plugin implements a persistent cookie jar for the duration of a session.
     #
-    # It also adds a *#cookies* helper, so that you can pre-fill the cookies of a session.
-    #
     # https://gitlab.com/os85/httpx/wikis/Cookies
     #
     module Cookies
@@ -46,6 +44,12 @@ module HTTPX
           request
         end
 
+        # factory method to return a Jar to the user, which can then manipulate
+        # externally to the session.
+        def make_jar(*args)
+          Jar.new(*args)
+        end
+
         private
 
         def set_request_callbacks(request)
@@ -96,7 +100,7 @@ module HTTPX
           cookies.each do |ck|
             ck.split(/ *; */).each do |cookie|
               name, value = cookie.split("=", 2)
-              jar.add(Cookie.new(name, value))
+              jar.set(name, value)
             end
           end
         end

data/lib/httpx/plugins/expect.rb

@@ -9,10 +9,34 @@ module HTTPX
     #
     module Expect
       EXPECT_TIMEOUT = 2
+      NOEXPECT_STORE_MUTEX = Thread::Mutex.new
+
+      class Store
+        def initialize
+          @store = []
+          @mutex = Thread::Mutex.new
+        end
+
+        def include?(host)
+          @mutex.synchronize { @store.include?(host) }
+        end
+
+        def add(host)
+          @mutex.synchronize { @store << host }
+        end
+
+        def delete(host)
+          @mutex.synchronize { @store.delete(host) }
+        end
+      end
 
       class << self
         def no_expect_store
-          @no_expect_store ||= []
+          return Ractor.store_if_absent(:httpx_no_expect_store) { Store.new } if Utils.in_ractor?
+
+          @no_expect_store ||= NOEXPECT_STORE_MUTEX.synchronize do
+            @no_expect_store || Store.new
+          end
         end
 
         def extra_options(options)
@@ -89,7 +113,7 @@ module HTTPX
           set_request_timeout(:expect_timeout, request, expect_timeout, :expect, %i[body response]) do
             # expect timeout expired
             if request.state == :expect && !request.expects?
-              Expect.no_expect_store << request.origin
+              Expect.no_expect_store.add(request.origin)
               request.headers.delete("expect")
               consume
             end

data/lib/httpx/plugins/fiber_concurrency.rb

@@ -160,9 +160,7 @@ module HTTPX
         end
       end
 
-      module NativeResolverMethods
-        private
-
+      module ResolverNativeMethods
         def calculate_interests
           return if @queries.empty?
 
@@ -172,7 +170,7 @@ module HTTPX
         end
       end
 
-      module SystemResolverMethods
+      module ResolverSystemMethods
         def interests
           return unless @queries.any? { |_, conn| conn.current_context? }