patient_http 1.4.0 → 1.6.0

data/lib/patient_http/response_reader.rb

@@ -1,10 +1,13 @@
 # frozen_string_literal: true
 
 module PatientHttp
-  # Reads and validates HTTP response bodies.
+  # Reads and decodes HTTP response bodies.
   #
-  # Encapsulates the logic for reading async HTTP responses with size validation
-  # and building Response objects from the raw response data.
+  # Reading happens on the reactor thread and collects the raw (possibly
+  # compressed) body chunks with size validation. Decoding — joining the
+  # chunks, inflating compressed content, and applying the charset — is a
+  # separate step so it can run on a completion worker thread instead of
+  # blocking the event loop.
   class ResponseReader
     # Raised when a body read is aborted because the processor was stopped
     # past its shutdown deadline. The shutdown sequence re-enqueues the task,
@@ -13,41 +16,210 @@ module PatientHttp
     # @api private
     class ReadAbortedError < StandardError; end
 
+    # Content encodings that are inflated during decoding, mapped to the window
+    # bits of each wire format the encoding may arrive in. Formats are tried in
+    # order until one inflates the body.
+    #
+    # A "deflate" body should carry a zlib header (RFC 9110 specifies the zlib
+    # format), but some servers send a bare deflate stream instead, so the raw
+    # format is kept as a fallback.
+    INFLATE_WINDOW_BITS = {
+      "gzip" => [Zlib::MAX_WBITS | 16].freeze,
+      "deflate" => [Zlib::MAX_WBITS, -Zlib::MAX_WBITS].freeze
+    }.freeze
+
+    # Content encoding that means the body was not encoded at all. It needs no
+    # work to decode, but it still has to be recognized so it does not stop the
+    # decode of the encodings applied before it.
+    IDENTITY_ENCODING = "identity"
+
+    class << self
+      # Split the encodings named in the content-encoding header into the ones
+      # that stay applied to the body and the ones that can be decoded.
+      #
+      # A body can carry more than one encoding. They are listed in the order
+      # they were applied, so decoding runs from the last name backwards and
+      # stops at the first name it does not recognize. Everything before that
+      # point stays applied to the body.
+      #
+      # @param headers_hash [Hash] the response headers
+      # @return [Array(Array<String>, Array<String>)] the encodings that remain
+      #   applied and the encodings that can be decoded, both in applied order
+      def split_encodings(headers_hash)
+        encodings = content_encodings(headers_hash)
+        boundary = encodings.rindex { |name| !decodable?(name) }
+        return [[], encodings] if boundary.nil?
+
+        [encodings[0..boundary], encodings[(boundary + 1)..]]
+      end
+
+      # Parse the content-encoding header into encoding names.
+      #
+      # @param headers_hash [Hash] the response headers
+      # @return [Array<String>] the lowercased encoding names in applied order
+      def content_encodings(headers_hash)
+        headers_hash["content-encoding"].to_s.split(",").filter_map do |name|
+          name = name.strip.downcase
+          name unless name.empty?
+        end
+      end
+
+      # @param name [String] a lowercased content encoding name
+      # @return [Boolean] true if the reader can remove this encoding
+      def decodable?(name)
+        name == IDENTITY_ENCODING || INFLATE_WINDOW_BITS.key?(name)
+      end
+
+      # Restate the content-encoding header for a decoded body. The header is
+      # removed when nothing is left applied, and narrowed to the encodings the
+      # reader could not remove otherwise, so the header always describes the
+      # body delivered with it.
+      #
+      # @param headers_hash [Hash] the response headers
+      # @return [Hash] the headers with content-encoding updated or removed
+      def rewrite_content_encoding(headers_hash)
+        return headers_hash unless headers_hash.key?("content-encoding")
+
+        remaining, _decodable = split_encodings(headers_hash)
+
+        if remaining.empty?
+          headers_hash.except("content-encoding")
+        else
+          headers_hash.merge("content-encoding" => remaining.join(", "))
+        end
+      end
+    end
+
     # Initialize the reader.
     #
-    # @param processor [Processor] the processor object
-    def initialize(processor)
+    # Reading needs a processor so it can abort once the processor is past its
+    # shutdown deadline. Decoding needs only the configuration, so a caller
+    # that does its own reading can supply the configuration on its own.
+    #
+    # @param processor [Processor, nil] the processor object
+    # @param config [Configuration, nil] the configuration; defaults to the
+    #   processor's configuration
+    def initialize(processor, config: nil)
       @processor = processor
+      @config = config || processor.config
     end
 
-    # Read the response body with size validation.
+    # Read the raw response body chunks with size validation.
     #
     # Reads the async HTTP response body asynchronously to completion, which allows
     # the connection to be reused. The async-http client handles connection pooling
     # and keep-alive internally. Using iteration instead of read() ensures non-blocking
-    # I/O that yields to the reactor.
+    # I/O that yields to the reactor. The chunks are the wire bytes: when the
+    # response is compressed, the size check here applies to the compressed
+    # bytes and {#decode_body} applies the same limit to the inflated bytes.
+    #
+    # The Content-Length header is checked against the size limit before the
+    # read starts. A response to a HEAD request has an empty body but reports
+    # the Content-Length of the resource, so the header check is skipped when
+    # the body reports itself as empty.
     #
     # @param async_response [Async::HTTP::Protocol::Response] the async HTTP response
     # @param headers_hash [Hash] the response headers
-    # @return [String, nil] the response body or nil if no body present
-    # @raise [ResponseTooLargeError] if body exceeds max_response_size
+    # @return [Array<String>, nil] the raw body chunks or nil if no body present
+    # @raise [ResponseTooLargeError] if the body exceeds max_response_size
     # @raise [ReadAbortedError] if the processor stopped past its shutdown deadline mid-read
-    def read_body(async_response, headers_hash)
-      return nil unless async_response.body
+    def read_raw_body(async_response, headers_hash)
+      body = async_response.body
+      return nil unless body
+
+      validate_content_length(headers_hash) unless body.empty?
+      read_body_chunks(async_response)
+    end
+
+    # Decode raw body chunks into the final body string.
+    #
+    # Joins the chunks, inflates gzip/deflate content (enforcing
+    # max_response_size on the inflated bytes), and applies the charset from
+    # the Content-Type header. This is CPU-bound work intended to run on a
+    # completion worker thread.
+    #
+    # An encoding the reader does not support leaves the body encoded, and a
+    # body that is still encoded keeps its binary encoding because the charset
+    # does not describe it. Use {.split_encodings} to find what stays applied
+    # so the content-encoding header delivered with the response describes the
+    # body it carries.
+    #
+    # @param chunks [Array<String>, nil] the raw body chunks
+    # @param headers_hash [Hash] the response headers
+    # @return [String, nil] the decoded body or nil if there was no body
+    # @raise [ResponseTooLargeError] if the inflated body exceeds max_response_size
+    def decode_body(chunks, headers_hash)
+      return nil if chunks.nil?
+
+      remaining, decodable = self.class.split_encodings(headers_hash)
+      warn_undecodable(remaining) unless remaining.empty?
+
+      body = inflate_encodings(chunks, decodable).join
+      body.force_encoding(Encoding::ASCII_8BIT)
+      # A body that is still encoded is not text yet, so the charset does not
+      # describe its bytes. Leave it binary for the caller to decode.
+      return body unless remaining.empty?
 
-      validate_content_length(headers_hash)
-      body = read_body_chunks(async_response)
       apply_charset_encoding(body, headers_hash)
     end
 
     private
 
+    # Remove the given encodings from the body, starting with the one applied
+    # last. Identity needs no work; every other name here inflates.
+    #
+    # @param chunks [Array<String>] the encoded body chunks
+    # @param encodings [Array<String>] decodable encoding names in applied order
+    # @return [Array<String>] the decoded chunks
+    # @raise [ResponseTooLargeError] if the inflated body exceeds max_response_size
+    def inflate_encodings(chunks, encodings)
+      encodings.reverse_each do |name|
+        chunks = [inflate_encoding(chunks, name)] if INFLATE_WINDOW_BITS.key?(name)
+      end
+
+      chunks
+    end
+
+    # Inflate one encoding, trying each wire format the encoding can use. The
+    # chunks are all in memory, so a format that turns out to be wrong can be
+    # abandoned and the next one started from the beginning of the body.
+    #
+    # @param chunks [Array<String>] the encoded body chunks
+    # @param name [String] a lowercased content encoding name
+    # @return [String] the inflated body
+    # @raise [Zlib::Error] if no format could inflate the body
+    # @raise [ResponseTooLargeError] if the inflated body exceeds max_response_size
+    def inflate_encoding(chunks, name)
+      formats = INFLATE_WINDOW_BITS.fetch(name)
+      last_index = formats.size - 1
+
+      formats.each_with_index do |window_bits, index|
+        return inflate_chunks(chunks, window_bits)
+      rescue Zlib::DataError, Zlib::BufError
+        raise if index == last_index
+      end
+    end
+
+    # Report an encoding that could not be removed. The body is still delivered
+    # with its content-encoding header, so the caller can decode it, but the
+    # server ignored the accept-encoding header and that is worth recording.
+    #
+    # @param remaining [Array<String>] the encodings left on the body
+    # @return [void]
+    def warn_undecodable(remaining)
+      logger&.warn(
+        "[PatientHttp] Cannot decode response body with content-encoding " \
+        "'#{remaining.join(", ")}'; returning the encoded body"
+      )
+      nil
+    end
+
     def max_response_size
-      @processor.config.max_response_size
+      @config.max_response_size
     end
 
     def logger
-      @processor.config.logger
+      @config.logger
     end
 
     # Validate content-length header doesn't exceed max size.
@@ -66,7 +238,7 @@ module PatientHttp
     # Read body chunks while checking size.
     #
     # @param async_response [Async::HTTP::Protocol::Response] the async HTTP response
-    # @return [String] the response body in ASCII-8BIT encoding
+    # @return [Array<String>] the raw body chunks
     # @raise [ResponseTooLargeError] if body size exceeds max_response_size during read
     # @raise [ReadAbortedError] if the processor stopped past its shutdown deadline mid-read
     def read_body_chunks(async_response)
@@ -80,7 +252,7 @@ module PatientHttp
           # Reads are allowed to finish while the processor is merely stopping
           # (the graceful shutdown window) so in-flight responses can still be
           # delivered.
-          if @processor.stopped?
+          if @processor&.stopped?
             raise ReadAbortedError.new("Processor stopped while reading response body")
           end
 
@@ -97,8 +269,7 @@ module PatientHttp
 
         finished = true
 
-        # Join chunks and force to binary encoding to preserve raw bytes
-        chunks.join.force_encoding(Encoding::ASCII_8BIT)
+        chunks
       ensure
         # Always close the body if we were interrupted or if an error occurred
         # This ensures the connection is properly released back to the pool
@@ -106,6 +277,49 @@ module PatientHttp
       end
     end
 
+    # Inflate compressed body chunks with streaming size enforcement, so a
+    # small compressed body cannot expand past max_response_size.
+    #
+    # @param chunks [Array<String>] the raw compressed chunks
+    # @param window_bits [Integer] Zlib window bits for the content encoding
+    # @return [String] the inflated body
+    # @raise [ResponseTooLargeError] if the inflated size exceeds max_response_size
+    def inflate_chunks(chunks, window_bits)
+      # A response can declare a content encoding and still carry no body.
+      # There is nothing to inflate, and finishing an empty stream would
+      # raise a buffer error.
+      return +"" if chunks.all?(&:empty?)
+
+      inflater = Zlib::Inflate.new(window_bits)
+      body = +""
+
+      begin
+        # The block form yields the inflated output in buffer-sized pieces, so
+        # the size is checked before the whole expansion is materialized. A
+        # single small compressed chunk can otherwise inflate to gigabytes
+        # before any check runs.
+        appender = ->(output) do
+          body << output
+          validate_inflated_size(body)
+        end
+
+        chunks.each { |chunk| inflater.inflate(chunk, &appender) }
+        inflater.finish(&appender) unless inflater.finished?
+      ensure
+        inflater.close
+      end
+
+      body
+    end
+
+    def validate_inflated_size(body)
+      if body.bytesize > max_response_size
+        raise ResponseTooLargeError.new(
+          "Response body size exceeded maximum allowed size (#{max_response_size} bytes)"
+        )
+      end
+    end
+
     # Extract charset from Content-Type header.
     #
     # @param headers_hash [Hash] the response headers

data/lib/patient_http/synchronous_executor.rb

@@ -20,6 +20,7 @@ module PatientHttp
       @on_error = on_error
       @proxy_client = nil
       @request_preparer = RequestPreparer.new(config)
+      @response_reader = ResponseReader.new(nil, config: config)
     end
 
     # Execute the request synchronously.
@@ -62,11 +63,12 @@ module PatientHttp
               # flattened to a single joined string value.
               headers_hash = async_response.headers.to_h.transform_values(&:to_s)
 
-              body_content = read_response_body(async_response, headers_hash)
+              chunks = @response_reader.read_raw_body(async_response, headers_hash)
+              body_content = @response_reader.decode_body(chunks, headers_hash)
 
               {
                 status: async_response.status,
-                headers: headers_hash,
+                headers: ResponseReader.rewrite_content_encoding(headers_hash),
                 body: body_content
               }
             end
@@ -79,8 +81,7 @@ module PatientHttp
             redirect_error = check_redirect_error(@task, response_data)
             break if redirect_error
 
-            location = response_data[:headers]["location"]
-            @task = @task.redirect_task(location: location, status: response_data[:status])
+            @task = build_redirect_task(@task, response_data)
           end
 
           if redirect_error
@@ -133,19 +134,22 @@ module PatientHttp
 
     # Create HTTP client with config settings (retries, proxy, connection timeout).
     #
+    # The client is not wrapped in a Protocol::HTTP::AcceptEncoding middleware.
+    # That wrapper overwrites the request's accept-encoding header, which would
+    # ignore a caller opting out of compression, so response bodies are decoded
+    # by ResponseReader here exactly as they are on the async path.
+    #
     # @param url [String] the resolved request URL
-    # @return [Protocol::HTTP::AcceptEncoding] wrapped HTTP client
+    # @return [Async::HTTP::Client] the HTTP client
     def create_http_client(url)
       endpoint = Async::HTTP::Endpoint.parse(url)
       endpoint = configure_endpoint(endpoint) if @config.connection_timeout
 
-      client = if @config.proxy_url
+      if @config.proxy_url
         create_proxied_client(endpoint)
       else
         Async::HTTP::Client.new(endpoint, retries: @config.retries)
       end
-
-      Protocol::HTTP::AcceptEncoding.new(client)
     end
 
     # Create a proxied HTTP client.
@@ -174,69 +178,6 @@ module PatientHttp
       )
     end
 
-    # Read the response body with size validation.
-    #
-    # @param async_response [Async::HTTP::Protocol::Response] the async HTTP response
-    # @param headers_hash [Hash] the response headers
-    # @return [String, nil] the response body
-    def read_response_body(async_response, headers_hash)
-      return nil unless async_response.body
-
-      content_length = headers_hash["content-length"]&.to_i
-      if content_length && content_length > @config.max_response_size
-        raise ResponseTooLargeError.new(
-          "Response body size (#{content_length} bytes) exceeds maximum allowed size (#{@config.max_response_size} bytes)"
-        )
-      end
-
-      chunks = []
-      total_size = 0
-      finished = false
-
-      begin
-        async_response.body.each do |chunk|
-          total_size += chunk.bytesize
-          if total_size > @config.max_response_size
-            raise ResponseTooLargeError.new(
-              "Response body size exceeded maximum allowed size (#{@config.max_response_size} bytes)"
-            )
-          end
-          chunks << chunk
-        end
-
-        finished = true
-      ensure
-        # Close the body if the read was interrupted so the connection is released
-        async_response.body.close unless finished
-      end
-
-      body = chunks.join.force_encoding(Encoding::ASCII_8BIT)
-
-      charset = extract_charset(headers_hash)
-      if charset
-        begin
-          encoding = Encoding.find(charset)
-          body.force_encoding(encoding)
-        rescue ArgumentError
-          # Invalid charset, keep binary
-        end
-      end
-
-      body
-    end
-
-    # Extract charset from Content-Type header.
-    def extract_charset(headers_hash)
-      content_type = headers_hash["content-type"]
-      return nil unless content_type
-
-      match = content_type.match(/;\s*charset\s*=\s*([^;\s]+)/i)
-      return nil unless match
-
-      charset = match[1].strip
-      charset.gsub(/\A["']|["']\z/, "")
-    end
-
     # Invoke callback synchronously.
     #
     # @param result [Response, Error] the result to pass to callback

data/lib/patient_http.rb

@@ -30,8 +30,15 @@ module PatientHttp
 
   class ResponseTooLargeError < StandardError; end
 
-  # HTTP redirect status codes that should be followed
-  FOLLOWABLE_REDIRECT_STATUSES = [301, 302, 303, 307, 308].freeze
+  # Raised when a request names a processor that is not configured. Handlers
+  # that support named processors raise this at enqueue time; the executing
+  # side raises it for a job that names an unconfigured processor so the job
+  # lands in the job system's retry mechanism instead of being dropped.
+  class UnknownProcessorError < StandardError; end
+
+  # HTTP redirect status codes that are followed when a Location header is present.
+  # A 300 response is followed only when the server names a preferred choice in Location.
+  FOLLOWABLE_REDIRECT_STATUSES = [300, 301, 302, 303, 307, 308].freeze
 
   VERSION = File.read(File.join(__dir__, "../VERSION")).strip
 
@@ -45,6 +52,7 @@ module PatientHttp
   autoload :Client, File.join(__dir__, "patient_http/client")
   autoload :ClientError, File.join(__dir__, "patient_http/http_error")
   autoload :ClientPool, File.join(__dir__, "patient_http/client_pool")
+  autoload :CompletionExecutor, File.join(__dir__, "patient_http/completion_executor")
   autoload :Configuration, File.join(__dir__, "patient_http/configuration")
   autoload :Encryptor, File.join(__dir__, "patient_http/encryptor")
   autoload :Error, File.join(__dir__, "patient_http/error")
@@ -268,6 +276,16 @@ module PatientHttp
       request(:get, uri, callback: callback, **kwargs)
     end
 
+    # Enqueues an HTTP HEAD request.
+    #
+    # @param uri [String] absolute URL
+    # @param callback [Class, String] callback class to handle the response
+    # @param kwargs [Hash] forwarded to `request`
+    # @return [Object] return value from the registered request handler
+    def head(uri, callback:, **kwargs)
+      request(:head, uri, callback: callback, **kwargs)
+    end
+
     # Enqueues an HTTP POST request.
     #
     # @param uri [String] absolute URL
@@ -308,9 +326,19 @@ module PatientHttp
       request(:delete, uri, callback: callback, **kwargs)
     end
 
+    # Enqueues an HTTP QUERY request.
+    #
+    # @param uri [String] absolute URL
+    # @param callback [Class, String] callback class to handle the response
+    # @param kwargs [Hash] forwarded to `request`
+    # @return [Object] return value from the registered request handler
+    def query(uri, callback:, **kwargs)
+      request(:query, uri, callback: callback, **kwargs)
+    end
+
     # Builds and dispatches an HTTP request.
     #
-    # @param method [Symbol] HTTP method (`:get`, `:post`, `:put`, `:patch`, `:delete`)
+    # @param method [Symbol] HTTP method (`:get`, `:head`, `:post`, `:put`, `:patch`, `:delete`, `:query`)
     # @param url [String] absolute URL
     # @param callback [Class, String] callback class to handle the response
     # @param headers [Hash, nil] request headers
@@ -321,8 +349,16 @@ module PatientHttp
     # @param raise_error_responses [Boolean, nil] when true, non-success responses are
     #   reported as errors
     # @param callback_args [Hash, nil] JSON-compatible callback arguments
+    # @param max_redirects [Integer, nil] maximum redirects to follow (nil uses the configuration
+    #   default, 0 disables redirects)
+    # @param follow_method_changing_redirects [Boolean, nil] whether to follow a redirect that changes the
+    #   HTTP method (nil uses the configuration default)
+    # @param redirect_strip_headers [String, Array<String>, nil] header names (case insensitive)
+    #   to strip from redirected requests, in addition to the configured names
     # @param preprocessors [String, Symbol, Array<String, Symbol>, nil] names of preprocessors
     #   registered on the configuration to apply to the request when it is sent
+    # @param processor [String, Symbol, nil] name of the processor that should execute
+    #   the request; handlers that support named processors route on this value
     # @return [Object] return value from the registered request handler
     def request(
       method,
@@ -335,7 +371,11 @@ module PatientHttp
       timeout: nil,
       raise_error_responses: nil,
       callback_args: nil,
-      preprocessors: nil
+      max_redirects: nil,
+      follow_method_changing_redirects: nil,
+      redirect_strip_headers: nil,
+      preprocessors: nil,
+      processor: nil
     )
       request = Request.new(
         method,
@@ -345,7 +385,11 @@ module PatientHttp
         headers: headers,
         params: params,
         timeout: timeout,
-        preprocessors: preprocessors
+        max_redirects: max_redirects,
+        follow_method_changing_redirects: follow_method_changing_redirects,
+        redirect_strip_headers: redirect_strip_headers,
+        preprocessors: preprocessors,
+        processor: processor
       )
       execute(
         request: request,

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: patient_http
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.6.0
 platform: ruby
 authors:
 - Brian Durand
@@ -89,6 +89,7 @@ files:
 - lib/patient_http/class_helper.rb
 - lib/patient_http/client.rb
 - lib/patient_http/client_pool.rb
+- lib/patient_http/completion_executor.rb
 - lib/patient_http/configuration.rb
 - lib/patient_http/encryptor.rb
 - lib/patient_http/error.rb
@@ -145,7 +146,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 4.0.3
 specification_version: 4
 summary: Generic async HTTP connection pool for Ruby applications using Fiber-based
   concurrency