patient_http 1.4.0 → 1.5.0

data/lib/patient_http/request_helper.rb

@@ -116,14 +116,16 @@ module PatientHttp
       # @param timeout [Float] default timeout in seconds
       # @param preprocessors [String, Symbol, Array<String, Symbol>, nil] default names of
       #   preprocessors registered on the configuration to apply to requests
+      # @param processor [String, Symbol, nil] default processor name for requests
       # @return [void]
-      def request_template(base_url: nil, headers: {}, params: nil, timeout: 30, preprocessors: nil)
+      def request_template(base_url: nil, headers: {}, params: nil, timeout: 30, preprocessors: nil, processor: nil)
         @patient_http_request_template = RequestTemplate.new(
           base_url: base_url,
           headers: headers,
           params: params,
           timeout: timeout,
-          preprocessors: preprocessors
+          preprocessors: preprocessors,
+          processor: processor
         )
       end
 
@@ -145,6 +147,8 @@ module PatientHttp
       # @param callback_args [Hash, nil] JSON-compatible callback arguments
       # @param preprocessors [String, Symbol, Array<String, Symbol>, nil] names of preprocessors
       #   registered on the configuration to apply to the request when it is sent
+      # @param processor [String, Symbol, nil] name of the processor that should execute
+      #   the request
       # @return [Object] return value from the registered request handler
       def async_request(
         method,
@@ -157,10 +161,11 @@ module PatientHttp
         timeout: nil,
         raise_error_responses: nil,
         callback_args: nil,
-        preprocessors: nil
+        preprocessors: nil,
+        processor: nil
       )
         template = async_request_template
-        kwargs = {body: body, json: json, headers: headers, params: params, timeout: timeout, preprocessors: preprocessors}
+        kwargs = {body: body, json: json, headers: headers, params: params, timeout: timeout, preprocessors: preprocessors, processor: processor}
         request = if template
           template.request(method, url, **kwargs)
         else
@@ -206,6 +211,8 @@ module PatientHttp
     # @param callback_args [Hash, nil] JSON-compatible callback arguments
     # @param preprocessors [String, Symbol, Array<String, Symbol>, nil] names of preprocessors
     #   registered on the configuration to apply to the request when it is sent
+    # @param processor [String, Symbol, nil] name of the processor that should execute
+    #   the request
     # @return [Object] return value from the registered request handler
     def async_request(
       method,
@@ -218,7 +225,8 @@ module PatientHttp
       timeout: nil,
       raise_error_responses: nil,
       callback_args: nil,
-      preprocessors: nil
+      preprocessors: nil,
+      processor: nil
     )
       self.class.async_request(
         method,
@@ -231,7 +239,8 @@ module PatientHttp
         timeout: timeout,
         raise_error_responses: raise_error_responses,
         callback_args: callback_args,
-        preprocessors: preprocessors
+        preprocessors: preprocessors,
+        processor: processor
       )
     end
 

data/lib/patient_http/request_preparer.rb

@@ -28,6 +28,13 @@ module PatientHttp
       headers = @config.secret_manager.resolve_headers(request.headers.to_h)
       headers["x-request-id"] = request_id
       headers["user-agent"] ||= @config.user_agent if @config.user_agent
+      # Compressed responses are inflated by ResponseReader during response
+      # decoding rather than by a client middleware wrapper. Requesting gzip is
+      # the default because it is what the reader can decode, but a caller that
+      # sets the header keeps its own value: "identity" opts out of compression,
+      # and any other encoding is delivered still encoded with its
+      # content-encoding header intact.
+      headers["accept-encoding"] ||= "gzip"
       url = @config.secret_manager.resolve_url(request.url, request.secret_params)
 
       outgoing = OutgoingRequest.new(

data/lib/patient_http/request_task.rb

@@ -236,7 +236,8 @@ module PatientHttp
         body: redirect_body,
         timeout: request.timeout,
         max_redirects: request.max_redirects,
-        preprocessors: redirect_preprocessors
+        preprocessors: redirect_preprocessors,
+        processor: request.processor
       )
 
       redirect_task_id = "#{id.split("/").first}/#{@redirects.size + 2}"

data/lib/patient_http/request_template.rb

@@ -34,12 +34,14 @@ module PatientHttp
     # @param timeout [Float] Default request timeout in seconds
     # @param preprocessors [String, Symbol, Array<String, Symbol>, nil] Default preprocessors
     #   to apply to all requests
-    def initialize(base_url: nil, headers: {}, params: nil, timeout: 30, preprocessors: nil)
+    # @param processor [String, Symbol, nil] Default processor name for all requests
+    def initialize(base_url: nil, headers: {}, params: nil, timeout: 30, preprocessors: nil, processor: nil)
       @base_url = base_url
       @headers = HttpHeaders.new(headers)
       @params = params
       @timeout = timeout
       @preprocessors = preprocessors
+      @processor = processor
     end
 
     # Build an async HTTP request. Returns a Request object.
@@ -52,8 +54,10 @@ module PatientHttp
     # @param params [Hash, nil] query parameters to add to URL
     # @param preprocessors [String, Symbol, Array<String, Symbol>, nil] preprocessors to apply
     #   to the request (overrides the template default)
+    # @param processor [String, Symbol, nil] processor name for the request (overrides the
+    #   template default)
     # @return [Request] request object
-    def request(method, uri, body: nil, json: nil, headers: nil, params: nil, timeout: nil, preprocessors: nil)
+    def request(method, uri, body: nil, json: nil, headers: nil, params: nil, timeout: nil, preprocessors: nil, processor: nil)
       full_uri = @base_url ? URI.join(@base_url, uri.to_s) : URI(uri)
 
       merged_headers = headers&.any? ? @headers.merge(headers) : @headers
@@ -68,7 +72,8 @@ module PatientHttp
         json: json,
         params: merged_params,
         timeout: timeout || @timeout,
-        preprocessors: preprocessors || @preprocessors
+        preprocessors: preprocessors || @preprocessors,
+        processor: processor || @processor
       )
     end
 

data/lib/patient_http/response_reader.rb

@@ -1,10 +1,13 @@
 # frozen_string_literal: true
 
 module PatientHttp
-  # Reads and validates HTTP response bodies.
+  # Reads and decodes HTTP response bodies.
   #
-  # Encapsulates the logic for reading async HTTP responses with size validation
-  # and building Response objects from the raw response data.
+  # Reading happens on the reactor thread and collects the raw (possibly
+  # compressed) body chunks with size validation. Decoding — joining the
+  # chunks, inflating compressed content, and applying the charset — is a
+  # separate step so it can run on a completion worker thread instead of
+  # blocking the event loop.
   class ResponseReader
     # Raised when a body read is aborted because the processor was stopped
     # past its shutdown deadline. The shutdown sequence re-enqueues the task,
@@ -13,41 +16,204 @@ module PatientHttp
     # @api private
     class ReadAbortedError < StandardError; end
 
+    # Content encodings that are inflated during decoding, mapped to the window
+    # bits of each wire format the encoding may arrive in. Formats are tried in
+    # order until one inflates the body.
+    #
+    # A "deflate" body should carry a zlib header (RFC 9110 specifies the zlib
+    # format), but some servers send a bare deflate stream instead, so the raw
+    # format is kept as a fallback.
+    INFLATE_WINDOW_BITS = {
+      "gzip" => [Zlib::MAX_WBITS | 16].freeze,
+      "deflate" => [Zlib::MAX_WBITS, -Zlib::MAX_WBITS].freeze
+    }.freeze
+
+    # Content encoding that means the body was not encoded at all. It needs no
+    # work to decode, but it still has to be recognized so it does not stop the
+    # decode of the encodings applied before it.
+    IDENTITY_ENCODING = "identity"
+
+    class << self
+      # Split the encodings named in the content-encoding header into the ones
+      # that stay applied to the body and the ones that can be decoded.
+      #
+      # A body can carry more than one encoding. They are listed in the order
+      # they were applied, so decoding runs from the last name backwards and
+      # stops at the first name it does not recognize. Everything before that
+      # point stays applied to the body.
+      #
+      # @param headers_hash [Hash] the response headers
+      # @return [Array(Array<String>, Array<String>)] the encodings that remain
+      #   applied and the encodings that can be decoded, both in applied order
+      def split_encodings(headers_hash)
+        encodings = content_encodings(headers_hash)
+        boundary = encodings.rindex { |name| !decodable?(name) }
+        return [[], encodings] if boundary.nil?
+
+        [encodings[0..boundary], encodings[(boundary + 1)..]]
+      end
+
+      # Parse the content-encoding header into encoding names.
+      #
+      # @param headers_hash [Hash] the response headers
+      # @return [Array<String>] the lowercased encoding names in applied order
+      def content_encodings(headers_hash)
+        headers_hash["content-encoding"].to_s.split(",").filter_map do |name|
+          name = name.strip.downcase
+          name unless name.empty?
+        end
+      end
+
+      # @param name [String] a lowercased content encoding name
+      # @return [Boolean] true if the reader can remove this encoding
+      def decodable?(name)
+        name == IDENTITY_ENCODING || INFLATE_WINDOW_BITS.key?(name)
+      end
+
+      # Restate the content-encoding header for a decoded body. The header is
+      # removed when nothing is left applied, and narrowed to the encodings the
+      # reader could not remove otherwise, so the header always describes the
+      # body delivered with it.
+      #
+      # @param headers_hash [Hash] the response headers
+      # @return [Hash] the headers with content-encoding updated or removed
+      def rewrite_content_encoding(headers_hash)
+        return headers_hash unless headers_hash.key?("content-encoding")
+
+        remaining, _decodable = split_encodings(headers_hash)
+
+        if remaining.empty?
+          headers_hash.except("content-encoding")
+        else
+          headers_hash.merge("content-encoding" => remaining.join(", "))
+        end
+      end
+    end
+
     # Initialize the reader.
     #
-    # @param processor [Processor] the processor object
-    def initialize(processor)
+    # Reading needs a processor so it can abort once the processor is past its
+    # shutdown deadline. Decoding needs only the configuration, so a caller
+    # that does its own reading can supply the configuration on its own.
+    #
+    # @param processor [Processor, nil] the processor object
+    # @param config [Configuration, nil] the configuration; defaults to the
+    #   processor's configuration
+    def initialize(processor, config: nil)
       @processor = processor
+      @config = config || processor.config
     end
 
-    # Read the response body with size validation.
+    # Read the raw response body chunks with size validation.
     #
     # Reads the async HTTP response body asynchronously to completion, which allows
     # the connection to be reused. The async-http client handles connection pooling
     # and keep-alive internally. Using iteration instead of read() ensures non-blocking
-    # I/O that yields to the reactor.
+    # I/O that yields to the reactor. The chunks are the wire bytes: when the
+    # response is compressed, the size check here applies to the compressed
+    # bytes and {#decode_body} applies the same limit to the inflated bytes.
     #
     # @param async_response [Async::HTTP::Protocol::Response] the async HTTP response
     # @param headers_hash [Hash] the response headers
-    # @return [String, nil] the response body or nil if no body present
-    # @raise [ResponseTooLargeError] if body exceeds max_response_size
+    # @return [Array<String>, nil] the raw body chunks or nil if no body present
+    # @raise [ResponseTooLargeError] if the body exceeds max_response_size
     # @raise [ReadAbortedError] if the processor stopped past its shutdown deadline mid-read
-    def read_body(async_response, headers_hash)
+    def read_raw_body(async_response, headers_hash)
       return nil unless async_response.body
 
       validate_content_length(headers_hash)
-      body = read_body_chunks(async_response)
+      read_body_chunks(async_response)
+    end
+
+    # Decode raw body chunks into the final body string.
+    #
+    # Joins the chunks, inflates gzip/deflate content (enforcing
+    # max_response_size on the inflated bytes), and applies the charset from
+    # the Content-Type header. This is CPU-bound work intended to run on a
+    # completion worker thread.
+    #
+    # An encoding the reader does not support leaves the body encoded, and a
+    # body that is still encoded keeps its binary encoding because the charset
+    # does not describe it. Use {.split_encodings} to find what stays applied
+    # so the content-encoding header delivered with the response describes the
+    # body it carries.
+    #
+    # @param chunks [Array<String>, nil] the raw body chunks
+    # @param headers_hash [Hash] the response headers
+    # @return [String, nil] the decoded body or nil if there was no body
+    # @raise [ResponseTooLargeError] if the inflated body exceeds max_response_size
+    def decode_body(chunks, headers_hash)
+      return nil if chunks.nil?
+
+      remaining, decodable = self.class.split_encodings(headers_hash)
+      warn_undecodable(remaining) unless remaining.empty?
+
+      body = inflate_encodings(chunks, decodable).join
+      body.force_encoding(Encoding::ASCII_8BIT)
+      # A body that is still encoded is not text yet, so the charset does not
+      # describe its bytes. Leave it binary for the caller to decode.
+      return body unless remaining.empty?
+
       apply_charset_encoding(body, headers_hash)
     end
 
     private
 
+    # Remove the given encodings from the body, starting with the one applied
+    # last. Identity needs no work; every other name here inflates.
+    #
+    # @param chunks [Array<String>] the encoded body chunks
+    # @param encodings [Array<String>] decodable encoding names in applied order
+    # @return [Array<String>] the decoded chunks
+    # @raise [ResponseTooLargeError] if the inflated body exceeds max_response_size
+    def inflate_encodings(chunks, encodings)
+      encodings.reverse_each do |name|
+        chunks = [inflate_encoding(chunks, name)] if INFLATE_WINDOW_BITS.key?(name)
+      end
+
+      chunks
+    end
+
+    # Inflate one encoding, trying each wire format the encoding can use. The
+    # chunks are all in memory, so a format that turns out to be wrong can be
+    # abandoned and the next one started from the beginning of the body.
+    #
+    # @param chunks [Array<String>] the encoded body chunks
+    # @param name [String] a lowercased content encoding name
+    # @return [String] the inflated body
+    # @raise [Zlib::Error] if no format could inflate the body
+    # @raise [ResponseTooLargeError] if the inflated body exceeds max_response_size
+    def inflate_encoding(chunks, name)
+      formats = INFLATE_WINDOW_BITS.fetch(name)
+      last_index = formats.size - 1
+
+      formats.each_with_index do |window_bits, index|
+        return inflate_chunks(chunks, window_bits)
+      rescue Zlib::DataError, Zlib::BufError
+        raise if index == last_index
+      end
+    end
+
+    # Report an encoding that could not be removed. The body is still delivered
+    # with its content-encoding header, so the caller can decode it, but the
+    # server ignored the accept-encoding header and that is worth recording.
+    #
+    # @param remaining [Array<String>] the encodings left on the body
+    # @return [void]
+    def warn_undecodable(remaining)
+      logger&.warn(
+        "[PatientHttp] Cannot decode response body with content-encoding " \
+        "'#{remaining.join(", ")}'; returning the encoded body"
+      )
+      nil
+    end
+
     def max_response_size
-      @processor.config.max_response_size
+      @config.max_response_size
     end
 
     def logger
-      @processor.config.logger
+      @config.logger
     end
 
     # Validate content-length header doesn't exceed max size.
@@ -66,7 +232,7 @@ module PatientHttp
     # Read body chunks while checking size.
     #
     # @param async_response [Async::HTTP::Protocol::Response] the async HTTP response
-    # @return [String] the response body in ASCII-8BIT encoding
+    # @return [Array<String>] the raw body chunks
     # @raise [ResponseTooLargeError] if body size exceeds max_response_size during read
     # @raise [ReadAbortedError] if the processor stopped past its shutdown deadline mid-read
     def read_body_chunks(async_response)
@@ -80,7 +246,7 @@ module PatientHttp
           # Reads are allowed to finish while the processor is merely stopping
           # (the graceful shutdown window) so in-flight responses can still be
           # delivered.
-          if @processor.stopped?
+          if @processor&.stopped?
             raise ReadAbortedError.new("Processor stopped while reading response body")
           end
 
@@ -97,8 +263,7 @@ module PatientHttp
 
         finished = true
 
-        # Join chunks and force to binary encoding to preserve raw bytes
-        chunks.join.force_encoding(Encoding::ASCII_8BIT)
+        chunks
       ensure
         # Always close the body if we were interrupted or if an error occurred
         # This ensures the connection is properly released back to the pool
@@ -106,6 +271,49 @@ module PatientHttp
       end
     end
 
+    # Inflate compressed body chunks with streaming size enforcement, so a
+    # small compressed body cannot expand past max_response_size.
+    #
+    # @param chunks [Array<String>] the raw compressed chunks
+    # @param window_bits [Integer] Zlib window bits for the content encoding
+    # @return [String] the inflated body
+    # @raise [ResponseTooLargeError] if the inflated size exceeds max_response_size
+    def inflate_chunks(chunks, window_bits)
+      # A response can declare a content encoding and still carry no body.
+      # There is nothing to inflate, and finishing an empty stream would
+      # raise a buffer error.
+      return +"" if chunks.all?(&:empty?)
+
+      inflater = Zlib::Inflate.new(window_bits)
+      body = +""
+
+      begin
+        # The block form yields the inflated output in buffer-sized pieces, so
+        # the size is checked before the whole expansion is materialized. A
+        # single small compressed chunk can otherwise inflate to gigabytes
+        # before any check runs.
+        appender = ->(output) do
+          body << output
+          validate_inflated_size(body)
+        end
+
+        chunks.each { |chunk| inflater.inflate(chunk, &appender) }
+        inflater.finish(&appender) unless inflater.finished?
+      ensure
+        inflater.close
+      end
+
+      body
+    end
+
+    def validate_inflated_size(body)
+      if body.bytesize > max_response_size
+        raise ResponseTooLargeError.new(
+          "Response body size exceeded maximum allowed size (#{max_response_size} bytes)"
+        )
+      end
+    end
+
     # Extract charset from Content-Type header.
     #
     # @param headers_hash [Hash] the response headers

data/lib/patient_http/synchronous_executor.rb

@@ -20,6 +20,7 @@ module PatientHttp
       @on_error = on_error
       @proxy_client = nil
       @request_preparer = RequestPreparer.new(config)
+      @response_reader = ResponseReader.new(nil, config: config)
     end
 
     # Execute the request synchronously.
@@ -62,11 +63,12 @@ module PatientHttp
               # flattened to a single joined string value.
               headers_hash = async_response.headers.to_h.transform_values(&:to_s)
 
-              body_content = read_response_body(async_response, headers_hash)
+              chunks = read_response_body(async_response, headers_hash)
+              body_content = @response_reader.decode_body(chunks, headers_hash)
 
               {
                 status: async_response.status,
-                headers: headers_hash,
+                headers: ResponseReader.rewrite_content_encoding(headers_hash),
                 body: body_content
               }
             end
@@ -133,19 +135,22 @@ module PatientHttp
 
     # Create HTTP client with config settings (retries, proxy, connection timeout).
     #
+    # The client is not wrapped in a Protocol::HTTP::AcceptEncoding middleware.
+    # That wrapper overwrites the request's accept-encoding header, which would
+    # ignore a caller opting out of compression, so response bodies are decoded
+    # by ResponseReader here exactly as they are on the async path.
+    #
     # @param url [String] the resolved request URL
-    # @return [Protocol::HTTP::AcceptEncoding] wrapped HTTP client
+    # @return [Async::HTTP::Client] the HTTP client
     def create_http_client(url)
       endpoint = Async::HTTP::Endpoint.parse(url)
       endpoint = configure_endpoint(endpoint) if @config.connection_timeout
 
-      client = if @config.proxy_url
+      if @config.proxy_url
         create_proxied_client(endpoint)
       else
         Async::HTTP::Client.new(endpoint, retries: @config.retries)
       end
-
-      Protocol::HTTP::AcceptEncoding.new(client)
     end
 
     # Create a proxied HTTP client.
@@ -174,11 +179,13 @@ module PatientHttp
       )
     end
 
-    # Read the response body with size validation.
+    # Read the raw response body chunks with size validation. The chunks are
+    # the wire bytes; ResponseReader#decode_body inflates and applies the
+    # charset, enforcing the same limit on the inflated bytes.
     #
     # @param async_response [Async::HTTP::Protocol::Response] the async HTTP response
     # @param headers_hash [Hash] the response headers
-    # @return [String, nil] the response body
+    # @return [Array<String>, nil] the raw body chunks or nil if no body present
     def read_response_body(async_response, headers_hash)
       return nil unless async_response.body
 
@@ -210,31 +217,7 @@ module PatientHttp
         async_response.body.close unless finished
       end
 
-      body = chunks.join.force_encoding(Encoding::ASCII_8BIT)
-
-      charset = extract_charset(headers_hash)
-      if charset
-        begin
-          encoding = Encoding.find(charset)
-          body.force_encoding(encoding)
-        rescue ArgumentError
-          # Invalid charset, keep binary
-        end
-      end
-
-      body
-    end
-
-    # Extract charset from Content-Type header.
-    def extract_charset(headers_hash)
-      content_type = headers_hash["content-type"]
-      return nil unless content_type
-
-      match = content_type.match(/;\s*charset\s*=\s*([^;\s]+)/i)
-      return nil unless match
-
-      charset = match[1].strip
-      charset.gsub(/\A["']|["']\z/, "")
+      chunks
     end
 
     # Invoke callback synchronously.

data/lib/patient_http.rb

@@ -30,6 +30,12 @@ module PatientHttp
 
   class ResponseTooLargeError < StandardError; end
 
+  # Raised when a request names a processor that is not configured. Handlers
+  # that support named processors raise this at enqueue time; the executing
+  # side raises it for a job that names an unconfigured processor so the job
+  # lands in the job system's retry mechanism instead of being dropped.
+  class UnknownProcessorError < StandardError; end
+
   # HTTP redirect status codes that should be followed
   FOLLOWABLE_REDIRECT_STATUSES = [301, 302, 303, 307, 308].freeze
 
@@ -45,6 +51,7 @@ module PatientHttp
   autoload :Client, File.join(__dir__, "patient_http/client")
   autoload :ClientError, File.join(__dir__, "patient_http/http_error")
   autoload :ClientPool, File.join(__dir__, "patient_http/client_pool")
+  autoload :CompletionExecutor, File.join(__dir__, "patient_http/completion_executor")
   autoload :Configuration, File.join(__dir__, "patient_http/configuration")
   autoload :Encryptor, File.join(__dir__, "patient_http/encryptor")
   autoload :Error, File.join(__dir__, "patient_http/error")
@@ -323,6 +330,8 @@ module PatientHttp
     # @param callback_args [Hash, nil] JSON-compatible callback arguments
     # @param preprocessors [String, Symbol, Array<String, Symbol>, nil] names of preprocessors
     #   registered on the configuration to apply to the request when it is sent
+    # @param processor [String, Symbol, nil] name of the processor that should execute
+    #   the request; handlers that support named processors route on this value
     # @return [Object] return value from the registered request handler
     def request(
       method,
@@ -335,7 +344,8 @@ module PatientHttp
       timeout: nil,
       raise_error_responses: nil,
       callback_args: nil,
-      preprocessors: nil
+      preprocessors: nil,
+      processor: nil
     )
       request = Request.new(
         method,
@@ -345,7 +355,8 @@ module PatientHttp
         headers: headers,
         params: params,
         timeout: timeout,
-        preprocessors: preprocessors
+        preprocessors: preprocessors,
+        processor: processor
       )
       execute(
         request: request,

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: patient_http
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.5.0
 platform: ruby
 authors:
 - Brian Durand
@@ -89,6 +89,7 @@ files:
 - lib/patient_http/class_helper.rb
 - lib/patient_http/client.rb
 - lib/patient_http/client_pool.rb
+- lib/patient_http/completion_executor.rb
 - lib/patient_http/configuration.rb
 - lib/patient_http/encryptor.rb
 - lib/patient_http/error.rb