html2rss 0.25.0 → 0.27.0

data/lib/html2rss/request_service/botasaurus_contract.rb

@@ -5,35 +5,40 @@ require 'json'
 module Html2rss
   class RequestService
     ##
-    # Maps html2rss request/response handling to the botasaurus-scrape-api contract.
-    class BotasaurusContract
-      # Default Botasaurus scrape options when no explicit config is provided.
-      DEFAULT_OPTIONS = {
-        navigation_mode: 'auto',
-        max_retries: 1,
-        headless: false
-      }.freeze
-
-      # Allowlisted request.botasaurus keys forwarded to upstream.
-      OPTION_KEYS = %i[
-        navigation_mode
-        max_retries
-        wait_for_selector
-        wait_timeout_seconds
-        block_images
-        block_images_and_css
-        wait_for_complete_page_load
-        headless
-        proxy
-        user_agent
-        window_size
-        lang
+    # Maps html2rss request/response handling to botasaurus-scrape-api OpenAPI 2.0
+    # +ScrapeRequest+ / +ScrapeSuccess+ / +ScrapeError+ (sibling +openapi.yaml+).
+    class BotasaurusContract # rubocop:disable Metrics/ClassLength -- Success/Error envelopes stay with the owner
+      # Closed set from OpenAPI ExecutionMode.
+      EXECUTION_MODES = %w[auto request browser].freeze
+      # Closed set from OpenAPI NavigationMode.
+      NAVIGATION_MODES = %w[auto get google_get google_get_bypass organic_get].freeze
+      # Closed set from OpenAPI ErrorCategory.
+      ERROR_CATEGORIES = %w[timeout challenge_block navigation_error metadata_error validation].freeze
+
+      # ScrapeRequest properties except +url+ (html2rss supplies the target URL).
+      REQUEST_OPTION_KEYS = %i[
+        execution_mode navigation_mode max_retries wait_for_selector wait_timeout_seconds
+        scroll block_images block_images_and_css block_trackers
+        wait_for_complete_page_load user_agent headers cookies window_size lang headless proxy
       ].freeze
 
-      # Allowlisted upstream response keys exposed as Response#transport_meta.
-      META_KEYS = %w[
-        request_id strategy_used render_ms challenge_detected blocked_detected attempts error_category
-      ].freeze
+      # OpenAPI WindowSize required keys (positive integers).
+      WINDOW_SIZE_PROPERTIES = %i[width height].freeze
+
+      # Published wait clamp floor (`[1, 20]` in OpenAPI description).
+      MIN_WAIT_TIMEOUT_SECONDS = 1
+      # Published wait clamp ceiling; YAML values above this are rejected.
+      MAX_WAIT_TIMEOUT_SECONDS = 20
+      # OpenAPI wait_timeout_seconds default (omitted from the client payload).
+      DEFAULT_WAIT_TIMEOUT_SECONDS = 15
+
+      # OpenAPI max_retries maximum (default 2 is applied upstream when omitted).
+      MAX_RETRIES = 3
+
+      # Allowlisted ScrapeDiagnostics keys nested under diagnostics.
+      DIAGNOSTICS_KEYS = %w[request_id attempts strategy_used render_ms execution_tier challenge].freeze
+      # Allowlisted ChallengeSignal keys nested under diagnostics.challenge.
+      CHALLENGE_KEYS = %w[blocked detected marker].freeze
 
       # Remaining seconds at or below which Botasaurus retries are disabled.
       TIGHT_BUDGET_SECONDS = 12
@@ -41,145 +46,311 @@ module Html2rss
       # Seconds reserved from remaining budget before setting wait_timeout_seconds.
       BUDGET_WAIT_RESERVE_SECONDS = 2
 
-      # Parsed Botasaurus response wrapper.
-      class ParsedResponse
-        # Fallback headers when upstream omits response headers.
-        DEFAULT_HEADERS = { 'content-type' => 'text/html' }.freeze
+      ##
+      # Parsed OpenAPI ScrapeSuccess envelope (HTTP 200).
+      class Success
+        # Per-body size cap for captured XHR JSON (defense-in-depth vs upstream).
+        MAX_XHR_BODY_BYTES = 500_000
+        # Aggregate size cap across all captured XHR bodies.
+        MAX_XHR_AGGREGATE_BYTES = 2_000_000
+
+        # @param payload [Hash{String => Object}] parsed ScrapeSuccess object
+        def initialize(payload:)
+          @payload = payload
+          html
+          diagnostics
+        end
+
+        # @return [String] rendered HTML body
+        def html
+          value = payload['html']
+          raise BotasaurusServiceError, 'Botasaurus scrape success requires html' unless value.is_a?(String)
+
+          value
+        end
+
+        # @return [Hash{String => String}] normalized response headers (null → {})
+        def headers
+          raw_headers = payload['headers']
+          return {} unless raw_headers.is_a?(Hash) && raw_headers.any?
+
+          raw_headers.to_h { |key, value| [key.to_s, value.to_s] }
+        end
+
+        # @return [Integer, nil] document status_code when present
+        def status
+          status_code = payload['status_code']
+          status_code.is_a?(Integer) ? status_code : nil
+        end
+
+        # @return [String, nil] final URL reported by upstream
+        def final_url = payload['final_url']
+
+        # @return [Hash{String => Object}] diagnostics plus success-only metadata_error (frozen)
+        def transport_meta
+          meta = diagnostics.dup
+          meta['metadata_error'] = metadata_error if metadata_error
+          meta.freeze
+        end
+
+        # @return [Array<Hash{String => Object}>] size-capped XHR/fetch captures (absent → [])
+        def xhr_responses
+          raw = payload['xhr_responses']
+          return [] unless raw.is_a?(Array)
+
+          aggregate_bytes = 0
+          raw.filter_map do |entry|
+            normalized = normalize_xhr_entry(entry)
+            next unless normalized
+            next unless (aggregate_bytes = advance_xhr_budget(normalized, aggregate_bytes))
+
+            normalized
+          end
+        end
+
+        private
+
+        attr_reader :payload
+
+        def diagnostics = BotasaurusContract.diagnostics_from(payload)
+
+        def metadata_error
+          value = payload['metadata_error']
+          value.is_a?(String) && !value.empty? ? value : nil
+        end
+
+        def normalize_xhr_entry(entry)
+          return unless entry.is_a?(Hash)
+
+          body = xhr_field(entry, :body)
+          return unless body.is_a?(String) && !body.empty?
+
+          {
+            'url' => xhr_field(entry, :url).to_s,
+            'body' => body,
+            'headers' => xhr_headers(xhr_field(entry, :headers)),
+            'status_code' => xhr_status_code(xhr_field(entry, :status_code))
+          }
+        end
+
+        # @return [Integer, nil] next aggregate byte total when accepted
+        def advance_xhr_budget(normalized, aggregate_bytes)
+          body_bytes = normalized.fetch('body').bytesize
+          return if body_bytes > MAX_XHR_BODY_BYTES
+          return if aggregate_bytes + body_bytes > MAX_XHR_AGGREGATE_BYTES
+
+          aggregate_bytes + body_bytes
+        end
+
+        def xhr_field(entry, key)
+          entry[key.to_s] || entry[key]
+        end
+
+        def xhr_headers(raw)
+          return {} unless raw.is_a?(Hash)
 
-        # @param payload [Hash{String => Object}] parsed Botasaurus response payload
+          raw.to_h { |key, value| [key.to_s, value.to_s] }
+        end
+
+        def xhr_status_code(value)
+          value.is_a?(Integer) ? value : nil
+        end
+      end
+
+      ##
+      # Parsed OpenAPI ScrapeError envelope (HTTP 400/403/422/502/504).
+      class Error
+        # @param payload [Hash{String => Object}] parsed ScrapeError object
         # @param transport_status [Integer] HTTP status returned by Botasaurus
         def initialize(payload:, transport_status:)
           @payload = payload
           @transport_status = transport_status
+          error
+          error_category
+          diagnostics
         end
 
         # @return [Boolean] true when upstream classified request as challenge blocked
         def challenge_block? = error_category == 'challenge_block'
 
-        # @return [Boolean] true when upstream returned non-200 or an error payload
-        def upstream_failure?
-          status != 200 || error_message?
+        # @return [Boolean] true when the scrape envelope reports a timeout
+        def timeout?
+          transport_status == 504 || error_category == 'timeout'
         end
 
         # @return [String] normalized challenge error message
         def challenge_message
-          error || 'Botasaurus challenge block detected.'
+          [error, challenge_marker].find { |value| value.is_a?(String) && !value.empty? } ||
+            'Botasaurus challenge block detected.'
         end
 
         # @return [String] actionable upstream failure summary
-        def upstream_failure_message
-          details = ["status=#{status}"]
-          details << "error_category=#{error_category}" if error_category
-          details << "error=#{error}" if error
+        def failure_message
+          details = ["status=#{transport_status}", "error_category=#{error_category}", "error=#{error}"]
           details << "request_id=#{request_id}" if request_id
           "Botasaurus scrape failed (#{details.join(', ')})."
         end
 
-        # @return [String] rendered HTML body from Botasaurus
-        # @raise [BotasaurusConnectionFailed] when html is missing
-        def html
-          value = payload['html']
-          raise BotasaurusConnectionFailed, "Botasaurus response missing required 'html' field" if value.nil?
-
-          value.to_s
-        end
+        private
 
-        # @return [Hash{String => String}] normalized response headers
-        def headers
-          raw_headers = payload['headers']
-          return DEFAULT_HEADERS.dup unless raw_headers.is_a?(Hash) && raw_headers.any?
+        attr_reader :payload, :transport_status
 
-          raw_headers.to_h { |key, value| [key.to_s, value.to_s] }
-        end
+        def error
+          value = payload['error']
+          raise BotasaurusServiceError, 'Botasaurus scrape error requires error' unless value.is_a?(String)
 
-        # @return [Integer] resolved status code (payload status_code or transport status)
-        def status
-          status_code = payload['status_code']
-          status_code.is_a?(Integer) ? status_code : transport_status
+          value
         end
 
-        # @return [String, nil] final URL reported by upstream
-        def final_url = payload['final_url']
+        def error_category
+          value = payload['error_category']
+          return value if ERROR_CATEGORIES.include?(value)
 
-        # @return [Hash{String => Object}] allowlisted upstream telemetry (frozen)
-        def transport_meta = payload.slice(*META_KEYS).compact.freeze
+          raise BotasaurusServiceError, 'Botasaurus scrape error requires error_category'
+        end
 
-        private
+        def diagnostics = BotasaurusContract.diagnostics_from(payload)
 
-        attr_reader :payload, :transport_status
+        def request_id = diagnostics['request_id']
 
-        def error = payload['error']
+        def challenge_marker
+          challenge = diagnostics['challenge']
+          challenge.is_a?(Hash) ? challenge['marker'] : nil
+        end
+      end
 
-        def request_id = payload['request_id']
+      ##
+      # @param payload [Hash{String => Object}] scrape envelope
+      # @return [Hash{String => Object}] allowlisted diagnostics (frozen)
+      # @raise [BotasaurusServiceError] when diagnostics or request_id are missing
+      def self.diagnostics_from(payload)
+        raw = payload['diagnostics']
+        raise BotasaurusServiceError, 'Botasaurus scrape envelope requires diagnostics' unless raw.is_a?(Hash)
+        raise BotasaurusServiceError, 'Botasaurus diagnostics require request_id' if raw['request_id'].to_s.empty?
+
+        sliced = raw.slice(*DIAGNOSTICS_KEYS)
+        sliced['challenge'] = compact_challenge(sliced['challenge'])
+        sliced.compact.freeze
+      end
 
-        def error_category = payload['error_category']
+      # @param challenge [Object] diagnostics.challenge value
+      # @return [Hash{String => Object}, nil]
+      def self.compact_challenge(challenge)
+        return unless challenge.is_a?(Hash)
 
-        def error_message?
-          value = error
-          value.is_a?(String) ? !value.empty? : !value.nil?
-        end
+        challenge.slice(*CHALLENGE_KEYS).compact
       end
+      private_class_method :compact_challenge
 
       ##
       # @param url [Html2rss::Url] canonical URL to scrape
+      # @param headers [Hash] request headers from context
       # @param options [Hash] validated request.botasaurus options
       # @param remaining_timeout_seconds [Numeric, nil] shared request budget remainder for clamps
+      # @option options [String] :execution_mode
       # @option options [String] :navigation_mode
       # @option options [Integer] :max_retries
       # @option options [String] :wait_for_selector
       # @option options [Integer] :wait_timeout_seconds
+      # @option options [Boolean] :scroll
       # @option options [Boolean] :block_images
       # @option options [Boolean] :block_images_and_css
+      # @option options [Boolean] :block_trackers
       # @option options [Boolean] :wait_for_complete_page_load
       # @option options [Boolean] :headless
       # @option options [String] :proxy
       # @option options [String] :user_agent
-      # @option options [Array<Integer>] :window_size
+      # @option options [Hash] :window_size
       # @option options [String] :lang
-      def initialize(url:, options: {}, remaining_timeout_seconds: nil)
+      # @option options [Hash] :cookies
+      # @option options [Hash] :headers
+      def initialize(url:, headers: {}, options: {}, remaining_timeout_seconds: nil)
         @url = url
+        @headers = headers
         @options = options
         @remaining_timeout_seconds = remaining_timeout_seconds
       end
 
-      # @return [Hash] payload for POST /scrape (budget-clamped when remaining is known)
+      # @return [Hash] payload for POST /scrape (explicit options plus budget clamp-downs)
       def request_payload
-        DEFAULT_OPTIONS.merge(filtered_options).merge(url: url.to_s).then { clamp_for_budget(_1) }
+        payload = { url: url.to_s }.merge(filtered_options)
+        forwarded_headers = merged_headers
+        payload[:headers] = forwarded_headers if forwarded_headers&.any?
+        clamp_for_budget(payload)
       end
 
       # @param transport_response [Faraday::Response] upstream HTTP response
-      # @return [ParsedResponse]
-      # @raise [BotasaurusConnectionFailed] when payload is not valid JSON object
+      # @return [Success, Error]
+      # @raise [BotasaurusServiceError] when payload is not a scrape envelope
       def parse_response(transport_response)
-        payload = JSON.parse(transport_response.body.to_s)
-        raise BotasaurusConnectionFailed, 'Botasaurus response must be a JSON object' unless payload.is_a?(Hash)
+        payload = json_object(transport_response.body.to_s)
+        return Success.new(payload:) if transport_response.status == 200
 
-        ParsedResponse.new(payload:, transport_status: transport_response.status)
-      rescue JSON::ParserError => error
-        raise BotasaurusConnectionFailed, "Botasaurus response JSON parse failed: #{error.message}"
+        Error.new(payload:, transport_status: transport_response.status)
       end
 
       private
 
-      attr_reader :url, :options, :remaining_timeout_seconds
+      attr_reader :url, :headers, :options, :remaining_timeout_seconds
+
+      def json_object(body)
+        payload = JSON.parse(body)
+        return payload if payload.is_a?(Hash)
+
+        raise BotasaurusServiceError, 'Botasaurus response must be a JSON object'
+      rescue JSON::ParserError => error
+        raise BotasaurusServiceError, "Botasaurus response JSON parse failed: #{error.message}"
+      end
+
+      def merged_headers
+        explicit = options[:headers] || {}
+        base = headers.is_a?(Hash) ? headers : {}
+        merged = base.merge(explicit).transform_keys(&:to_s).compact
+        merged.empty? ? nil : merged
+      end
 
       def filtered_options
-        OPTION_KEYS.each_with_object({}) do |key, normalized|
+        REQUEST_OPTION_KEYS.each_with_object({}) do |key, normalized|
           normalized[key] = options[key] if options.key?(key)
         end
       end
 
       def clamp_for_budget(payload)
         remaining = remaining_timeout_seconds
-        return payload if remaining.nil?
-
         clamped = payload.dup
-        clamped[:max_retries] = 0 if remaining <= TIGHT_BUDGET_SECONDS
-        budget_wait = [1, (remaining - BUDGET_WAIT_RESERVE_SECONDS).floor].max
-        configured = clamped[:wait_timeout_seconds]
-        clamped[:wait_timeout_seconds] = configured ? [configured, budget_wait].min : budget_wait
+        unless remaining.nil?
+          apply_tight_retries!(clamped, remaining)
+          apply_budget_wait!(clamped, remaining)
+        end
+        cap_wait_timeout!(clamped)
         clamped
       end
+
+      def apply_tight_retries!(payload, remaining)
+        payload[:max_retries] = 0 if remaining <= TIGHT_BUDGET_SECONDS
+      end
+
+      def apply_budget_wait!(payload, remaining)
+        budget_wait = budget_wait_seconds(remaining)
+        configured = payload[:wait_timeout_seconds]
+        if configured
+          payload[:wait_timeout_seconds] = [configured, budget_wait].min
+        elsif budget_wait < DEFAULT_WAIT_TIMEOUT_SECONDS
+          payload[:wait_timeout_seconds] = budget_wait
+        end
+      end
+
+      def budget_wait_seconds(remaining)
+        reserved = [MIN_WAIT_TIMEOUT_SECONDS, (remaining - BUDGET_WAIT_RESERVE_SECONDS).floor].max
+        [reserved, MAX_WAIT_TIMEOUT_SECONDS].min
+      end
+
+      def cap_wait_timeout!(payload)
+        wait = payload[:wait_timeout_seconds]
+        return unless wait
+
+        payload[:wait_timeout_seconds] = [wait, MAX_WAIT_TIMEOUT_SECONDS].min
+      end
     end
   end
 end

data/lib/html2rss/request_service/botasaurus_strategy.rb

@@ -12,8 +12,8 @@ module Html2rss
 
       def fetch
         parsed_response = post_scrape_request
-        raise_if_challenge_blocked!(parsed_response)
-        raise_if_upstream_failed!(parsed_response)
+        raise_from_error!(parsed_response) if parsed_response.is_a?(BotasaurusContract::Error)
+
         build_response(parsed_response)
       end
 
@@ -28,20 +28,28 @@ module Html2rss
           headers: parsed_response.headers,
           url: response_url(parsed_response.final_url),
           status: parsed_response.status,
-          transport_meta: parsed_response.transport_meta
+          transport_meta: parsed_response.transport_meta,
+          captured_responses: parsed_response.xhr_responses
         )
       end
 
-      def raise_if_challenge_blocked!(parsed_response)
-        return unless parsed_response.challenge_block?
+      def raise_from_error!(error)
+        raise_if_challenge_blocked!(error)
+        raise_if_timed_out!(error)
+        raise BotasaurusServiceError, error.failure_message
+      end
+
+      def raise_if_challenge_blocked!(error)
+        return unless error.challenge_block?
 
-        raise BlockedSurfaceDetected, "Blocked surface detected: #{parsed_response.challenge_message}"
+        raise BlockedSurfaceDetected, "Blocked surface detected: #{error.challenge_message}"
       end
 
-      def raise_if_upstream_failed!(parsed_response)
-        return unless parsed_response.upstream_failure?
+      def raise_if_timed_out!(error)
+        return unless error.timeout?
 
-        raise BotasaurusConnectionFailed, parsed_response.upstream_failure_message
+        log_timeout!(reason: 'botasaurus_upstream')
+        raise RequestTimedOut, error.failure_message
       end
 
       def response_url(final_url)
@@ -55,6 +63,7 @@ module Html2rss
       def contract
         @contract ||= BotasaurusContract.new(
           url: ctx.url,
+          headers: ctx.headers,
           options: ctx.request.fetch(:botasaurus, {}),
           remaining_timeout_seconds: attempt_timeout_seconds
         )

data/lib/html2rss/request_service/budget.rb

@@ -3,20 +3,17 @@
 module Html2rss
   class RequestService
     ##
-    # Tracks request slots, interaction budget, and wall-clock deadline for one feed build.
+    # Tracks HTTP request slots and wall-clock deadline for one feed build.
     #
-    # HTTP fetches consume request slots; Browserless preload clicks/scrolls/waits consume
-    # interaction budget. Deadline remains on `#remaining_timeout_seconds`.
+    # HTTP fetches consume request slots. Deadline remains on `#remaining_timeout_seconds`.
     class Budget
       ##
       # @param max_requests [Integer] maximum HTTP request slots
-      # @param max_interactions [Integer] maximum preload interaction slots (default 0)
       # @param total_timeout_seconds [Integer, nil] wall-clock timeout for the feed build
-      def initialize(max_requests:, max_interactions: 0, total_timeout_seconds: nil)
-        validate_slot_limits!(max_requests:, max_interactions:)
+      def initialize(max_requests:, total_timeout_seconds: nil)
+        validate_slot_limits!(max_requests:)
 
         @remaining_requests = max_requests
-        @remaining_interactions = max_interactions
         @start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
         @total_timeout_seconds = total_timeout_seconds
         @mutex = Mutex.new
@@ -44,21 +41,6 @@ module Html2rss
         end
       end
 
-      ##
-      # Consumes one Browserless preload interaction slot.
-      #
-      # Preload must not steal pagination HTTP request slots.
-      #
-      # @return [Integer] remaining interaction slots after consumption
-      # @raise [InteractionBudgetExceeded] if no interaction slots remain
-      def consume_interaction!
-        @mutex.synchronize do
-          raise InteractionBudgetExceeded, 'Interaction budget exhausted' if @remaining_interactions.zero?
-
-          @remaining_interactions -= 1
-        end
-      end
-
       ##
       # @return [Integer] HTTP request slots still available
       def remaining
@@ -71,12 +53,6 @@ module Html2rss
         @mutex.synchronize { @remaining_requests }
       end
 
-      ##
-      # @return [Integer] preload interaction slots still available
-      def remaining_interactions
-        @mutex.synchronize { @remaining_interactions }
-      end
-
       ##
       # @return [Float, nil] the remaining timeout in seconds, or nil if not tracked
       def remaining_timeout_seconds
@@ -118,14 +94,10 @@ module Html2rss
 
       private
 
-      def validate_slot_limits!(max_requests:, max_interactions:)
-        unless max_requests.is_a?(Integer) && max_requests.positive?
-          raise ArgumentError,
-                'max_requests must be positive'
-        end
-        return if max_interactions.is_a?(Integer) && !max_interactions.negative?
+      def validate_slot_limits!(max_requests:)
+        return if max_requests.is_a?(Integer) && max_requests.positive?
 
-        raise ArgumentError, 'max_interactions must be a non-negative integer'
+        raise ArgumentError, 'max_requests must be positive'
       end
     end
   end

data/lib/html2rss/request_service/compressed_body.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require 'brotli'
+require 'stringio'
+require 'zlib'
+
+module Html2rss
+  class RequestService
+    # Decodes gzip/brotli bodies when Content-Encoding is missing or the type is octet-stream.
+    module CompressedBody
+      # Gzip stream magic (RFC 1952).
+      GZIP_MAGIC = "\x1F\x8B".b.freeze
+      # HTML prefix for unlabeled brotli trial decode. Same object as {Response::HTML_BODY_SNIFF}.
+      HTML_BODY_SNIFF = Response::HTML_BODY_SNIFF
+
+      module_function
+
+      # @param body [String, nil] raw response body
+      # @param headers [Hash] response headers (Content-Encoding / Content-Type)
+      # @return [String, nil] decoded body when compressed; otherwise the original body
+      def decode(body, headers: {})
+        return body if body.nil? || body.empty?
+
+        raw = body.to_s.b
+        decode_labeled(raw, headers) || decode_unlabeled(raw, headers) || body
+      end
+
+      def decode_labeled(raw, headers)
+        encodings = content_encodings(headers)
+        return if encodings.empty?
+
+        encodings.reverse.reduce(raw) { |buffer, encoding| inflate(buffer, encoding) || buffer }
+      end
+      module_function :decode_labeled
+      private_class_method :decode_labeled
+
+      def decode_unlabeled(raw, headers)
+        return uncompress_gzip(raw) if raw.start_with?(GZIP_MAGIC)
+        return unless octet_stream?(headers)
+
+        try_brotli(raw)
+      end
+      module_function :decode_unlabeled
+      private_class_method :decode_unlabeled
+
+      def inflate(raw, encoding)
+        case encoding
+        when 'gzip' then uncompress_gzip(raw)
+        when 'deflate' then inflate_deflate(raw)
+        when 'br' then try_brotli(raw, require_html: false)
+        end
+      rescue Zlib::Error, Brotli::Error, ArgumentError
+        nil
+      end
+      module_function :inflate
+      private_class_method :inflate
+
+      def uncompress_gzip(raw)
+        Zlib::GzipReader.wrap(StringIO.new(raw), encoding: 'ASCII-8BIT', &:read)
+      end
+      module_function :uncompress_gzip
+      private_class_method :uncompress_gzip
+
+      def inflate_deflate(raw)
+        inflater = nil
+        Zlib::Inflate.inflate(raw)
+      rescue Zlib::DataError
+        inflater = Zlib::Inflate.new(-Zlib::MAX_WBITS)
+        inflater.inflate(raw)
+      ensure
+        inflater&.close
+      end
+      module_function :inflate_deflate
+      private_class_method :inflate_deflate
+
+      def try_brotli(raw, require_html: true)
+        inflated = Brotli.inflate(raw)
+        return inflated unless require_html
+        return inflated if inflated.b.match?(HTML_BODY_SNIFF)
+
+        nil
+      rescue Brotli::Error, ArgumentError
+        nil
+      end
+      module_function :try_brotli
+      private_class_method :try_brotli
+
+      def content_encodings(headers)
+        header(headers, 'content-encoding').to_s.split(',').map { |value| value.strip.downcase }.reject(&:empty?)
+      end
+      module_function :content_encodings
+      private_class_method :content_encodings
+
+      def octet_stream?(headers)
+        header(headers, 'content-type').to_s.downcase.include?('application/octet-stream')
+      end
+      module_function :octet_stream?
+      private_class_method :octet_stream?
+
+      def header(headers, name)
+        headers.fetch(name) do
+          headers.find { |key, _value| key.to_s.casecmp?(name) }&.last
+        end
+      end
+      module_function :header
+      private_class_method :header
+    end
+  end
+end