html2rss 0.33.0 → 0.34.0

data/lib/html2rss/mcp/config_argument.rb

@@ -8,12 +8,15 @@ module Html2rss
     #
     # Config does not know MCP. YAML parsing stays on {Html2rss::Config.from_yaml}.
     class ConfigArgument
+      # Raised unless exactly one of +config+ or +yaml+ is present.
+      class ExclusiveInputError < Html2rss::InvalidOption; end
+
       class << self
         ##
         # @param config [Hash, nil] feed configuration hash
         # @param yaml [String, nil] feed configuration YAML
         # @return [ConfigArgument]
-        # @raise [ArgumentError] unless exactly one of +config+ or +yaml+ is present
+        # @raise [ExclusiveInputError] unless exactly one of +config+ or +yaml+ is present
         # @raise [Html2rss::InvalidOption] when +config+ is not a Hash or +yaml+ is not a String
         # @raise [Contract::UnpublishedRequestError] when the parsed config uses
         #   an unpublished MCP strategy or +request.local_file_path+
@@ -24,7 +27,7 @@ module Html2rss
                    in [false, true]
                      Config.from_yaml(require_type!(yaml, String, 'yaml'))
                    else
-                     raise ArgumentError, 'Provide exactly one of config or yaml'
+                     raise ExclusiveInputError, 'Provide exactly one of config or yaml'
                    end
           Contract.assert_published_request!(parsed)
           new(config: parsed)

data/lib/html2rss/mcp/contract.rb

@@ -16,11 +16,11 @@ module Html2rss
       ALL_ACCEPTED_STRATEGIES = (STRATEGIES + MIGRATION_STRATEGIES).freeze
 
       # Bump when tool names, required inputs, or envelope semantics change (independent of gem +VERSION+).
-      MCP_CONTRACT_VERSION = 3
+      MCP_CONTRACT_VERSION = 4
       public_constant :MCP_CONTRACT_VERSION
 
       # Raised when apply/validate config uses an unpublished MCP request adapter.
-      class UnpublishedRequestError < ArgumentError; end
+      class UnpublishedRequestError < Html2rss::InvalidOption; end
 
       # JSON Schema property for a source page URL.
       URL_PROPERTY = {
@@ -244,7 +244,14 @@ module Html2rss
               ok: { type: 'boolean' },
               next_step: { type: 'string', enum: Outcome::NextStep::ALL.keys.map(&:to_s) },
               guidance: { type: 'string' },
-              payload: { type: 'object' }
+              payload: { type: 'object' },
+              cache: {
+                type: 'object',
+                description: 'Present when part of this result came from the response cache instead of the live page',
+                additionalProperties: false,
+                required: %w[hits max_age_seconds],
+                properties: { hits: { type: 'integer' }, max_age_seconds: { type: 'integer' } }
+              }
             }
           }
         end
@@ -253,9 +260,11 @@ module Html2rss
         # One envelope Hash, one compact JSON body, no +_meta+.
         #
         # @param outcome [Outcome]
+        # @param cache_ages [Array<Integer>] ages in seconds of cached page fetches behind this result
         # @return [::MCP::Tool::Response]
-        def response(outcome)
+        def response(outcome, cache_ages: [])
           wire = outcome.to_h
+          wire[:cache] = { hits: cache_ages.size, max_age_seconds: cache_ages.max } unless cache_ages.empty?
           ::MCP::Tool::Response.new(
             [{ type: 'text', text: JSON.generate(wire) }],
             error: !outcome.ok,

data/lib/html2rss/mcp/outcome.rb

@@ -8,8 +8,9 @@ module Html2rss
     # Typed MCP tool result. Owns next-step policy; guidance copy lives in
     # {Playbook}.
     class Outcome # rubocop:disable Metrics/ClassLength -- next-step policy + factories stay co-located
-      # Matches {ConfigArgument} XOR +ArgumentError+ messages.
-      XOR_ERROR = /exactly one of config or yaml/
+      # Raised when an outcome is built with a wrong-typed next step or payload.
+      class InvalidError < Html2rss::ContractError; end
+
       NextStep = Data.define(:name, :guidance)
 
       ##
@@ -22,7 +23,7 @@ module Html2rss
         ##
         # @param name [Symbol, String]
         # @return [NextStep]
-        def self.[](name) = ALL.fetch(name.to_sym) { raise ArgumentError, "unknown next_step: #{name.inspect}" }
+        def self.[](name) = ALL.fetch(name.to_sym) { raise InvalidOption, "unknown next_step: #{name.inspect}" }
       end
 
       ##
@@ -31,8 +32,8 @@ module Html2rss
       # @param guidance [String]
       # @param payload [Hash]
       def initialize(ok:, next_step:, guidance:, payload:) # rubocop:disable Naming/MethodParameterName -- +ok+ is the envelope field
-        raise ArgumentError, 'next_step must be a NextStep' unless next_step.is_a?(NextStep)
-        raise ArgumentError, 'payload must be a Hash' unless payload.is_a?(Hash)
+        raise InvalidError, 'next_step must be a NextStep' unless next_step.is_a?(NextStep)
+        raise InvalidError, 'payload must be a Hash' unless payload.is_a?(Hash)
 
         super(ok: !!ok, next_step:, guidance: guidance.to_s.freeze, payload: payload.dup.freeze)
       end
@@ -218,7 +219,13 @@ module Html2rss
           warnings = test_result.quality_report&.warnings
           return base if warnings.nil? || warnings.empty?
 
-          "#{base} Review payload.quality_report warnings: #{warnings.join(', ')}."
+          "#{base} Review payload.quality_report warnings: #{warnings.join(', ')}.#{items_without_url_hint(warnings)}"
+        end
+
+        def items_without_url_hint(warnings)
+          return '' unless warnings.include?(:items_without_url)
+
+          ' Items were dropped for having no URL: fix the selectors.url selector (it must match a link on each item).'
         end
 
         def capture_payload(yaml:, articles_count:, has_selectors:, channel_title:, requested_strategy:, # rubocop:disable Metrics/ParameterLists
@@ -237,15 +244,10 @@ module Html2rss
         def next_step_for_error(error)
           case error
           when RequestService::BotasaurusConfigurationError then NextStep[:read_runtime]
-          when Contract::UnpublishedRequestError then NextStep[:validate]
-          when ArgumentError then argument_error_next_step(error)
+          when Contract::UnpublishedRequestError, ConfigArgument::ExclusiveInputError then NextStep[:validate]
           else NextStep[:inspect]
           end
         end
-
-        def argument_error_next_step(error)
-          XOR_ERROR.match?(error.message) ? NextStep[:validate] : NextStep[:inspect]
-        end
       end
     end
   end

data/lib/html2rss/mcp/runtime.rb

@@ -11,9 +11,16 @@ module Html2rss
         :mcp_contract_version,
         :catalog_fingerprint,
         :tools,
-        :botasaurus_configured
+        :botasaurus_configured,
+        :response_cache_ttl
       )
 
+      # Seconds an initial page fetch is reused when +HTML2RSS_RESPONSE_CACHE_TTL+ is unset.
+      DEFAULT_RESPONSE_CACHE_TTL = 300
+      # Values of +HTML2RSS_RESPONSE_CACHE_TTL+ that disable the cache.
+      CACHE_OFF_VALUES = %w[0 off false].freeze
+      private_constant :CACHE_OFF_VALUES
+
       class << self
         ##
         # @return [Boolean] whether Botasaurus transport is configured in this process
@@ -21,6 +28,18 @@ module Html2rss
           !ENV['BOTASAURUS_SCRAPER_URL'].to_s.strip.empty?
         end
 
+        ##
+        # Installs the initial-fetch cache for the long-lived MCP process, so an agent iterating
+        # +inspect -> capture -> validate -> test+ on one URL fetches it once per TTL window.
+        # +HTML2RSS_RESPONSE_CACHE_TTL+ sets the seconds ({DEFAULT_RESPONSE_CACHE_TTL} when unset);
+        # +0+, +off+ or +false+ disables caching. An unusable value logs a warning and uses the default.
+        #
+        # @return [void]
+        def install_response_cache!
+          ttl = response_cache_ttl
+          RequestService.response_cache = ttl && RequestService::ResponseCache.new(ttl:)
+        end
+
         ##
         # @return [Snapshot] runtime capabilities and catalog identity for MCP clients
         def snapshot
@@ -29,9 +48,27 @@ module Html2rss
             mcp_contract_version: Contract::MCP_CONTRACT_VERSION,
             catalog_fingerprint: Contract.catalog_fingerprint,
             tools: Contract.catalog_tools,
-            botasaurus_configured: botasaurus_configured?
+            botasaurus_configured: botasaurus_configured?,
+            response_cache_ttl: RequestService.response_cache&.ttl
           )
         end
+
+        private
+
+        def response_cache_ttl
+          raw = ENV['HTML2RSS_RESPONSE_CACHE_TTL'].to_s.strip.downcase
+          return DEFAULT_RESPONSE_CACHE_TTL if raw.empty?
+          return if CACHE_OFF_VALUES.include?(raw)
+
+          ttl = Integer(raw, 10, exception: false)
+          ttl&.positive? ? ttl : invalid_cache_ttl(raw)
+        end
+
+        def invalid_cache_ttl(raw)
+          Log.warn("HTML2RSS_RESPONSE_CACHE_TTL=#{raw.inspect} is not a positive integer; " \
+                   "using #{DEFAULT_RESPONSE_CACHE_TTL}s")
+          DEFAULT_RESPONSE_CACHE_TTL
+        end
       end
     end
   end

data/lib/html2rss/mcp/server.rb

@@ -23,6 +23,9 @@ module Html2rss
       # Supported transport names.
       TRANSPORTS = Set[:stdio, :http].freeze
 
+      # Raised when the HTTP transport's optional gems (+rackup+, +webrick+) cannot be loaded.
+      class HttpTransportUnavailable < Html2rss::Error; end
+
       # Declarative MCP resource registrations consumed by {register_resources}.
       RESOURCES = [
         {
@@ -95,10 +98,13 @@ module Html2rss
         #
         # @param transport [Symbol] +:stdio+ or +:http+
         # @param port [Integer] port for HTTP transport
+        # @raise [InvalidOption] for an unknown transport
+        # @raise [HttpTransportUnavailable] when +:http+ is requested without the rackup and webrick gems
         def start(transport: :stdio, port: 8080)
-          raise ArgumentError, "Unknown transport: #{transport.inspect}" unless TRANSPORTS.include?(transport)
+          raise InvalidOption, "Unknown transport: #{transport.inspect}" unless TRANSPORTS.include?(transport)
 
           configure_daemon_logging!
+          Runtime.install_response_cache!
           app = build
           Log.info(start_banner(transport:, port:))
           return start_http(app, port:) if transport == :http
@@ -204,7 +210,7 @@ module Html2rss
             Logger: Html2rss.logger
           )
         rescue LoadError => error
-          raise LoadError,
+          raise HttpTransportUnavailable,
                 'HTTP transport requires the rackup and webrick gems ' \
                 "(#{error.message}). Install them or use --transport stdio."
         end
@@ -218,8 +224,11 @@ module Html2rss
           Contract.response(Outcome.from_error(error))
         end
 
-        def handle_tool_call
-          Contract.response(yield)
+        def handle_tool_call(&)
+          outcome, cache_ages = RequestService.observe_cache_hits(&)
+          Contract.response(outcome, cache_ages:)
+        rescue Html2rss::ContractError
+          raise
         rescue StandardError => error
           tool_error_response(error)
         end

data/lib/html2rss/page_recon/README.md

@@ -10,7 +10,7 @@
 | `final_url`        | URL after redirects (may differ from requested)                             |
 | `status`           | HTTP status of the **final** response                                       |
 | `scheme_downgrade` | `true` when HTTPS entry landed on HTTP                                      |
-| `alternate_feeds`  | `rel=alternate` RSS/Atom links found in HTML                                |
+| `alternate_feeds`  | `rel=alternate` RSS/Atom/RDF links found in HTML                                |
 | `surface_category` | AutoSource surface class (`high_entropy_surface`, `unsupported_surface`, …) |
 | `articles_count`   | Cheap AutoSource extract (limit 10) — diagnostic only, not ship quality     |
 | `strategy`         | Concrete transport used (`inspect` maps `auto` → default (HTTPX))           |

data/lib/html2rss/page_recon/diagnostics.rb

@@ -57,6 +57,8 @@ module Html2rss
       def self.batch(urls:, strategy: :auto, concurrency: Batch::DEFAULT_CONCURRENCY)
         Batch.map(Array(urls), concurrency:) do |url|
           call(url:, strategy:)
+        rescue Html2rss::ContractError
+          raise
         rescue Html2rss::Error => error
           error_report(url, error)
         end

data/lib/html2rss/recon.rb

@@ -7,7 +7,7 @@ require 'fileutils'
 module Html2rss
   ##
   # Service that runs reconnaissance on a URL to discover redirect chains,
-  # native RSS/Atom feeds, surface categorization, and emit an actionable verdict.
+  # native RSS/Atom/RDF feeds, surface categorization, and emit an actionable verdict.
   module Recon # rubocop:disable Metrics/ModuleLength
     Verdict = Data.define(:name)
 
@@ -21,7 +21,7 @@ module Html2rss
       ##
       # @param name [Symbol, String]
       # @return [Verdict]
-      def self.[](name) = ALL.fetch(name.to_sym) { raise ArgumentError, "unknown verdict: #{name.inspect}" }
+      def self.[](name) = ALL.fetch(name.to_sym) { raise InvalidOption, "unknown verdict: #{name.inspect}" }
 
       ##
       # @return [Boolean]
@@ -109,6 +109,8 @@ module Html2rss
 
       begin
         probe = PageRecon.probe(url_obj, strategy:, **)
+      rescue Html2rss::ContractError
+        raise
       rescue Html2rss::Error => error
         return error_result(url_obj, error)
       end
@@ -116,7 +118,7 @@ module Html2rss
       cache_html_body(probe.response.body, url_obj, cache_dir, cache_mutex) if cache_dir && probe.response.body
 
       recon = probe.result
-      native_feed, discovery_note = find_native_feed(url_obj, probe.session, probe.response)
+      native_feed, discovery_note = find_native_feed(probe.session, probe.response)
       notes = build_notes(recon, native_feed, probe.response)
       notes << discovery_note if discovery_note
       verdict = determine_verdict(recon, native_feed)
@@ -160,18 +162,14 @@ module Html2rss
     end
 
     ##
-    # @param url_obj [Html2rss::Url]
     # @param session [Html2rss::RequestSession]
-    # @param response [Html2rss::RequestService::Response]
+    # @param response [Html2rss::RequestService::Response] the fetched page, where any redirect landed
     # @return [Array(Html2rss::Url, nil), String, nil)] feed URL and optional discovery error note
-    def self.find_native_feed(url_obj, session, response)
-      feed = Syndication::Discovery.best_feed_url(
-        page_url: url_obj,
-        request_session: session,
-        parsed_body: (response.parsed_body if response.html_response?),
-        html: response.body
-      )
+    def self.find_native_feed(session, response)
+      feed = Syndication::Discovery.native_feed_url(response:, request_session: session)
       [feed, nil]
+    rescue Html2rss::ContractError
+      raise
     rescue Html2rss::Error => error
       [nil, "discovery_error=#{error.class}: #{error.message}"]
     end

data/lib/html2rss/request_service/blocked_surface.rb

@@ -34,7 +34,8 @@ module Html2rss
             /interstitial/i
           ],
           message: 'Blocked surface detected: DataDome anti-bot challenge page. ' \
-                   'Target a direct listing URL, or scrape via a session with resolved DataDome cookies.'
+                   'Target a direct listing URL. Clearance cookies from a successful Botasaurus scrape ' \
+                   'are reused automatically; to supply your own, set request.botasaurus.cookies.'
         },
         {
           key: :vercel_security_checkpoint,

data/lib/html2rss/request_service/botasaurus_contract.rb

@@ -11,7 +11,7 @@ module Html2rss
       # Closed set from OpenAPI ExecutionMode.
       EXECUTION_MODES = Set['auto', 'request', 'browser'].freeze
       # Closed set from OpenAPI NavigationMode.
-      NAVIGATION_MODES = Set['auto', 'get', 'google_get', 'google_get_bypass', 'organic_get'].freeze
+      NAVIGATION_MODES = Set['auto', 'get', 'google_get', 'google_get_bypass'].freeze
       # Closed set from OpenAPI ErrorCategory.
       ERROR_CATEGORIES = Set['timeout', 'challenge_block', 'navigation_error', 'metadata_error', 'validation'].freeze
 
@@ -84,6 +84,15 @@ module Html2rss
         # @return [String, nil] final URL reported by upstream
         def final_url = payload['final_url']
 
+        # Session credentials: keep out of logs, transport_meta and feed output.
+        # @return [Hash{String => String}] cookies earned for the target host (absent or malformed → {})
+        def cookies
+          raw = payload['cookies']
+          return {} unless raw.is_a?(Hash)
+
+          raw.select { |name, value| name.is_a?(String) && value.is_a?(String) }
+        end
+
         # @return [Hash{String => Object}] diagnostics plus success-only metadata_error (frozen)
         def transport_meta
           meta = diagnostics.dup
@@ -258,6 +267,7 @@ module Html2rss
       # @param url [Html2rss::Url] canonical URL to scrape
       # @param headers [Hash] request headers from context
       # @param options [Hash] validated request.botasaurus options
+      # @param stored_cookies [Hash{String => String}] cookies replayed from the cookie store
       # @option options [String] :execution_mode
       # @option options [String] :navigation_mode
       # @option options [Integer] :max_retries
@@ -275,18 +285,23 @@ module Html2rss
       # @option options [String] :lang
       # @option options [Hash] :cookies
       # @option options [Hash] :headers
-      def initialize(url:, headers: {}, options: {})
+      def initialize(url:, headers: {}, options: {}, stored_cookies: {})
         @url = url
         @headers = headers
         @options = options
+        @stored_cookies = stored_cookies
       end
 
+      # @return [String] omits options and stored cookies (session credentials)
+      def inspect = "#<#{self.class} url=#{url}>"
+
       # @return [Hash] payload for POST /scrape (explicit options plus wait cap)
       def request_payload
         payload = { url: url.to_s }.merge(filtered_options)
         forwarded_headers = merged_headers
         payload[:headers] = forwarded_headers if forwarded_headers&.any?
         cap_wait_timeout!(payload)
+        merge_cookies!(payload)
         payload
       end
 
@@ -302,7 +317,7 @@ module Html2rss
 
       private
 
-      attr_reader :url, :headers, :options
+      attr_reader :url, :headers, :options, :stored_cookies
 
       def json_object(body)
         payload = JSON.parse(body)
@@ -326,6 +341,13 @@ module Html2rss
         end
       end
 
+      # Stored cookies are replayed in every execution_mode; cookies from the feed config win.
+      def merge_cookies!(payload)
+        configured = (options[:cookies] || {}).transform_keys(&:to_s)
+        merged = stored_cookies.merge(configured)
+        merged.any? ? payload[:cookies] = merged : payload.delete(:cookies)
+      end
+
       def cap_wait_timeout!(payload)
         wait = payload[:wait_timeout_seconds]
         return unless wait

data/lib/html2rss/request_service/botasaurus_strategy.rb

@@ -8,7 +8,7 @@ module Html2rss
   class RequestService
     ##
     # Strategy to delegate fetching to a Botasaurus scrape API.
-    class BotasaurusStrategy < Strategy
+    class BotasaurusStrategy < Strategy # rubocop:disable Metrics/ClassLength -- transport + cookie-jar retry stay co-located
       # Content-Type negotiation for the scrape API transport hop.
       TRANSPORT_ACCEPT = 'application/json'
       # Disable compressed bodies so HTTPX returns raw JSON without implicit decoding surprises.
@@ -18,16 +18,49 @@ module Html2rss
 
       private
 
+      # With a stored jar, tries the cheap HTTP tier first; cookies never reach the browser tier,
+      # so a block falls back to a plain scrape without them.
       def fetch
-        parsed_response = post_scrape_request
+        stored = jar.stored
+        return scrape({}) unless replay?(stored)
+
+        replay(stored)
+      end
+
+      def replay?(stored) = botasaurus_options[:execution_mode] != 'browser' && jar.replayable?(stored)
+
+      def replay(stored)
+        response = scrape(stored, execution_mode: 'request')
+        jar.replayed(:pass, stored)
+        response
+      rescue BlockedSurfaceDetected
+        jar.replayed(:block, stored)
+        scrape({})
+      end
+
+      def scrape(stored, overrides = {})
+        parsed_response = post_scrape_request(stored, overrides)
         raise_from_error!(parsed_response) if parsed_response.is_a?(BotasaurusContract::Error)
 
-        build_response(parsed_response)
+        response = build_response(parsed_response)
+        ResponseGuard.new(policy: ctx.policy).inspect_body!(response.body)
+        jar.refresh(stored, response.cookies)
+        response
       end
 
-      def post_scrape_request
+      def jar
+        @jar ||= CookieJar.new(
+          host: ctx.url.host, scraper_base_url: scraper_base_url.to_s, proxy: botasaurus_options[:proxy],
+          user_agent: botasaurus_options[:user_agent] || ctx.headers['User-Agent']
+        )
+      end
+
+      def botasaurus_options = ctx.request.fetch(:botasaurus, {})
+
+      def post_scrape_request(stored, overrides)
         request_id = SecureRandom.uuid
         Log.debug("#{self.class}: POST /scrape #{REQUEST_ID_HEADER}=#{request_id}")
+        contract = contract(stored, overrides)
         payload = JSON.generate(contract.request_payload)
         response = client.post(scrape_endpoint, body: payload, headers: post_headers(request_id))
         raise response.error if response.is_a?(HTTPX::ErrorResponse)
@@ -44,7 +77,8 @@ module Html2rss
           url: response_url(parsed_response.final_url),
           status: parsed_response.status,
           transport_meta: parsed_response.transport_meta,
-          captured_responses: parsed_response.xhr_responses
+          captured_responses: parsed_response.xhr_responses,
+          cookies: parsed_response.cookies
         )
       end
 
@@ -69,11 +103,9 @@ module Html2rss
         ctx.url
       end
 
-      def contract
-        @contract ||= BotasaurusContract.new(
-          url: ctx.url,
-          headers: ctx.headers,
-          options: ctx.request.fetch(:botasaurus, {})
+      def contract(stored, overrides)
+        BotasaurusContract.new(
+          url: ctx.url, headers: ctx.headers, options: botasaurus_options.merge(overrides), stored_cookies: stored
         )
       end
 

data/lib/html2rss/request_service/context.rb

@@ -6,6 +6,9 @@ module Html2rss
     # Holds information needed to send requests to websites.
     # To be passed down to the RequestService's strategies.
     class Context
+      # Raised when a context is built without its runtime policy or request budget.
+      class InvalidError < Html2rss::ContractError; end
+
       ##
       # @param url [String, Html2rss::Url] the URL to request
       # @param headers [Hash] HTTP request headers
@@ -15,7 +18,8 @@ module Html2rss
       # @option request_options [String, Html2rss::Url, nil] :origin_url originating URL for same-origin checks
       # @option request_options [Policy] :policy runtime request policy
       # @option request_options [Budget] :budget shared request budget for the feed build
-      # @raise [ArgumentError] if policy or budget is explicitly nil
+      # @option request_options [FeedAdvertisement, nil] :advertisement page proof for an +:advertised_feed+ follow-up
+      # @raise [InvalidError] if policy or budget is nil or budget is missing
       def initialize(url:, headers: {}, request: {}, **request_options)
         @url = Html2rss::Url.from_absolute(url)
         @headers = normalize_headers(headers).freeze
@@ -44,23 +48,19 @@ module Html2rss
       # @return [Budget] the shared request budget
       attr_reader :budget
 
+      # @return [FeedAdvertisement, nil] the page's advertised feeds, for an +:advertised_feed+ follow-up
+      attr_reader :advertisement
+
       ##
       # Builds a follow-up request context sharing headers, budget, and policy.
       #
       # @param url [String, Html2rss::Url] the follow-up URL
       # @param relation [Symbol] why the follow-up is being made
       # @param origin_url [String, Html2rss::Url] effective origin for same-origin checks
+      # @param advertisement [FeedAdvertisement, nil] page proof for an +:advertised_feed+ follow-up
       # @return [Context] derived request context
-      def follow_up(url:, relation:, origin_url: self.origin_url)
-        self.class.new(
-          url:,
-          headers:,
-          request:,
-          relation:,
-          origin_url:,
-          policy:,
-          budget:
-        )
+      def follow_up(url:, relation:, origin_url: self.origin_url, advertisement: nil)
+        self.class.new(url:, headers:, request:, relation:, origin_url:, policy:, budget:, advertisement:)
       end
 
       private
@@ -68,11 +68,12 @@ module Html2rss
       def assign_request_options(request_options)
         @relation = request_options.fetch(:relation, :initial)
         @policy = request_options.fetch(:policy, Policy.default)
-        raise ArgumentError, 'policy must not be nil' if @policy.nil?
+        raise InvalidError, 'policy must not be nil' if @policy.nil?
 
         @origin_url = normalized_origin_url(request_options[:origin_url])
-        @budget = request_options.fetch(:budget) { raise ArgumentError, 'budget is required' }
-        raise ArgumentError, 'budget must not be nil' if @budget.nil?
+        @advertisement = request_options[:advertisement]
+        @budget = request_options.fetch(:budget) { raise InvalidError, 'budget is required' }
+        raise InvalidError, 'budget must not be nil' if @budget.nil?
       end
 
       def normalized_origin_url(origin_url)

data/lib/html2rss/request_service/cookie_jar.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestService
+    ##
+    # One host/egress-scoped view onto the {CookieStore}: replay, refresh and evict with count-only telemetry.
+    # Cookie names and values are session credentials and never reach logs.
+    class CookieJar
+      # Share of replayable requests that deliberately skip replay, so the pass-rate effect stays measurable.
+      HOLDOUT_RATE = Float(ENV.fetch('BOTASAURUS_COOKIE_HOLDOUT_RATE', 0.1))
+      # Shared across jars so blocks accumulate per key within the process.
+      BREAKER = ReplayBreaker.new
+
+      ##
+      # @param host [String] target host
+      # @param scraper_base_url [String] scrape-api base URL
+      # @param proxy [String, nil] egress proxy
+      # @param user_agent [String, nil] browser user agent
+      # @param store [#fetch, #write, #evict] the configured cookie store
+      def initialize(host:, scraper_base_url:, proxy:, user_agent:, store: RequestService.cookie_store)
+        @store = store
+        @host = host
+        @key = CookieStore.key(host:, scraper_base_url:, proxy:, user_agent:)
+      end
+
+      # @return [Hash{String => String}] live stored cookies ({} when none)
+      def stored = store.fetch(key)
+
+      ##
+      # @param stored [Hash{String => String}] cookies in the jar
+      # @return [Boolean] whether to replay them (not when empty, the breaker is open, or in the holdout)
+      def replayable?(stored)
+        return false if stored.empty?
+
+        reason = skip_reason
+        log(reason, 0) if reason
+        reason.nil?
+      end
+
+      ##
+      # Records what a replay led to: a pass resets the breaker; a block evicts the jar and counts toward it.
+      #
+      # @param outcome [Symbol] +:pass+ or +:block+
+      # @param stored [Hash{String => String}] cookies that were replayed
+      # @return [void]
+      def replayed(outcome, stored)
+        log("replay_#{outcome}", stored.size)
+        return BREAKER.record_pass(key) if outcome == :pass
+
+        evict(stored)
+        log('breaker_open', 0) if BREAKER.record_block(key, ttl: CookieStore::DEFAULT_TTL_SECONDS)
+      end
+
+      ##
+      # @param stored [Hash{String => String}] cookies that were replayed
+      # @param earned [Hash{String => String}] cookies returned by the scrape
+      # @return [void]
+      def refresh(stored, earned)
+        ttl = CookieStore::DEFAULT_TTL_SECONDS
+        return if earned.empty? || ttl <= 0
+
+        store.write(key, stored.merge(earned), ttl:)
+        log('refresh', earned.size)
+      end
+
+      ##
+      # @param stored [Hash{String => String}] cookies that were rejected
+      # @return [void]
+      def evict(stored)
+        store.evict(key)
+        log('evict', stored.size)
+      end
+
+      private
+
+      attr_reader :store, :key, :host
+
+      def skip_reason
+        return 'breaker_open' if BREAKER.open?(key)
+
+        'holdout' if rand < HOLDOUT_RATE
+      end
+
+      def log(action, count)
+        Log.info({ event_name: 'botasaurus.cookie_jar', action:, host:, cookie_count: count }.to_json)
+      end
+    end
+  end
+end