html2rss 0.32.4 → 0.34.0

data/lib/html2rss/request_service/policy.rb

@@ -9,18 +9,26 @@ module Html2rss
     #
     # Public façade for request limits, same-origin/follow-up rules, and
     # network reachability. DNS/host/IP enforcement lives on {NetworkGuard}.
+    #
+    # Follow-ups stay on the origin (scheme, host, port) unless the relation is
+    # +:advertised_feed+ and the URL is listed in the page's {FeedAdvertisement}; redirects
+    # of such a request may change host over +https+. Budget, scheme, public-network
+    # and response-size guards apply to every request and hop either way.
     class Policy
       # Hard ceiling for configured max_requests during a feed build.
       MAX_REQUESTS_CEILING = 10
 
+      # Default response-size ceiling (10 MiB); owns both streamed and decompressed defaults.
+      MAX_RESPONSE_BYTES = 10 * 1024 * 1024
+
       # Default policy values used when request controls are not explicitly set.
       DEFAULTS = {
         connect_timeout_seconds: Integer(ENV.fetch('HTML2RSS_CONNECT_TIMEOUT_SECONDS', 5)),
         read_timeout_seconds: Integer(ENV.fetch('HTML2RSS_READ_TIMEOUT_SECONDS', 10)),
         total_timeout_seconds: Integer(ENV.fetch('HTML2RSS_TOTAL_TIMEOUT_SECONDS', 30)),
         max_redirects: 5,
-        max_response_bytes: 5_242_880,
-        max_decompressed_bytes: 10_485_760,
+        max_response_bytes: MAX_RESPONSE_BYTES,
+        max_decompressed_bytes: MAX_RESPONSE_BYTES,
         max_requests: 1,
         allow_private_networks: false,
         allow_cross_origin_followups: false
@@ -47,13 +55,13 @@ module Html2rss
                      allow_private_networks: DEFAULTS[:allow_private_networks],
                      allow_cross_origin_followups: DEFAULTS[:allow_cross_origin_followups],
                      resolver: Socket)
-        @connect_timeout_seconds = validate_positive_integer!(:connect_timeout_seconds, connect_timeout_seconds)
-        @read_timeout_seconds = validate_positive_integer!(:read_timeout_seconds, read_timeout_seconds)
-        @total_timeout_seconds = validate_positive_integer!(:total_timeout_seconds, total_timeout_seconds)
-        @max_redirects = validate_non_negative_integer!(:max_redirects, max_redirects)
-        @max_response_bytes = validate_positive_integer!(:max_response_bytes, max_response_bytes)
-        @max_decompressed_bytes = validate_positive_integer!(:max_decompressed_bytes, max_decompressed_bytes)
-        @max_requests = [validate_positive_integer!(:max_requests, max_requests), MAX_REQUESTS_CEILING].min
+        @connect_timeout_seconds = Option.integer(:connect_timeout_seconds, connect_timeout_seconds, min: 1)
+        @read_timeout_seconds = Option.integer(:read_timeout_seconds, read_timeout_seconds, min: 1)
+        @total_timeout_seconds = Option.integer(:total_timeout_seconds, total_timeout_seconds, min: 1)
+        @max_redirects = Option.integer(:max_redirects, max_redirects, min: 0)
+        @max_response_bytes = Option.integer(:max_response_bytes, max_response_bytes, min: 1)
+        @max_decompressed_bytes = Option.integer(:max_decompressed_bytes, max_decompressed_bytes, min: 1)
+        @max_requests = [Option.integer(:max_requests, max_requests, min: 1), MAX_REQUESTS_CEILING].min
         @allow_private_networks = allow_private_networks ? true : false
         @allow_cross_origin_followups = allow_cross_origin_followups ? true : false
         @network_guard = NetworkGuard.new(allow_private_networks: @allow_private_networks, resolver:)
@@ -80,6 +88,12 @@ module Html2rss
         @allow_cross_origin_followups
       end
 
+      ##
+      # @return [Array] the policy fields that shape a fetched response (for response caching)
+      def cache_key
+        [allow_private_networks?, max_redirects, max_response_bytes, max_decompressed_bytes]
+      end
+
       ##
       # Returns the default request policy.
       #
@@ -89,17 +103,30 @@ module Html2rss
         new
       end
 
+      ##
+      # Whether two URLs share an origin (scheme, host, and port).
+      #
+      # @param url [Html2rss::Url]
+      # @param other [Html2rss::Url]
+      # @return [Boolean]
+      def self.same_origin?(url, other)
+        [url.scheme, url.host, url.port] == [other.scheme, other.host, other.port]
+      end
+
       ##
       # Validates whether a request target is permitted for the given context.
       #
       # @param url [Html2rss::Url] destination URL
       # @param origin_url [Html2rss::Url] initial URL of the feed build
       # @param relation [Symbol] logical reason for the request
+      # @param advertisement [FeedAdvertisement, nil] the page's advertised feeds (+:advertised_feed+ only)
       # @return [void]
       # @raise [CrossOriginFollowUpDenied] if a follow-up leaves the origin host
       # @raise [PrivateNetworkDenied] if the target resolves to a private address
-      def validate_request!(url:, origin_url:, relation:)
-        enforce_same_origin!(url, origin_url, relation)
+      def validate_request!(url:, origin_url:, relation:, advertisement: nil)
+        enforce_same_origin!(url, origin_url, relation) do
+          advertised_feed?(origin_url, relation, advertisement) && advertisement.advertises?(url)
+        end
         network_guard.enforce_public_network!(url)
       end
 
@@ -110,14 +137,20 @@ module Html2rss
       # @param to_url [Html2rss::Url] redirect destination
       # @param origin_url [Html2rss::Url] initial URL of the feed build
       # @param relation [Symbol] logical reason for the request
+      # @param advertisement [FeedAdvertisement, nil] the page's advertised feeds (+:advertised_feed+ only)
       # @return [void]
       # @raise [UnsupportedUrlScheme] if the redirect downgrades from HTTPS to HTTP
-      def validate_redirect!(from_url:, to_url:, origin_url:, relation:)
+      # @raise [CrossOriginFollowUpDenied] if a follow-up hop leaves the origin host
+      # @raise [PrivateNetworkDenied] if the hop resolves to a private address
+      def validate_redirect!(from_url:, to_url:, origin_url:, relation:, advertisement: nil)
         if from_url.scheme == 'https' && to_url.scheme == 'http'
           raise UnsupportedUrlScheme, "Redirect downgraded from #{from_url} to #{to_url}"
         end
 
-        validate_request!(url: to_url, origin_url:, relation:)
+        enforce_same_origin!(to_url, origin_url, relation) do
+          advertised_feed?(origin_url, relation, advertisement) && to_url.scheme == 'https'
+        end
+        network_guard.enforce_public_network!(to_url)
       end
 
       ##
@@ -135,42 +168,28 @@ module Html2rss
 
       attr_reader :network_guard
 
-      def validate_positive_integer!(name, value)
-        raise ArgumentError, "#{name} must be positive" unless value.is_a?(Integer) && value.positive?
-
-        value
-      end
-
-      def validate_non_negative_integer!(name, value)
-        raise ArgumentError, "#{name} must be non-negative" unless value.is_a?(Integer) && !value.negative?
-
-        value
-      end
-
+      # Yields only for a cross-origin follow-up: a truthy block admits it.
       def enforce_same_origin!(url, origin_url, relation)
         return if relation == :initial || allow_cross_origin_followups?
 
         enforce_follow_up_scheme!(url, origin_url)
-        return if comparable_origin(url) == comparable_origin(origin_url)
+        return if self.class.same_origin?(url, origin_url) || yield
 
         raise CrossOriginFollowUpDenied, "Cross-origin follow-up denied for #{url}"
       end
 
+      # The proof must come from the page this follow-up originates from; an unfrozen one was not built by
+      # {FeedAdvertisement.from_document}.
+      def advertised_feed?(origin_url, relation, advertisement)
+        relation == :advertised_feed && advertisement.instance_of?(FeedAdvertisement) && advertisement.frozen? &&
+          self.class.same_origin?(advertisement.page_url, origin_url)
+      end
+
       def enforce_follow_up_scheme!(url, origin_url)
         return unless origin_url.scheme == 'https' && url.scheme == 'http'
 
         raise UnsupportedUrlScheme, "Follow-up downgraded from https to http for #{url}"
       end
-
-      def comparable_origin(url)
-        [url.host, normalized_port(url)]
-      end
-
-      def normalized_port(url)
-        return url.port if url.port
-
-        url.scheme == 'https' ? 443 : 80
-      end
     end
 
     # Shared immutable policy instance used for default request execution.

data/lib/html2rss/request_service/replay_breaker.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestService
+    ##
+    # Process-local circuit breaker for cookie replay: after {MAX_BLOCKS} consecutive replay-then-block
+    # outcomes for a jar key, replay stays off until the cookie TTL elapses.
+    class ReplayBreaker
+      # Consecutive blocked replays before the breaker opens.
+      MAX_BLOCKS = Integer(ENV.fetch('BOTASAURUS_COOKIE_REPLAY_MAX_BLOCKS', 3))
+
+      def initialize
+        @state = {}
+        @mutex = Mutex.new
+      end
+
+      ##
+      # @param key [String] jar key
+      # @return [Boolean] true while replay is suspended for the key
+      def open?(key)
+        @mutex.synchronize do
+          open_until = @state[key]&.last
+          next false unless open_until
+          next true if now < open_until
+
+          @state.delete(key)
+          false
+        end
+      end
+
+      ##
+      # @param key [String] jar key
+      # @param ttl [Numeric] seconds replay stays off once the breaker trips
+      # @return [Boolean] true when this block tripped the breaker
+      def record_block(key, ttl:)
+        @mutex.synchronize do
+          blocks = (@state[key]&.first || 0) + 1
+          tripped = blocks >= MAX_BLOCKS
+          @state[key] = [blocks, (now + ttl if tripped)]
+          tripped
+        end
+      end
+
+      ##
+      # @param key [String] jar key
+      # @return [void]
+      def record_pass(key)
+        @mutex.synchronize { @state.delete(key) }
+      end
+
+      private
+
+      def now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+end

data/lib/html2rss/request_service/response.rb

@@ -6,18 +6,20 @@ module Html2rss
   class RequestService
     ##
     # To be used by strategies to provide their response.
-    class Response
+    class Response # rubocop:disable Metrics/ClassLength -- transport + capture + cache fields stay co-located
       # Default when a strategy does not attach transport telemetry.
       EMPTY_TRANSPORT_META = {}.freeze
       # Default when a strategy does not capture sub-resource responses.
       EMPTY_CAPTURED_RESPONSES = [].freeze
+      # Default when a strategy returns no cookies.
+      EMPTY_COOKIES = {}.freeze
 
       # Bodies that look like HTML even when Content-Type is missing, empty, or wrong.
       HTML_BODY_SNIFF = /\A\s*(?:<!DOCTYPE\s+html|<html)/i
-      # Content-Type markers for RSS/Atom syndication responses.
-      FEED_CT_MARKERS = %w[rss+xml atom+xml rss atom].freeze
+      # Content-Type markers for RSS/Atom/RDF syndication responses.
+      FEED_CT_MARKERS = %w[rss+xml atom+xml rdf+xml rss atom].freeze
       # Body sniff for unlabeled syndication (first 800 bytes, downcased).
-      FEED_BODY_MARKERS = ['<rss', '<feed xmlns', '<feed '].freeze
+      FEED_BODY_MARKERS = ['<rss', '<feed xmlns', '<feed ', '<rdf:rdf'].freeze
       # Charset from Content-Type or a leading <meta charset>.
       CHARSET_PARAMETER = /charset\s*=\s*["']?([\w.:-]+)/i
       # Bytes scanned for a meta charset hint (HTML spec looks at the first 1024).
@@ -31,23 +33,20 @@ module Html2rss
       # @param status [Integer, nil] the HTTP status code when available
       # @param transport_meta [Hash] allowlisted upstream telemetry (frozen when present)
       # @param captured_responses [Array<Hash>] JSON XHR/fetch bodies captured during browser scrapes
-      # rubocop:disable-next Metrics/MethodLength, Metrics/ParameterLists -- transport + capture fields stay co-located
+      # @param cookies [Hash{String => String}] cookies earned during the scrape (session credentials; never log)
+      # @param cache_age [Integer, nil] seconds since fetch when served from {ResponseCache}; nil when fetched live
+      # rubocop:disable-next Metrics/ParameterLists -- transport + capture fields stay co-located
       def initialize(body:, url:, headers: {}, status: nil, transport_meta: EMPTY_TRANSPORT_META,
-                     captured_responses: EMPTY_CAPTURED_RESPONSES)
+                     captured_responses: EMPTY_CAPTURED_RESPONSES, cookies: EMPTY_COOKIES, cache_age: nil)
         @body = body
+        @cache_age = cache_age
 
-        headers = (headers || {}).transform_keys(&:to_s)
-        HashUtil.assert_string_keys!(headers, context: 'response headers', deep: false)
-
-        @headers = headers
+        @headers = (headers || {}).transform_keys(&:to_s)
         @status = status
         @url = url
-        @transport_meta = transport_meta.nil? || transport_meta.empty? ? EMPTY_TRANSPORT_META : transport_meta.freeze
-        @captured_responses = if captured_responses.nil? || captured_responses.empty?
-                                EMPTY_CAPTURED_RESPONSES
-                              else
-                                captured_responses.freeze
-                              end
+        @transport_meta = freeze_or(transport_meta, EMPTY_TRANSPORT_META)
+        @cookies = freeze_or(cookies, EMPTY_COOKIES)
+        @captured_responses = freeze_or(captured_responses, EMPTY_CAPTURED_RESPONSES)
       end
 
       # @return [String] the raw body of the response
@@ -68,6 +67,19 @@ module Html2rss
       # @return [Array<Hash>] captured JSON XHR/fetch responses (empty when unsupported)
       attr_reader :captured_responses
 
+      # @return [Hash{String => String}] cookies earned during the scrape (empty when unsupported)
+      attr_reader :cookies
+
+      # @return [Integer, nil] seconds since fetch when served from {ResponseCache}; nil when fetched live
+      attr_reader :cache_age
+
+      # @param cache_age [Integer, nil] cache age of the copy (defaults to this response's)
+      # @return [Response] an independent copy that shares no parsed document with this one
+      def replica(cache_age: @cache_age)
+        self.class.new(body: body.dup, url:, headers:, status:, transport_meta:, captured_responses:, cookies:,
+                       cache_age:)
+      end
+
       # @return [String] normalized content type header value
       def content_type = header('content-type').to_s
 
@@ -82,7 +94,7 @@ module Html2rss
         content_type.include?('text/html') || (!json_response? && !feed_response? && html_looking_body?)
       end
 
-      # @return [Boolean] whether response content is RSS/Atom (header or sniffed body)
+      # @return [Boolean] whether response content is RSS/Atom/RDF (header or sniffed body)
       def feed_response?
         return @feed_response if defined?(@feed_response)
 
@@ -104,6 +116,8 @@ module Html2rss
 
       private
 
+      def freeze_or(value, default) = value.nil? || value.empty? ? default : value.freeze
+
       # @param name [String] canonical header name
       # @return [Object, nil] header value when present
       def header(name)
@@ -158,14 +172,14 @@ module Html2rss
 
       def feed_content_type?
         ct = content_type.downcase
-        return false if ct.empty? || ct.include?('html') || ct.include?('json')
+        return false if ct.include?('html') || ct.include?('json')
 
         # Substring match against Content-Type (not Array#intersect? on a String).
         FEED_CT_MARKERS.any? { |marker| ct.include?(marker) }
       end
 
       def feed_looking_body?
-        snippet = body.to_s[0, 800].downcase
+        snippet = body.to_s.b[0, 800].downcase
         FEED_BODY_MARKERS.any? { |marker| snippet.include?(marker) }
       end
     end

data/lib/html2rss/request_service/response_cache.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestService
+    ##
+    # Bounded, thread-safe TTL cache for initial page fetches; install with
+    # {RequestService.response_cache=}. Only successful (2xx) responses are stored, so a
+    # block page or rate limit is never replayed. Hits are returned tagged with their
+    # {Response#cache_age}; memory is capped in body bytes as well as entries.
+    class ResponseCache
+      # Default entry ceiling; oldest entries are evicted first.
+      DEFAULT_MAX_ENTRIES = 32
+      # Default body-byte budget across all entries.
+      DEFAULT_MAX_BYTES = 32 * 1024 * 1024
+      # Monotonic seconds source, immune to wall-clock jumps.
+      MONOTONIC_CLOCK = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
+      Entry = Data.define(:stored_at, :response, :bytes)
+      private_constant :Entry
+
+      # @return [Integer] seconds an entry stays fresh
+      attr_reader :ttl
+
+      ##
+      # @param ttl [Integer] seconds an entry stays fresh
+      # @param max_entries [Integer] maximum stored entries
+      # @param max_bytes [Integer] maximum summed body bytes
+      # @param clock [#call] monotonic seconds source (injectable for specs)
+      def initialize(ttl:, max_entries: DEFAULT_MAX_ENTRIES, max_bytes: DEFAULT_MAX_BYTES, clock: MONOTONIC_CLOCK)
+        @ttl = Option.integer(:ttl, ttl, min: 1)
+        @max_entries = Option.integer(:max_entries, max_entries, min: 1)
+        @max_bytes = Option.integer(:max_bytes, max_bytes, min: 1)
+        @clock = clock
+        @entries = {}
+        @bytes = 0
+        @mutex = Mutex.new
+        @flight_done = ConditionVariable.new
+        @inflight = Set.new
+      end
+
+      ##
+      # Concurrent callers missing the same key share one fetch: the first builds the response while
+      # the others wait, then each is served its own replica. A raising or uncacheable fetch is not
+      # stored; waiters then retry in turn so none is left blocked.
+      # @param key [Object] hashable cache key
+      # @yieldreturn [Response] the freshly fetched response, built only on a miss
+      # @return [Response] a cache-aged replica on a hit, otherwise the freshly built response
+      def fetch(key, &)
+        cached = claim(key)
+        return cached if cached
+
+        lead(key, &)
+      end
+
+      private
+
+      # Returns a fresh replica, or nil after registering the caller as the key's fetcher.
+      def claim(key)
+        @mutex.synchronize do
+          loop do
+            hit = lookup(key)
+            break hit if hit
+            break @inflight.add(key) && nil unless @inflight.include?(key)
+
+            @flight_done.wait(@mutex)
+          end
+        end
+      end
+
+      def lead(key)
+        yield.tap { |response| store(key, response) if cacheable?(response) }
+      ensure
+        @mutex.synchronize do
+          @inflight.delete(key)
+          @flight_done.broadcast
+        end
+      end
+
+      # Callers must hold @mutex.
+      def lookup(key)
+        entry = @entries[key]
+        age = entry && (@clock.call - entry.stored_at)
+        entry.response.replica(cache_age: age.floor) if age && age < @ttl
+      end
+
+      def store(key, response)
+        @mutex.synchronize do
+          evict(key)
+          bytes = response.body.bytesize
+          @entries[key] = Entry.new(stored_at: @clock.call, response:, bytes:)
+          @bytes += bytes
+          evict_oldest while @entries.size > @max_entries || @bytes > @max_bytes
+        end
+      end
+
+      def evict(key)
+        @bytes -= @entries.delete(key)&.bytes.to_i
+      end
+
+      def evict_oldest
+        _key, entry = @entries.shift
+        @bytes -= entry.bytes
+      end
+
+      def cacheable?(response)
+        (200..299).cover?(response.status) && response.body.bytesize <= @max_bytes
+      end
+    end
+  end
+end

data/lib/html2rss/request_service/response_caching.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+require 'digest'
+
+module Html2rss
+  class RequestService
+    ##
+    # Opt-in reuse of initial page fetches, e.g. while a person iterates on selectors for one page.
+    # Extended onto {RequestService}; the cache is consulted only for +relation == :initial+ requests,
+    # follow-ups always hit the network.
+    #
+    # The key is a SHA256 digest of the URL, strategy, headers, request options and the policy fields
+    # that shape a response, so credentials never reach the cache and a response fetched under a
+    # permissive {Policy} is never served to a stricter one. Cookie-store state is not part of the key.
+    module ResponseCaching
+      # Mutex guarding the process-wide response cache reference.
+      RESPONSE_CACHE_MUTEX = Mutex.new
+      # Fiber-storage key for the active {#observe_cache_hits} collector.
+      CACHE_HITS_KEY = :html2rss_cache_hits
+      # Fiber-storage key for the cache installed by {#with_response_cache}.
+      SCOPED_CACHE_KEY = :html2rss_scoped_response_cache
+      private_constant :RESPONSE_CACHE_MUTEX, :CACHE_HITS_KEY, :SCOPED_CACHE_KEY
+
+      # @param cache [#fetch, nil] receives +fetch(key) { response }+ and returns the cached or freshly
+      #   built {Response} (tagged with {Response#cache_age} on a hit), e.g. {ResponseCache};
+      #   +nil+ (the default) disables caching. A raising block must not be cached.
+      # @return [void]
+      def response_cache=(cache)
+        RESPONSE_CACHE_MUTEX.synchronize { @response_cache = cache }
+      end
+
+      # @return [#fetch, nil] the scoped cache from {#with_response_cache} if one is active, otherwise
+      #   the globally configured initial-response cache, if any
+      def response_cache
+        scoped = Fiber[SCOPED_CACHE_KEY]
+        return scoped.first if scoped
+
+        RESPONSE_CACHE_MUTEX.synchronize { @response_cache }
+      end
+
+      # Runs the block with +cache+ as the initial-response cache for the current fiber (and fibers or
+      # threads it spawns), taking precedence over {#response_cache=}. The previous scope is restored
+      # when the block exits, even by raising.
+      #
+      # @param cache [#fetch, nil] cache for the block; +nil+ disables caching inside it
+      # @yieldreturn [Object] the block's result
+      # @return [Object] the block's result
+      def with_response_cache(cache)
+        previous = Fiber[SCOPED_CACHE_KEY]
+        Fiber[SCOPED_CACHE_KEY] = [cache]
+        yield
+      ensure
+        Fiber[SCOPED_CACHE_KEY] = previous
+      end
+
+      # Runs the block and reports the cache ages of every initial fetch it served from {#response_cache},
+      # including fetches on threads the block spawns (fiber storage is inherited by reference).
+      #
+      # @yieldreturn [Object] the block's result
+      # @return [Array(Object, Array<Integer>)] the block's result and the served cache ages in seconds
+      def observe_cache_hits
+        hits = Fiber[CACHE_HITS_KEY] = Queue.new
+        result = yield
+        [result, Array.new(hits.size) { hits.pop }]
+      ensure
+        Fiber[CACHE_HITS_KEY] = nil
+      end
+
+      private
+
+      ##
+      # Runs +fetch+ directly, or through {#response_cache} for initial requests.
+      # Callers always receive their own {Response} so parsed documents are never shared.
+      # @param ctx [Context] the request context
+      # @param strategy [Symbol] the resolved strategy name
+      # @yieldreturn [Response] the freshly fetched response
+      # @return [Response]
+      def fetch_initial_through_cache(ctx, strategy, &)
+        cache = response_cache
+        return yield unless cache && ctx.relation == :initial
+
+        cache.fetch(response_cache_key(ctx, strategy), &).replica.tap { |response| record_cache_hit(response) }
+      end
+
+      def record_cache_hit(response)
+        Fiber[CACHE_HITS_KEY]&.push(response.cache_age) if response.cache_age
+      end
+
+      def response_cache_key(ctx, strategy)
+        parts = [ctx.url.to_s, strategy, ctx.headers.sort, ctx.request, ctx.policy.cache_key]
+        Digest::SHA256.hexdigest(parts.inspect)
+      end
+    end
+  end
+end

data/lib/html2rss/request_service/strategy.rb

@@ -18,12 +18,13 @@ module Html2rss
       # Executes the request through the shared preflight/postflight path.
       #
       # @return [Response] the response from the strategy
-      # @raise [NotImplementedError] if the subclass does not implement `#fetch`
       def execute
         check_timeout!
         response = perform_execute
         postflight!(response)
         response
+      rescue Html2rss::ContractError
+        raise
       rescue StandardError => error
         handle_error(error)
       end
@@ -42,15 +43,6 @@ module Html2rss
         fetch
       end
 
-      ##
-      # Adapter hook: perform the transport-specific fetch after preflight.
-      #
-      # @return [Response] normalized response
-      # @raise [NotImplementedError] when a subclass omits the hook
-      def fetch
-        raise NotImplementedError, 'Subclass must implement #fetch'
-      end
-
       ##
       # Consumes a request slot and validates the destination before transport work.
       #
@@ -60,7 +52,9 @@ module Html2rss
         return if skip_preflight?
 
         ctx.budget.consume! if consume_budget
-        ctx.policy.validate_request!(url: ctx.url, origin_url: ctx.origin_url, relation: ctx.relation)
+        ctx.policy.validate_request!(
+          url: ctx.url, origin_url: ctx.origin_url, relation: ctx.relation, advertisement: ctx.advertisement
+        )
       end
 
       ##