html2rss 0.27.2 → 0.28.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '09b4a391aa87101ab6612c07c0c434b9ebc6198fc3bed8c14c2f8b90bd61bc6a'
-  data.tar.gz: d7fac305311719a6b131be4e89fe78919aef1b45c763a22c471b24583b41d7fa
+  metadata.gz: 38cdc153241c05684ee40477e4df06ca2c88f013c971387ddd0ee6d8163c178a
+  data.tar.gz: eda1c02f0199568f8f71d24b22480b5eaa9a83c7e48c23678e9695e98e2dab8f
 SHA512:
-  metadata.gz: d5fb2cdfc2b95e64de5ac42ffbe1479c15fe63b01f4b922d721d3099ffdd418c9293a1672217838612d5cf4f7bbc2e153cf83ffbcc85e8f30bf8a1a17bb20a79
-  data.tar.gz: a9f51aa30dfdf8e3e222ed1a49216e857cd056f473408273ad3781e862d743f657a67473fe691423e1fb1252fbe2935ee6eb1cef009d3b2662d03b66f7bdb90c
+  metadata.gz: 5e063256aa063027d53986e38703cc0065d12e7aaee45cdfa7724a724f599a0f645e836a0573c9633865d007532bb82c3c2a6632984c5385dcc6fafd5ed0409d
+  data.tar.gz: 146ab91ee7925fc678ecdbe16895bd830bf9d7620bfc5239ccf12e109ef0913142da27f77c1be34f839cce350f51552ebcf163a5c92448c0b836b6a9fd8d7e4f

data/lib/html2rss/auto_source/README.md

@@ -10,9 +10,10 @@ Entry: `FeedPipeline` → `AutoSource#articles` → `Scraper.build_instance` →
 
 ## Live flow
 
-1. **Request** — `RequestSession` returns a `Response` with `body` (String) and `parsed_body` (Nokogiri HTML).
+1. **Request** — `RequestSession` returns a `Response` with `body` (String) and `parsed_body` (Nokogiri HTML). Direct syndication responses skip HTML scrapers and parse via `Syndication::Parser`.
 2. **Scraper tiers** — Enabled scrapers that claim the page (shallow `articles?` or instance `extractable?`) run in `Scraper::SCRAPER_TIERS` order. Merge within a tier, then stop when enough articles survive Cleanup:
 
+   0. Native feed: NativeFeed (head alternates + path discovery → RSS/Atom parse)
    1. In-page structured: Schema, Microdata, Microformats2, JsonState, XhrArticles
    2. Follow-up IO: WordPress API, Sitemap, MetaOembed
    3. Heuristic: SemanticHtml
@@ -25,6 +26,8 @@ Entry: `FeedPipeline` → `AutoSource#articles` → `Scraper.build_instance` →
 
 5. **Cleanup** — Merge, dedupe, hard-exclude non-article destinations (via `PathClassifier` facts), drop junk titles, and trim to `limit`. Html is skipped when earlier tiers already admitted clean items.
 
+6. **Entry URL resolution** (pipeline, not this class) — when AutoFallback sees a weak homepage extract, `FeedResolution` may rewrite the scrape URL to a listing/feed before escalating strategies. See {Html2rss::FeedPipeline} and {Html2rss::FeedResolution}.
+
 Segmenter strategies: `:semantic` (leaf containers + primary link), `:list` (repeated tag paths), `:cluster` (class/structure grids for anchorless cards). Scoring ranks and demotes; `LinkDestination::NoisePolicy` owns content-anchor eligibility. Cleanup owns feed-item admission.
 
 ## Nokogiri vs SST boundaries

data/lib/html2rss/auto_source/scraper/native_feed.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    module Scraper
+      ##
+      # Promotes a page's native RSS/Atom feed via {Syndication::Discovery} + {Syndication::Parser}.
+      class NativeFeed
+        include Enumerable
+
+        # @return [Symbol] scraper config key
+        def self.options_key = :native_feed
+
+        ##
+        # @param _opts [Hash] unused options
+        # @return [Integer] follow-up request slots (discovery probes + feed fetch share the budget)
+        def self.request_slots(_opts = {})
+          1
+        end
+
+        ##
+        # Shallow claim: head advertises a syndication alternate, or body looks feed-capable
+        # enough that path discovery is worth a follow-up slot.
+        #
+        # @param parsed_body [Nokogiri::HTML::Document, nil]
+        # @return [Boolean]
+        def self.articles?(parsed_body)
+          return false unless parsed_body.is_a?(Nokogiri::HTML::Document)
+
+          return true if ::Html2rss::Html::FeedLink.from_document(parsed_body).any?
+
+          parsed_body.css('head link[rel~="alternate"][href]').any? do |node|
+            href = node['href'].to_s
+            type = node['type'].to_s.downcase
+            type.include?('rss') || type.include?('atom') || href.match?(/rss|atom|feed|\.xml/i)
+          end
+        end
+
+        ##
+        # @param parsed_body [Nokogiri::HTML::Document]
+        # @param url [String, Html2rss::Url]
+        # @param request_session [Html2rss::RequestSession, nil]
+        # @param _opts [Hash]
+        # @option _opts [Object] :_reserved reserved for future scraper-specific options
+        def initialize(parsed_body, url:, request_session: nil, **_opts)
+          @parsed_body = parsed_body
+          @url = Html2rss::Url.from_absolute(url)
+          @request_session = request_session
+        end
+
+        ##
+        # @yieldparam article [Hash{Symbol => Object}]
+        # @return [Enumerator, void]
+        def each(&)
+          return enum_for(:each) unless block_given?
+
+          articles = fetch_articles
+          if articles.empty?
+            Log.info("#{self.class}: host=#{url.host} item_count=0 fallback=true")
+            return
+          end
+
+          Log.info("#{self.class}: host=#{url.host} item_count=#{articles.size} fallback=false")
+          articles.each(&)
+        end
+
+        private
+
+        attr_reader :parsed_body, :url, :request_session
+
+        def fetch_articles # rubocop:disable Metrics/MethodLength -- discovery + parse path
+          return [] unless request_session
+
+          response = Syndication::Discovery.best_feed_response(
+            page_url: url,
+            request_session:,
+            parsed_body:,
+            max_probes: self.class.request_slots
+          )
+          return [] unless response
+
+          Syndication::Parser.parse_response(response)
+        rescue Html2rss::Error, ArgumentError => error
+          Log.warn("#{self.class}: host=#{url.host} failed (#{error.class}: #{error.message})")
+          []
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/scraper.rb

@@ -23,7 +23,9 @@ module Html2rss
 
       # Extraction tiers: merge within a tier, then stop when AutoSource has enough articles.
       # Heuristic scrapers are separate tiers so SemanticHtml can satisfy before Html runs.
+      # NativeFeed is tier 0 so syndication wins before structured HTML scrapers.
       SCRAPER_TIERS = [
+        [NativeFeed].freeze,
         [Schema, Microdata, Microformats2, JsonState, XhrArticles].freeze,
         [WordpressApi, Sitemap, MetaOembed].freeze,
         [SemanticHtml].freeze,
@@ -36,7 +38,7 @@ module Html2rss
       # Heuristic scrapers that share one memoized SST::Document per page.
       HEURISTIC_SCRAPERS = [SemanticHtml, Html].freeze
       # Scrapers that accept a shared follow-up +request_session+.
-      REQUEST_SESSION_SCRAPERS = [WordpressApi, Sitemap, MetaOembed].freeze
+      REQUEST_SESSION_SCRAPERS = [NativeFeed, WordpressApi, Sitemap, MetaOembed].freeze
       # Scrapers that consume browser-captured XHR/fetch JSON bodies.
       CAPTURED_RESPONSE_SCRAPERS = [XhrArticles].freeze
 
@@ -83,6 +85,7 @@ module Html2rss
       # Returns an array of scraper classes that claim to find articles in the parsed body.
       # @param parsed_body [Nokogiri::HTML::Document] The parsed HTML document.
       # @param opts [Hash] The options hash.
+      # @option opts [Hash] :native_feed scraper toggle and configuration
       # @option opts [Hash] :wordpress_api scraper toggle and configuration
       # @option opts [Hash] :schema scraper toggle and configuration
       # @option opts [Hash] :microdata scraper toggle and configuration

data/lib/html2rss/auto_source.rb

@@ -23,7 +23,14 @@ module Html2rss
     # Default auto-source configuration shipped for scraper and cleanup behavior.
     DEFAULT_CONFIG = {
       limit: DEFAULT_LIMIT,
+      entry_resolution: {
+        enabled: true,
+        max_probes: 5
+      }.freeze,
       scraper: {
+        native_feed: {
+          enabled: true
+        },
         wordpress_api: {
           enabled: true
         },

data/lib/html2rss/config/auto_source_contract.rb

@@ -7,8 +7,15 @@ module Html2rss
     # Runtime source of truth for validating auto-source config values.
     AutoSourceContract = Dry::Schema.Params do # rubocop:disable Metrics/BlockLength
       optional(:limit).filled(:integer, gt?: 0)
+      optional(:entry_resolution).hash do
+        optional(:enabled).filled(:bool)
+        optional(:max_probes).filled(:integer, gt?: 0)
+      end
 
       optional(:scraper).hash do # rubocop:disable Metrics/BlockLength
+        optional(:native_feed).hash do
+          optional(:enabled).filled(:bool)
+        end
         optional(:wordpress_api).hash do
           optional(:enabled).filled(:bool)
         end

data/lib/html2rss/config.rb

@@ -264,8 +264,17 @@ module Html2rss
     def headers = config[:headers]
     # @return [Hash{Symbol => Object}] channel configuration
     def channel = config[:channel]
-    # @return [String] source channel URL
+    ##
+    # Source channel URL (also the default scrape URL).
+    #
+    # @return [String]
     def url = config.dig(:channel, :url)
+
+    ##
+    # @deprecated Use {ScrapeTarget} for entry vs effective URLs after resolution.
+    # @return [String]
+    def scrape_url = url
+
     # @return [String, nil] configured channel time zone
     def time_zone = config.dig(:channel, :time_zone)
 

data/lib/html2rss/feed_pipeline/README.md

@@ -11,6 +11,8 @@ Use `:auto` when you want Faraday first and a browser-backed hop only if that fe
 1. **Faraday** — plain HTTP (faster, cheaper).
 2. **Botasaurus** — attempted when Faraday raises a fallback-eligible error (for example `BlockedSurfaceDetected` or timeout) or extracts zero feed items.
 
+Before escalating Faraday → Botasaurus on a **weak** auto-source extract (empty / below floor / high-entropy / app-shell / unsupported — not blocked), `FeedResolution` may probe up to five same-origin listing or native-feed candidates and rewrite the effective scrape URL via {ScrapeTarget}. Botasaurus then uses the resolved URL. Direct RSS/Atom entry URLs are parsed via `Syndication::Parser` without HTML AutoSource.
+
 There is no Browserless / Puppeteer-in-gem tier. Pin `botasaurus` when you want browser rendering without Faraday first. Botasaurus needs `BOTASAURUS_SCRAPER_URL`.
 
 ## Surfaces

data/lib/html2rss/feed_pipeline/auto_fallback.rb

@@ -24,17 +24,33 @@ module Html2rss
       ##
       # Mutable run state for one auto-fallback chain: attempts plus selected result.
       class AttemptState
-        attr_reader :attempts, :result, :last_response
+        attr_reader :attempts, :result, :last_response, :entry_resolution
 
         def initialize
           @attempts = []
           @result = nil
           @last_response = nil
+          @entry_resolution = nil
+          @resolution_tried = false
         end
 
         # @return [Boolean] true when a strategy already yielded items
         def succeeded? = !result.nil?
 
+        # @return [Boolean] whether entry resolution already ran for this chain
+        def resolution_tried? = @resolution_tried
+
+        # @return [void]
+        def mark_resolution_tried!
+          @resolution_tried = true
+        end
+
+        # @param result [Html2rss::FeedResolution::Result]
+        # @return [void]
+        def remember_entry_resolution(result)
+          @entry_resolution = FeedResolution::Diag.from_result(result)
+        end
+
         # @param strategy [Symbol] strategy that raised
         # @param error [Exception] caught error
         # @return [void]
@@ -64,10 +80,11 @@ module Html2rss
         # @param selected_strategy [Symbol] concrete strategy that produced items
         # @param attempt_count [Integer] number of attempts recorded for this chain
         # @param admission_drops [Hash{String => Integer}] Cleanup drop tallies
+        # @param scrape_target [Html2rss::ScrapeTarget]
         # @return [void]
-        # rubocop:disable Metrics/ParameterLists -- PipelineOutcome kwargs stay co-located
+        # rubocop:disable Metrics/ParameterLists, Metrics/MethodLength -- PipelineOutcome kwargs stay co-located
         def succeed!(response:, articles:, dedup_dropped:, selected_strategy:, attempt_count:,
-                     admission_drops: {})
+                     scrape_target:, admission_drops: {})
           @result = PipelineOutcome.new(
             response:,
             articles:,
@@ -75,10 +92,12 @@ module Html2rss
             selected_strategy:,
             attempt_count:,
             strategy_attempts: attempts,
-            admission_drops:
+            admission_drops:,
+            scrape_target:,
+            entry_resolution:
           )
         end
-        # rubocop:enable Metrics/ParameterLists
+        # rubocop:enable Metrics/ParameterLists, Metrics/MethodLength
       end
 
       ##
@@ -94,6 +113,7 @@ module Html2rss
         @pipeline = pipeline
         @config = config
         @resources = resources
+        @scrape_target = ScrapeTarget.from_config(config)
       end
 
       ##
@@ -107,7 +127,7 @@ module Html2rss
 
       private
 
-      attr_reader :strategies, :budget, :pipeline, :config, :resources
+      attr_reader :strategies, :budget, :pipeline, :config, :resources, :scrape_target
 
       def run_attempts
         AttemptState.new.tap do |state|
@@ -119,7 +139,9 @@ module Html2rss
       end
 
       def attempt(strategy:, next_strategy:, state:)
-        request_session = pipeline.request_session_for(config, strategy:, resources:)
+        request_session = pipeline.request_session_for(
+          config, strategy:, resources:, scrape_url: scrape_target.effective_url
+        )
         response = fetch_response(request_session:, strategy:, next_strategy:, state:)
         return unless response
 
@@ -139,7 +161,8 @@ module Html2rss
         nil
       end
 
-      def process_response(response:, strategy:, next_strategy:, request_session:, state:) # rubocop:disable Metrics/MethodLength -- extract + success path
+      # rubocop:disable Metrics/MethodLength, Metrics/AbcSize -- extract + resolution delegate
+      def process_response(response:, strategy:, next_strategy:, request_session:, state:)
         articles, dedup_dropped, admission_drops = articles_for(response:, request_session:)
         state.remember_response(response)
         items_count = articles.size
@@ -147,6 +170,15 @@ module Html2rss
         Log.debug("#{self.class}: strategy=#{strategy} items=#{items_count} " \
                   "host=#{response.url.host} elapsed=#{format('%.3f', budget.elapsed_seconds)}s " \
                   "budget_remaining=#{budget_remaining_label}")
+
+        if (outcome = FeedResolution.try_apply!(
+          pipeline:, config:, response:, session: request_session, strategy:, resources:,
+          articles:, scrape_target:, state:, budget:
+        ))
+          @scrape_target = outcome.scrape_target
+          return
+        end
+
         if items_count.positive?
           return record_success(response:, strategy:, articles:, dedup_dropped:, admission_drops:,
                                 state:)
@@ -154,6 +186,7 @@ module Html2rss
 
         log_info_fallback_zero_items(strategy:, next_strategy:, response:) if next_strategy
       end
+      # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
 
       def articles_for(response:, request_session:)
         pipeline.deduplicated_articles(config:, response:, request_session:)
@@ -162,8 +195,11 @@ module Html2rss
       # rubocop:disable Metrics/ParameterLists -- success kwargs match PipelineOutcome
       def record_success(response:, strategy:, articles:, dedup_dropped:, admission_drops:, state:)
         attempt_count = state.attempts.size
-        state.succeed!(response:, articles:, dedup_dropped:, selected_strategy: strategy,
-                       attempt_count:, admission_drops:)
+        state.succeed!(
+          response:, articles:, dedup_dropped:, selected_strategy: strategy,
+          attempt_count:, admission_drops:,
+          scrape_target:
+        )
         return unless attempt_count > 1
 
         Log.info("#{self.class}: auto selected strategy=#{strategy} after attempts=#{attempt_count} " \
@@ -173,12 +209,16 @@ module Html2rss
       # rubocop:enable Metrics/ParameterLists
 
       def finalize_failure(attempts:, response:)
-        surface_category = response && AutoSource::Scraper.classify_no_scraper_surface(
-          response.parsed_body, body: response.body
-        )
+        surface_category = surface_category_for(response)
         raise NoFeedItemsExtracted.new(attempts:, surface_category:)
       end
 
+      def surface_category_for(response)
+        return unless response
+
+        PageRecon.surface_category_for(response:, url: response.url)
+      end
+
       # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       def log_fallback_error(strategy:, next_strategy:, error:, request_session:)
         host = request_session.url.host

data/lib/html2rss/feed_pipeline/runtime_policy.rb

@@ -60,6 +60,7 @@ module Html2rss
           1 +
             RequestSession::Pager.request_slots_for(config.selectors&.dig(:items, :pagination)) +
             AutoSource.request_slots_for(config.auto_source) +
+            FeedResolution.request_slots_for(config.auto_source) +
             StrategyPlan.resolve(config.strategy).request_slots
         end
       end

data/lib/html2rss/feed_pipeline.rb

@@ -12,7 +12,7 @@ module Html2rss
     # strategy_attempts: auto attempt hashes (with optional transport_meta); empty outside :auto.
     PipelineOutcome = Data.define(
       :response, :articles, :dedup_dropped, :selected_strategy, :attempt_count, :strategy_attempts,
-      :admission_drops
+      :admission_drops, :scrape_target, :entry_resolution
     )
 
     ##
@@ -46,7 +46,9 @@ module Html2rss
         selected_strategy: outcome.selected_strategy,
         attempt_count: outcome.attempt_count,
         strategy_attempts: outcome.strategy_attempts,
-        admission_drops: outcome.admission_drops
+        admission_drops: outcome.admission_drops,
+        scrape_target: outcome.scrape_target,
+        entry_resolution: outcome.entry_resolution
       )
       FeedResult.new(channel:, articles: outcome.articles, status:, stylesheets: config.stylesheets)
     end
@@ -56,13 +58,15 @@ module Html2rss
     # @param config [Html2rss::Config]
     # @param strategy [Symbol]
     # @param resources [Html2rss::FeedPipeline::RuntimePolicy::Resources]
+    # @param scrape_url [String, nil] effective fetch URL override for resolved targets
     # @return [Html2rss::RequestSession]
-    def request_session_for(config, strategy:, resources:)
+    def request_session_for(config, strategy:, resources:, scrape_url: nil)
       RequestSession.build(
         config:,
         strategy:,
         budget: resources.budget,
-        policy: resources.policy
+        policy: resources.policy,
+        scrape_url:
       )
     end
 
@@ -99,19 +103,22 @@ module Html2rss
 
       PipelineOutcome.new(
         response:, articles:, dedup_dropped:, selected_strategy: nil, attempt_count: 0,
-        strategy_attempts: [], admission_drops:
+        strategy_attempts: [], admission_drops:,
+        scrape_target: ScrapeTarget.from_config(config), entry_resolution: nil
       )
     end
 
     def raise_empty_auto_source!(strategy:, response:)
       raise NoFeedItemsExtracted.new(
         attempts: [{ strategy:, items_count: 0, error_class: nil }],
-        surface_category: AutoSource::Scraper.classify_no_scraper_surface(
-          response.parsed_body, body: response.body
-        )
+        surface_category: empty_auto_source_surface(response)
       )
     end
 
+    def empty_auto_source_surface(response)
+      PageRecon.surface_category_for(response:, url: response.url)
+    end
+
     def run_auto_pipeline(config, resources:)
       AutoFallback.new(
         strategies: AutoFallback::CHAIN,
@@ -131,6 +138,7 @@ module Html2rss
     # rubocop:disable Metrics/MethodLength
     def selector_articles(config:, response:, request_session:)
       return [] unless (selectors = config.selectors)
+      return [] if response.feed_response?
 
       page_responses = request_session.page_responses(
         response,
@@ -151,9 +159,17 @@ module Html2rss
     # @return [Array(Array<Html2rss::Article>, Hash{String => Integer})]
     def auto_source_articles(config:, response:, request_session:)
       return [[], {}] unless (auto_source = config.auto_source)
+      return syndication_articles(response) if response.feed_response?
 
       source = AutoSource.new(response, auto_source, request_session:)
       [source.articles, source.admission_drops]
     end
+
+    def syndication_articles(response)
+      articles = Syndication::Parser.parse_response(response).map do |hash|
+        Article.new(**hash, scraper: AutoSource::Scraper::NativeFeed)
+      end
+      [articles, {}]
+    end
   end
 end

data/lib/html2rss/feed_resolution/README.md

@@ -0,0 +1,52 @@
+# FeedResolution
+
+Backend-only entry URL tournament: when the pasted URL is a weak homepage/hub
+(or NativeFeed-majority extract), probe same-origin listing/feed candidates and
+sticky-rewrite the scrape URL only when the retry extract yields items
+(`try_apply!` `:succeeded`) before AutoFallback escalates strategies.
+
+Goal: best **article listing** for AutoSource. A native feed may still win the
+tournament on item count (feed-as-means) — intentional.
+
+## Ownership
+
+| Concern | Owner |
+| --- | --- |
+| When to resolve (`articles:` required; floor / weak / NativeFeed ≥50%) | `FeedResolution::Policy` |
+| Candidate mix (1 feed + up to 4 listing; taxonomy nav; segment first-wins) | `FeedResolution::CandidateGenerator` |
+| Listing path lexicon (`LISTING_PATHS`) | `Syndication::CandidateCatalog` (consumed, not owned here) |
+| Cheap probe + score + winner pick | `FeedResolution::Probe` + `FeedResolution::Scorer` |
+| Typed entry_resolution options | `FeedResolution::Options` |
+| Wire-safe resolution diag (`applied` = tournament win, not sticky URL) | `FeedResolution::Diag` |
+| Public `call` → `Result`; retry orchestration (sticky `ScrapeTarget` only on `:succeeded`) | `FeedResolution` (`try_apply!`) |
+| Page surface for policy/scoring | `PageRecon::Assessment` + `Html2rss::SurfaceCategory` |
+| Native feed discover/parse | `Syndication` (not this module) |
+
+### CandidateGenerator mix
+
+- **Feed slot:** first `Syndication::Discovery.candidate_urls` only (alternates precede path guesses).
+- **Listing fill** (cap = `max - 1` when `max > 1`; `max == 1` → feed only): nav (`taxonomy_path +3`, `content_path +1`) → segment first-wins (`:list` → `:cluster` → `:semantic`, ≥2 same-origin primary links) → `LISTING_PATHS`.
+- Same-page / same-registrable-domain filters; feed never pads into listing slots.
+
+### Policy gates
+
+Resolve when eligible config **and** not blocked **and** any of:
+
+- `articles.size < ARTICLE_FLOOR` (3), or
+- surface `weak?`, or
+- ≥50% of articles have `scraper == AutoSource::Scraper::NativeFeed` (`native * 2 >= size`; empty → false).
+
+`FeedResolution.call` / `Runner` take **`articles:`** (Array) — not `articles_count:`.
+
+## Non-goals
+
+- Scorer weight overhaul / ranked multi-feed bake-off
+- Cross-origin probes
+- JSON Feed ingest; Sitemap/WP claim-gates
+- Create UI / Site Explorer
+- Changing `PageRecon` `:list`-only `discover_segments` (CandidateGenerator owns its own first-wins)
+- Syndication parse inside `FeedBuilder`
+
+## Config
+
+`auto_source.entry_resolution: { enabled: true, max_probes: 5 }`

data/lib/html2rss/feed_resolution/candidate_generator.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module FeedResolution
+    ##
+    # Builds same-origin entry-resolution candidates: one feed slot plus listing seeds.
+    #
+    # Mix is 1 feed + up to +listing_cap+ listing URLs (nav → segment first-wins →
+    # {Syndication::CandidateCatalog::LISTING_PATHS}). Feed never pads into listing slots.
+    class CandidateGenerator
+      # Max candidates returned to the probe stage.
+      DEFAULT_MAX = 5
+      # Nav/header/footer anchor sources for DestinationFacts ranking.
+      NAV_SELECTORS = 'header a[href], nav a[href], footer a[href]'
+      # Segment strategies tried first-wins (≥2 same-origin primary links).
+      SEGMENT_STRATEGIES = %i[list cluster semantic].freeze
+
+      ##
+      # @param entry_url [String, Html2rss::Url]
+      # @param response [Html2rss::RequestService::Response]
+      # @param max [Integer]
+      # @return [Array<Html2rss::Url>]
+      def self.call(entry_url:, response:, max: DEFAULT_MAX)
+        new(entry_url:, response:, max:).call
+      end
+
+      ##
+      # @param entry_url [String, Html2rss::Url]
+      # @param response [Html2rss::RequestService::Response]
+      # @param max [Integer]
+      def initialize(entry_url:, response:, max: DEFAULT_MAX)
+        @entry_url = Html2rss::Url.from_absolute(entry_url)
+        @response = response
+        @max = max
+      end
+
+      ##
+      # @return [Array<Html2rss::Url>]
+      def call
+        [feed_slot, *listing_urls.first(listing_cap)].compact.uniq
+      end
+
+      private
+
+      attr_reader :entry_url, :response, :max
+
+      def listing_cap = max > 1 ? max - 1 : 0
+
+      def feed_slot
+        return unless parsed_html
+
+        url = Syndication::Discovery.candidate_urls(page_url: entry_url, parsed_body: parsed_html).first
+        return unless url
+        return unless eligible_url?(url)
+
+        url
+      end
+
+      def listing_urls
+        (nav_candidates + segment_seed_urls + listing_path_candidates)
+          .uniq
+          .select { eligible_url?(_1) }
+      end
+
+      def nav_candidates
+        return [] unless parsed_html
+
+        parsed_html.css(NAV_SELECTORS)
+                   .filter_map { |anchor| ranked_anchor_url(anchor) }
+                   .sort_by { |(_url, rank)| -rank }
+                   .map(&:first)
+      end
+
+      def ranked_anchor_url(anchor)
+        url = absolute_same_origin(anchor['href'])
+        return unless url
+
+        facts = LinkDestination::DestinationFacts.build(url)
+        rank = 0
+        rank += 3 if facts.taxonomy_path
+        rank += 1 if facts.content_path
+        [url, rank] if rank.positive?
+      end
+
+      def segment_seed_urls
+        return [] unless parsed_html
+
+        sst = SST::Normalizer.call(parsed_html)
+        link_resolver = Scoring::LinkResolver.new(entry_url)
+
+        SEGMENT_STRATEGIES.each do |strategy|
+          urls = segment_urls_for(sst, strategy, link_resolver)
+          return urls if urls.size >= 2
+        end
+        []
+      rescue StandardError
+        []
+      end
+
+      def segment_urls_for(sst, strategy, link_resolver)
+        segmenter = AutoSource::Segmenter.new(sst, base_url: entry_url, strategy:, link_resolver:)
+        primary = AutoSource::Segmenter::PrimaryLink.new(segmenter)
+
+        segmenter.call.filter_map do |segment|
+          link = segment.primary_link || primary.select(segment.root_node)
+          absolute_same_origin(link&.attrs&.href)
+        end.uniq
+      end
+
+      def listing_path_candidates
+        Syndication::CandidateCatalog::LISTING_PATHS.filter_map { |path| absolute_same_origin(path) }
+      end
+
+      def parsed_html
+        return @parsed_html if defined?(@parsed_html)
+
+        @parsed_html = response.html_response? ? response.parsed_body : nil
+      rescue RequestService::UnsupportedResponseContentType
+        @parsed_html = nil
+      end
+
+      def absolute_same_origin(href)
+        return if href.nil? || href.empty? || href.start_with?('#', 'mailto:', 'javascript:')
+
+        Html2rss::Url.from_relative(href, entry_url)
+      rescue ArgumentError
+        nil
+      end
+
+      def eligible_url?(url)
+        !entry_url.same_document?(url) && same_registrable_domain?(url)
+      end
+
+      def same_registrable_domain?(url)
+        url.domain == entry_url.domain
+      end
+    end
+  end
+end