html2rss 0.25.0 → 0.26.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1b6b0ddc57b5cf372cc9bc6405bb21f7dc6d00aa526e234132974ccc6ef609f7
-  data.tar.gz: 59a60ef044a99686838b5498f0712a455a6c5a832c27619db0c99ba21742f6a8
+  metadata.gz: 2df4782641361435095b1adbc805a0b5f8bbf47637450a271b212e0905e025a1
+  data.tar.gz: 38542dd16fd74c1983be1d20cfc49b64508ce70eeb501f0a312e0b9777c59f07
 SHA512:
-  metadata.gz: 8994aa3b90d791ea8cf75df9b576669fffe505fe0895e207745ab0e591f187ab490382d00a95d49fc1073570f886d6940f27f1c1ca80ba7c57817740f48c4bc4
-  data.tar.gz: dc62483a0ade2b582fb3839778a3a518a7f1a08e781d2105c0e93c70aaa26313e153a4e30121170b8a59e8472517e6cb8d9958b9c06fa3a862358631f469c18a
+  metadata.gz: e72f53ac552c7a0c65917d9d736a42e318360f205b7f532d845f89c0ea6bd219c09da0c1784596027566af8946aa2a9de937d21b39899a8838bd9fd70dd163ee
+  data.tar.gz: 69b705d89b11160fe0c392fb2045e61db8dc25d63a7e81af4d91201fce0face5e6fe3496b4b154af9d71eb41a6969491897d48162df382cd30facc30b3ab8bdc

data/README.md

@@ -13,7 +13,7 @@ Most people looking for a first working feed should start with `html2rss-web`, r
 Detailed usage guides, reference docs, and the feed directory live on the project website:
 
 - [Ruby gem documentation](https://html2rss.github.io/ruby-gem)
-- [Request strategies](https://html2rss.github.io/ruby-gem/reference/strategy) (`auto` = `faraday` → `botasaurus`; pin `browserless` explicitly)
+- [Request strategies](https://html2rss.github.io/ruby-gem/reference/strategy) (`auto` = `faraday` → `botasaurus`; or pin concrete strategies)
 - [Selectors & pagination](https://html2rss.github.io/ruby-gem/reference/selectors#paginated-feeds)
 - [Web application](https://html2rss.github.io/web-application)
 - [Feed directory](https://html2rss.github.io/feed-directory)
@@ -26,7 +26,7 @@ Cloud development: [Open in GitHub Codespaces](https://github.com/codespaces/new
 ## Architecture
 
 1. **Config** — loads and validates configuration (YAML/hash); schema via `html2rss schema` / `schema/html2rss-config.schema.json`
-2. **RequestService** — fetches pages (`faraday`, `botasaurus`, or explicit `browserless`)
+2. **RequestService** — fetches pages (`faraday`, `botasaurus`, or `local_file`)
 3. **Selectors** — extracts content via CSS selectors with extractors/post-processors
 4. **AutoSource** — auto-detects content (Schema.org, JSON state, semantic HTML, structural patterns)
 5. **FeedBuilder** — assembles Article objects and renders feeds (RSS 2.0 / JSON Feed 1.1)
@@ -35,6 +35,80 @@ Cloud development: [Open in GitHub Codespaces](https://github.com/codespaces/new
 Config -> Request -> Extraction -> Processing -> Building -> Output
 ```
 
+## Capture API
+
+The `Html2rss.capture` method analyzes any URL and produces a reusable feed config hash with derived CSS selectors. Use it to speed up writing feed configuration files.
+
+```ruby
+config = Html2rss.capture('https://example.com/articles')
+File.write('my-feed.yml', YAML.dump(Html2rss::HashUtil.deep_stringify_keys(config)))
+```
+
+The CLI alias `html2rss capture` prints the generated config as YAML to stdout. See [`docs/capture.md`](docs/capture.md) for detailed documentation.
+
+## MCP Server
+
+html2rss ships with an [MCP](https://modelcontextprotocol.io/) server that exposes gem capabilities as AI-consumable tools, resources, and prompts:
+
+```bash
+# Start with stdio transport (default; for Cursor/Claude Desktop)
+html2rss mcp
+
+# Start with HTTP transport (binds 127.0.0.1 only — local use)
+html2rss mcp --transport http --port 8080
+```
+
+HTTP transport needs `rack`, `rackup`, and `webrick` (declared gem dependencies). It listens on `127.0.0.1` only; do not expose it on a public interface without your own auth and Host/Origin controls.
+
+**Strategy note:** MCP tool `strategy: "auto"` collapses to `faraday` (no FeedPipeline botasaurus fallback). If results are empty or JS-gated, retry with `strategy: "botasaurus"` and `BOTASAURUS_SCRAPER_URL` set.
+
+### Tools
+
+| Name              | When to use                                               |
+| ----------------- | --------------------------------------------------------- |
+| `scrape_url`      | One-shot articles now (no saved config)                   |
+| `inspect_url`     | Diagnose weak scrape/capture (scrapers/SST/segments)      |
+| `capture_config`  | Derive a durable feed config (+ quality `_meta`)          |
+| `validate_config` | Schema-check a config before apply (`isError` on failure) |
+| `apply_config`    | Run a validated config → RSS XML                          |
+
+### Resources
+
+| URI                     | Description                                                     |
+| ----------------------- | --------------------------------------------------------------- |
+| `html2rss://schema`     | Full JSON Schema for feed configurations                        |
+| `html2rss://extractors` | Registered extractor **names** (options live in schema `$defs`) |
+| `html2rss://strategies` | Registered request strategy names                               |
+
+### Prompts
+
+| Name                  | Description                                             |
+| --------------------- | ------------------------------------------------------- |
+| `scrape-webpage`      | Guided scrape → inspect/retry with botasaurus if needed |
+| `capture-feed-config` | Guided capture → validate → optional apply              |
+
+The MCP module (`Html2rss::MCP`) lazy-loads the `mcp` gem — no cost when the server is not running.
+
+## Botasaurus scrape API (Docker)
+
+Start the Botasaurus scrape API for JavaScript-rendered pages (this compose file is **not** the MCP server):
+
+```bash
+docker compose -f docker-compose.botasaurus.yml up -d
+```
+
+Set `BOTASAURUS_SCRAPER_URL` to `http://127.0.0.1:4010` and use strategy `botasaurus` in MCP tools, Capture, or the CLI.
+
+## Request Strategies
+
+| Strategy     | Description                                                                   |
+| ------------ | ----------------------------------------------------------------------------- |
+| `auto`       | Tries `faraday`, falls back to `botasaurus` (default in gem/CLI FeedPipeline) |
+| `faraday`    | Plain HTTP requests via Faraday                                               |
+| `botasaurus` | Puppeteer-backed scraping for JavaScript pages                                |
+
+MCP tools intentionally collapse `auto` → `faraday` (see MCP section above). Elsewhere, strategy can be set via CLI (`--strategy`), gem API keyword argument, or feed config `request.strategy`. See the [request strategies docs](https://html2rss.github.io/ruby-gem/reference/strategy) for more details.
+
 ## License
 
 This project is licensed under the MIT License — see the [LICENSE](LICENSE) file for details.

data/html2rss.gemspec

@@ -39,14 +39,17 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'faraday-follow_redirects'
   spec.add_dependency 'faraday-gzip', '~> 3'
   spec.add_dependency 'kramdown'
+  spec.add_dependency 'mcp', '~> 1.0'
   spec.add_dependency 'mime-types', '> 3.0'
   spec.add_dependency 'nokogiri', '>= 1.10', '< 2.0'
-  spec.add_dependency 'puppeteer-ruby'
+  spec.add_dependency 'rack', '~> 3.0'
+  spec.add_dependency 'rackup', '~> 2.0'
   spec.add_dependency 'regexp_parser'
   spec.add_dependency 'reverse_markdown', '~> 3.0'
   spec.add_dependency 'rss'
   spec.add_dependency 'sanitize'
   spec.add_dependency 'thor'
   spec.add_dependency 'tzinfo'
+  spec.add_dependency 'webrick', '~> 1.9'
   spec.add_dependency 'zeitwerk'
 end

data/lib/html2rss/auto_source/cleanup.rb

@@ -9,81 +9,107 @@ module Html2rss
     class Cleanup
       # Default cleanup behavior for auto-sourced article lists.
       DEFAULT_CONFIG = {
-        keep_different_domain: false,
-        min_words_title: 3
+        keep_different_domain: false
       }.freeze
 
+      # Minimum alphanumeric word count for present titles.
+      MIN_WORDS = 3
+
       # Allowed URL schemes for article filtering.
       VALID_SCHEMES = %w[http https].to_set.freeze
 
+      # Credit-agency-only or photo-credit titles (not headlines).
+      CREDIT_TITLE = %r{
+        \A(?:AFP|Getty(?:\s+Images)?|Reuters|dpa|Imagn)
+          (?:\s*/\s*(?:AFP|Getty(?:\s+Images)?|Reuters|dpa|Imagn))*\z
+        |
+        \A(?:Image|Photo|Credit)\s*[:|]?\s*
+          (?:AFP|Getty(?:\s+Images)?|Reuters|dpa|Imagn)\b
+      }ix
+
+      # Dotted / methode CMS tokens mistaken for titles.
+      CMS_TOKEN_TITLE = /\A(?:lucy\.\w[\w.-]*|methode[-.][\w.-]+)\z/i
+
+      # Raw URL slug / token clusters (hyphen or underscore, no natural phrasing).
+      SLUG_TITLE = /\A\p{Alnum}+(?:[-_]\p{Alnum}+){2,}\z/
+
+      # Date-prefix path tokens, raw or titleized ("2026 08 16 …", "2026-08-16-…").
+      DATE_PREFIX_TITLE = /\A\d{4}(?:[\s.-]+\d{1,2}){2}\b/
+
+      # Titleized path ending in a long numeric CMS id.
+      TITLEIZED_PATH_TITLE = /\A(?:\d+|\p{Lu}[\p{L}\p{M}]*)(?:\s+(?:\d+|\p{Lu}[\p{L}\p{M}]*))*\s+\d{6,}\z/
+
+      # Template / placeholder tokens mistaken for titles.
+      TEMPLATE_TITLE = /(\{\{[^}]+\}\}|%\{\w+\})/
+
       class << self
         # @param articles [Array<Article>] extracted article candidates
         # @param url [Html2rss::Url] feed source URL used for same-host filtering
         # @param keep_different_domain [Boolean] whether to keep off-domain entries
-        # @param min_words_title [Integer] minimum word count for title filtering
         # @return [Array<Article>] cleaned article list
-        def call(articles, url:, keep_different_domain:, min_words_title:)
+        def call(articles, url:, keep_different_domain: DEFAULT_CONFIG.fetch(:keep_different_domain))
           Log.debug "Cleanup: start with #{articles.size} articles"
 
           articles.select!(&:valid?)
 
-          deduplicate_by!(articles, :url)
-
+          deduplicate_by_url!(articles)
           keep_only_http_urls!(articles)
+          reject_self_links!(articles, url)
           reject_different_domain!(articles, url) unless keep_different_domain
-          keep_only_with_min_words_title!(articles, min_words_title:)
+          reject_low_quality_titles!(articles)
 
           Log.debug "Cleanup: end with #{articles.size} articles"
           articles
         end
 
-        ##
-        # Deduplicates articles by a given key.
-        #
-        # @param articles [Array<Article>] The list of articles to process.
-        # @param key [Symbol] The key to deduplicate by.
-        # @return [Array<Article>] the mutated articles array
-        def deduplicate_by!(articles, key)
+        private
+
+        def deduplicate_by_url!(articles)
           seen = {}
           articles.reject! do |article|
-            value = article.public_send(key)
-            value.nil? || seen.key?(value).tap { seen[value] = true }
+            identity = url_identity(article.url)
+            identity.nil? || seen.key?(identity).tap { seen[identity] = true }
           end
         end
 
-        ##
-        # Keeps only articles with HTTP or HTTPS URLs.
-        #
-        # @param articles [Array<Article>] The list of articles to process.
-        # @return [Array<Article>] the mutated articles array
         def keep_only_http_urls!(articles)
           articles.select! { |article| VALID_SCHEMES.include?(article.url&.scheme) }
         end
 
-        ##
-        # Rejects articles that have a URL not on the same domain as the source.
-        #
-        # @param articles [Array<Article>] The list of articles to process.
-        # @param base_url [Html2rss::Url] The source URL to compare against.
-        # @return [Array<Article>] the mutated articles array
+        def reject_self_links!(articles, base_url)
+          source_identity = url_identity(base_url)
+          articles.reject! { |article| url_identity(article.url) == source_identity }
+        end
+
         def reject_different_domain!(articles, base_url)
           base_host = base_url.host
           articles.select! { |article| article.url&.host == base_host }
         end
 
-        ##
-        # Keeps only articles with a title that is present and has at least `min_words_title` words.
-        #
-        # @param articles [Array<Article>] The list of articles to process.
-        # @param min_words_title [Integer] The minimum number of words in the title.
-        # @return [Array<Article>] the mutated articles array
-        def keep_only_with_min_words_title!(articles, min_words_title:)
+        # Keep missing titles (nil provenance). Drop present junk/unnatural titles —
+        # blanking them would hide bad extraction as "unknown" and inflate empty items.
+        def reject_low_quality_titles!(articles)
           articles.select! do |article|
-            article.title ? word_count_at_least?(article.title, min_words_title) : true
+            title = article.title
+            title.nil? || (word_count_at_least?(title, MIN_WORDS) && !junk_title?(title))
           end
         end
 
-        private
+        def url_identity(url)
+          url&.without_fragment&.to_s
+        end
+
+        def junk_title?(title)
+          CREDIT_TITLE.match?(title) || CMS_TOKEN_TITLE.match?(title) || unnatural_title?(title)
+        end
+
+        def unnatural_title?(title)
+          stripped = title.to_s.strip
+          SLUG_TITLE.match?(stripped) ||
+            DATE_PREFIX_TITLE.match?(stripped) ||
+            TITLEIZED_PATH_TITLE.match?(stripped) ||
+            TEMPLATE_TITLE.match?(stripped)
+        end
 
         def word_count_at_least?(str, min_words)
           count = 0

data/lib/html2rss/auto_source/scraper/json_state.rb

@@ -27,6 +27,36 @@ module Html2rss
           def json_documents(parsed_body)
             DocumentScanner.json_documents(parsed_body)
           end
+
+          # Walks a JSON document tree and yields normalized article hashes.
+          # Shared with {XhrArticles} so XHR-captured JSON reuses one discovery algorithm.
+          #
+          # @param document [Hash, Array, Object] parsed JSON document node
+          # @param base_url [String, Html2rss::Url] base URL for relative link resolution
+          # @yield [Hash{Symbol => Object}, nil] normalized article hash
+          # @return [void]
+          def discover_articles(document, base_url:, &block)
+            case document
+            when Array then handle_array(document, base_url:, &block)
+            when Hash then document.each_value { discover_articles(_1, base_url:, &block) if traversable?(_1) }
+            end
+          end
+
+          private
+
+          def handle_array(array, base_url:, &block)
+            if CandidateDetector.array_of_articles?(array)
+              array.each do |entry|
+                yield(ArticleNormalizer.normalise(entry, base_url:))
+              end
+            else
+              array.each { discover_articles(_1, base_url:, &block) if traversable?(_1) }
+            end
+          end
+
+          def traversable?(value)
+            value.is_a?(Array) || value.is_a?(Hash)
+          end
         end
 
         # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML document
@@ -51,7 +81,7 @@ module Html2rss
           return enum_for(:each) unless block_given?
 
           json_documents.each do |document|
-            discover_articles(document) do |article|
+            self.class.discover_articles(document, base_url: url) do |article|
               yield article if article
             end
           end
@@ -64,27 +94,6 @@ module Html2rss
         def json_documents
           self.class.json_documents(parsed_body)
         end
-
-        def discover_articles(document, &block)
-          case document
-          when Array then handle_array(document, &block)
-          when Hash then document.each_value { discover_articles(_1, &block) if traversable?(_1) }
-          end
-        end
-
-        def handle_array(array, &block)
-          if CandidateDetector.array_of_articles?(array)
-            array.each do |entry|
-              yield(ArticleNormalizer.normalise(entry, base_url: url))
-            end
-          else
-            array.each { discover_articles(_1, &block) if traversable?(_1) }
-          end
-        end
-
-        def traversable?(value)
-          value.is_a?(Array) || value.is_a?(Hash)
-        end
       end
     end
   end

data/lib/html2rss/auto_source/scraper/schema/item_list.rb

@@ -31,9 +31,8 @@ module Html2rss
             return unless object.is_a?(Hash)
             return unless emit?(object, from_list_item:)
 
-            article = Thing.new(object, url: base_url || '').call
-            titleize_list_item_stub!(article) if from_list_item
-            article
+            # Leave empty titles empty — do not invent from URL path (Cleanup allows nil).
+            Thing.new(object, url: base_url || '').call
           end
 
           # @param element [Object] raw list entry
@@ -56,17 +55,6 @@ module Html2rss
 
             Schema.normalize_types(object[:@type]).intersect?(Thing::SUPPORTED_TYPES)
           end
-
-          # URL-only ListItem stubs historically used a titleized path as title.
-          #
-          # @param article [Hash] scraped article hash
-          # @return [void]
-          def titleize_list_item_stub!(article)
-            return unless article[:title].to_s.empty?
-            return unless (article_url = article[:url])
-
-            article[:title] = article_url.titleized
-          end
         end
       end
     end

data/lib/html2rss/auto_source/scraper/xhr_articles.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Html2rss
+  class AutoSource
+    module Scraper
+      # Extracts articles from JSON XHR/fetch responses captured during a
+      # browser-tier scrape (see RequestService::Response#captured_responses).
+      class XhrArticles
+        include Enumerable
+
+        # @return [Symbol] scraper config key
+        def self.options_key = :xhr_articles
+
+        # @param _opts [Hash] unused scraper options
+        # @return [Integer] zero — no additional HTTP requests
+        def self.request_slots(_opts = {}) = 0
+
+        # Not detectable from HTML alone; instance {#extractable?} uses captures.
+        #
+        # @param _parsed_body [Nokogiri::HTML::Document, nil]
+        # @return [Boolean]
+        def self.articles?(_parsed_body) = false
+
+        # @param _parsed_body [Nokogiri::HTML::Document, nil] unused HTML document
+        # @param url [String, Html2rss::Url] page URL used to resolve relative links
+        # @param captured_responses [Array<Hash>] JSON bodies from Response#captured_responses
+        # @param _opts [Hash] scraper-specific options
+        # @option _opts [Object] :_reserved reserved for future scraper-specific options
+        def initialize(_parsed_body, url:, captured_responses: [], **_opts)
+          @url = url
+          @captured_responses = captured_responses
+        end
+
+        # @return [Boolean] true when any captured body contains article-like arrays
+        def extractable?
+          parsed_bodies.any? { |doc| JsonState::CandidateDetector.candidate_array?(doc) }
+        end
+
+        # @yield [Hash{Symbol => Object}] normalized article hash
+        # @return [Enumerator, void] article enumerator when no block is given
+        def each
+          return enum_for(:each) unless block_given?
+
+          parsed_bodies.each do |doc|
+            JsonState.discover_articles(doc, base_url: @url) { |article| yield article if article }
+          end
+        end
+
+        private
+
+        def parsed_bodies
+          @parsed_bodies ||= @captured_responses.filter_map { |captured| parse(captured) }
+                                                .select { |doc| JsonState::CandidateDetector.candidate_array?(doc) }
+        end
+
+        def parse(captured)
+          body = captured[:body] || captured['body']
+          return unless body.is_a?(String)
+
+          JSON.parse(body, symbolize_names: true)
+        rescue JSON::ParserError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/scraper.rb

@@ -11,7 +11,7 @@ module Html2rss
     # Detection is intentionally shallow for most scrapers, but instance-based
     # matching is available for scrapers that need to carry expensive selection
     # state forward into extraction.
-    module Scraper
+    module Scraper # rubocop:disable Metrics/ModuleLength -- tier registry + construction helpers
       # Root markers indicating likely app-shell/client-rendered surfaces.
       APP_SHELL_ROOT_SELECTORS = '#app, #root, #__next, [data-reactroot], [ng-app], [id*="app-shell"]'
       # Maximum anchors tolerated before app-shell detection is considered unlikely.
@@ -22,7 +22,7 @@ module Html2rss
       # Extraction tiers: merge within a tier, then stop when AutoSource has enough articles.
       # Heuristic scrapers are separate tiers so SemanticHtml can satisfy before Html runs.
       SCRAPER_TIERS = [
-        [Schema, Microdata, Microformats2, JsonState].freeze,
+        [Schema, Microdata, Microformats2, JsonState, XhrArticles].freeze,
         [WordpressApi, Sitemap, MetaOembed].freeze,
         [SemanticHtml].freeze,
         [Html].freeze
@@ -35,19 +35,21 @@ module Html2rss
       HEURISTIC_SCRAPERS = [SemanticHtml, Html].freeze
       # Scrapers that accept a shared follow-up +request_session+.
       REQUEST_SESSION_SCRAPERS = [WordpressApi, Sitemap, MetaOembed].freeze
+      # Scrapers that consume browser-captured XHR/fetch JSON bodies.
+      CAPTURED_RESPONSE_SCRAPERS = [XhrArticles].freeze
 
       ##
       # Error raised when no suitable scraper is found.
       class NoScraperFound < Html2rss::Error
-        # User-facing messages grouped by no-scraper surface category.
+        # Surface diagnostics shared with {Html2rss::NoFeedItemsExtracted} (one string home).
         CATEGORY_MESSAGES = {
-          blocked_surface: 'No scrapers found: blocked surface likely (anti-bot or interstitial). ' \
-                           'Retry with --strategy browserless, try a more specific public listing URL, ' \
+          blocked_surface: 'blocked surface likely (anti-bot or interstitial). ' \
+                           'Target a direct listing URL, configure BOTASAURUS_SCRAPER_URL, ' \
                            'or run from an environment that can complete anti-bot checks.',
-          app_shell: 'No scrapers found: app-shell surface detected (client-rendered page with little or no ' \
-                     'server-rendered article HTML). Retry with --strategy browserless, or target a direct ' \
+          app_shell: 'app-shell surface detected (client-rendered page with little or no ' \
+                     'server-rendered article HTML). Configure BOTASAURUS_SCRAPER_URL or target a direct ' \
                      'listing/update URL instead of a homepage or shell entrypoint.',
-          unsupported_surface: 'No scrapers found: unsupported extraction surface for auto mode. ' \
+          unsupported_surface: 'unsupported extraction surface for auto mode. ' \
                                'Try a direct listing/changelog/category URL, ' \
                                'or use explicit selectors in a feed config.'
         }.freeze
@@ -57,7 +59,7 @@ module Html2rss
         def initialize(message = nil, category: :unsupported_surface)
           validate_category!(category)
           @category = category
-          super(message || CATEGORY_MESSAGES.fetch(@category))
+          super(message || "No scrapers found: #{CATEGORY_MESSAGES.fetch(@category)}")
         end
 
         attr_reader :category
@@ -81,6 +83,7 @@ module Html2rss
       # @option opts [Hash] :microdata scraper toggle and configuration
       # @option opts [Hash] :microformats2 scraper toggle and configuration
       # @option opts [Hash] :json_state scraper toggle and configuration
+      # @option opts [Hash] :xhr_articles scraper toggle and configuration
       # @option opts [Hash] :meta_oembed scraper toggle and configuration
       # @option opts [Hash] :semantic_html scraper toggle and configuration
       # @option opts [Hash] :html scraper toggle and configuration
@@ -122,27 +125,36 @@ module Html2rss
       # @param body [String, nil]
       # @param document [SST::Document, nil]
       # @param link_resolver [Scoring::LinkResolver, nil]
+      # @param captured_responses [Array<Hash>] browser-captured JSON XHR/fetch bodies
       # @option opts [Hash] :wordpress_api scraper toggle and configuration
       # @option opts [Hash] :schema scraper toggle and configuration
       # @option opts [Hash] :microdata scraper toggle and configuration
       # @option opts [Hash] :microformats2 scraper toggle and configuration
       # @option opts [Hash] :json_state scraper toggle and configuration
+      # @option opts [Hash] :xhr_articles scraper toggle and configuration
       # @option opts [Hash] :meta_oembed scraper toggle and configuration
       # @option opts [Hash] :semantic_html scraper toggle and configuration
       # @option opts [Hash] :html scraper toggle and configuration
       # @option opts [Hash] :sitemap scraper toggle and configuration
       # @return [Object, nil]
-      # rubocop:disable Metrics/ParameterLists -- construction context for structured and heuristic scrapers
+      # rubocop:disable Metrics/ParameterLists, Metrics/MethodLength -- construction context for structured and heuristic scrapers
       def self.build_instance(scraper, parsed_body, opts:, url:, request_session: nil, body: nil, document: nil,
-                              link_resolver: nil)
+                              link_resolver: nil, captured_responses: [])
         return unless opts.dig(scraper.options_key, :enabled)
         return if HEURISTIC_SCRAPERS.include?(scraper) && document.nil?
 
         scraper_opts = opts.fetch(scraper.options_key, {}).except(:enabled)
-        kwargs = construction_kwargs(scraper, request_session:, body:, document:, link_resolver:)
+        kwargs = construction_kwargs(
+          scraper,
+          request_session:,
+          body:,
+          document:,
+          link_resolver:,
+          captured_responses:
+        )
         scraper.new(parsed_body, url:, **kwargs, **scraper_opts)
       end
-      # rubocop:enable Metrics/ParameterLists
+      # rubocop:enable Metrics/ParameterLists, Metrics/MethodLength
 
       ##
       # @param instance [Object]
@@ -162,25 +174,32 @@ module Html2rss
         NoScraperFound.new(category: classify_no_scraper_surface(parsed_body, body:))
       end
 
-      def self.construction_kwargs(scraper, request_session:, body:, document:, link_resolver:)
+      def self.construction_kwargs(scraper, request_session:, body:, document:, link_resolver:, # rubocop:disable Metrics/ParameterLists -- scraper construction bag
+                                   captured_responses:)
         if HEURISTIC_SCRAPERS.include?(scraper)
           { document:, link_resolver: }.compact
         else
           {}.tap do |kwargs|
             kwargs[:request_session] = request_session if REQUEST_SESSION_SCRAPERS.include?(scraper)
             kwargs[:body] = body if scraper == Sitemap
+            kwargs[:captured_responses] = captured_responses if CAPTURED_RESPONSE_SCRAPERS.include?(scraper)
           end
         end
       end
       private_class_method :construction_kwargs
 
+      ##
+      # Classifies why scrapers could not extract from a parsed page.
+      #
+      # @param parsed_body [Nokogiri::HTML::Document]
+      # @param body [String, nil] raw body for blocked-surface detection
+      # @return [Symbol] one of {NoScraperFound::CATEGORY_MESSAGES} keys
       def self.classify_no_scraper_surface(parsed_body, body: nil)
         return :blocked_surface if blocked_surface?(parsed_body, body:)
         return :app_shell if app_shell_surface?(parsed_body)
 
         :unsupported_surface
       end
-      private_class_method :classify_no_scraper_surface
 
       def self.blocked_surface?(parsed_body, body: nil)
         Html2rss::RequestService::BlockedSurface.interstitial?(body || parsed_body.to_html)

data/lib/html2rss/auto_source/segmenter/list.rb

@@ -70,7 +70,9 @@ module Html2rss
           return false unless facts
 
           text = node.visible_text.to_s.strip
-          !segmenter.noise_policy.noise_anchor?(text:, destination_facts: facts, anchor: node)
+          !segmenter.noise_policy.noise_anchor?(
+            text:, destination_facts: facts, anchor: node, utility_landmark_ancestor: false
+          )
         end
         module_function :relevant_anchor?
         private_class_method :relevant_anchor?

data/lib/html2rss/auto_source/segmenter/primary_link.rb

@@ -51,16 +51,15 @@ module Html2rss
           heading_match = meaningful && heading_text.match?(/\p{Alnum}/) && heading_text == text
 
           return if @noise_policy.noise_anchor?(
-            text:, destination_facts: destination, anchor:, container:, heading_anchor:
+            text:, destination_facts: destination, anchor:, container:, heading_anchor:,
+            utility_landmark_ancestor: @segmenter.landmark_ancestor?(anchor, container)
           )
           return unless meaningful || content_like || heading_anchor
 
-          score = Scoring::AnchorScore.score(
-            heading_anchor:,
-            heading_text_match: heading_match,
-            meaningful_text: meaningful,
-            content_like_destination: content_like
-          )
+          score = (heading_anchor ? 100 : 0) +
+                  (heading_match ? 20 : 0) +
+                  (meaningful ? 10 : 0) +
+                  (content_like ? 10 : 0)
 
           { anchor:, destination: destination.destination, score: }
         end