html2rss 0.25.0 → 0.27.0

data/lib/html2rss/auto_source/scraper.rb

@@ -11,18 +11,20 @@ module Html2rss
     # Detection is intentionally shallow for most scrapers, but instance-based
     # matching is available for scrapers that need to carry expensive selection
     # state forward into extraction.
-    module Scraper
+    module Scraper # rubocop:disable Metrics/ModuleLength -- tier registry + construction helpers
       # Root markers indicating likely app-shell/client-rendered surfaces.
       APP_SHELL_ROOT_SELECTORS = '#app, #root, #__next, [data-reactroot], [ng-app], [id*="app-shell"]'
       # Maximum anchors tolerated before app-shell detection is considered unlikely.
       APP_SHELL_MAX_ANCHORS = 2
+      # Minimum same-page anchors suggesting a high-entropy homepage/hub surface.
+      HIGH_ENTROPY_MIN_ANCHORS = 20
       # Maximum visible text length tolerated for app-shell classification.
       APP_SHELL_MAX_VISIBLE_TEXT_LENGTH = 220
 
       # Extraction tiers: merge within a tier, then stop when AutoSource has enough articles.
       # Heuristic scrapers are separate tiers so SemanticHtml can satisfy before Html runs.
       SCRAPER_TIERS = [
-        [Schema, Microdata, Microformats2, JsonState].freeze,
+        [Schema, Microdata, Microformats2, JsonState, XhrArticles].freeze,
         [WordpressApi, Sitemap, MetaOembed].freeze,
         [SemanticHtml].freeze,
         [Html].freeze
@@ -35,19 +37,24 @@ module Html2rss
       HEURISTIC_SCRAPERS = [SemanticHtml, Html].freeze
       # Scrapers that accept a shared follow-up +request_session+.
       REQUEST_SESSION_SCRAPERS = [WordpressApi, Sitemap, MetaOembed].freeze
+      # Scrapers that consume browser-captured XHR/fetch JSON bodies.
+      CAPTURED_RESPONSE_SCRAPERS = [XhrArticles].freeze
 
       ##
       # Error raised when no suitable scraper is found.
       class NoScraperFound < Html2rss::Error
-        # User-facing messages grouped by no-scraper surface category.
+        # Surface diagnostics shared with {Html2rss::NoFeedItemsExtracted} (one string home).
         CATEGORY_MESSAGES = {
-          blocked_surface: 'No scrapers found: blocked surface likely (anti-bot or interstitial). ' \
-                           'Retry with --strategy browserless, try a more specific public listing URL, ' \
+          blocked_surface: 'blocked surface likely (anti-bot or interstitial). ' \
+                           'Target a direct listing URL, configure BOTASAURUS_SCRAPER_URL, ' \
                            'or run from an environment that can complete anti-bot checks.',
-          app_shell: 'No scrapers found: app-shell surface detected (client-rendered page with little or no ' \
-                     'server-rendered article HTML). Retry with --strategy browserless, or target a direct ' \
+          app_shell: 'app-shell surface detected (client-rendered page with little or no ' \
+                     'server-rendered article HTML). Configure BOTASAURUS_SCRAPER_URL or target a direct ' \
                      'listing/update URL instead of a homepage or shell entrypoint.',
-          unsupported_surface: 'No scrapers found: unsupported extraction surface for auto mode. ' \
+          high_entropy_surface: 'high-entropy surface (e.g. news homepage) with few or no durable ' \
+                                'article destinations after admission. Target a section, topic, or ' \
+                                'listing/update URL instead of the site homepage.',
+          unsupported_surface: 'unsupported extraction surface for auto mode. ' \
                                'Try a direct listing/changelog/category URL, ' \
                                'or use explicit selectors in a feed config.'
         }.freeze
@@ -57,7 +64,7 @@ module Html2rss
         def initialize(message = nil, category: :unsupported_surface)
           validate_category!(category)
           @category = category
-          super(message || CATEGORY_MESSAGES.fetch(@category))
+          super(message || "No scrapers found: #{CATEGORY_MESSAGES.fetch(@category)}")
         end
 
         attr_reader :category
@@ -81,6 +88,7 @@ module Html2rss
       # @option opts [Hash] :microdata scraper toggle and configuration
       # @option opts [Hash] :microformats2 scraper toggle and configuration
       # @option opts [Hash] :json_state scraper toggle and configuration
+      # @option opts [Hash] :xhr_articles scraper toggle and configuration
       # @option opts [Hash] :meta_oembed scraper toggle and configuration
       # @option opts [Hash] :semantic_html scraper toggle and configuration
       # @option opts [Hash] :html scraper toggle and configuration
@@ -122,27 +130,36 @@ module Html2rss
       # @param body [String, nil]
       # @param document [SST::Document, nil]
       # @param link_resolver [Scoring::LinkResolver, nil]
+      # @param captured_responses [Array<Hash>] browser-captured JSON XHR/fetch bodies
       # @option opts [Hash] :wordpress_api scraper toggle and configuration
       # @option opts [Hash] :schema scraper toggle and configuration
       # @option opts [Hash] :microdata scraper toggle and configuration
       # @option opts [Hash] :microformats2 scraper toggle and configuration
       # @option opts [Hash] :json_state scraper toggle and configuration
+      # @option opts [Hash] :xhr_articles scraper toggle and configuration
       # @option opts [Hash] :meta_oembed scraper toggle and configuration
       # @option opts [Hash] :semantic_html scraper toggle and configuration
       # @option opts [Hash] :html scraper toggle and configuration
       # @option opts [Hash] :sitemap scraper toggle and configuration
       # @return [Object, nil]
-      # rubocop:disable Metrics/ParameterLists -- construction context for structured and heuristic scrapers
+      # rubocop:disable Metrics/ParameterLists, Metrics/MethodLength -- construction context for structured and heuristic scrapers
       def self.build_instance(scraper, parsed_body, opts:, url:, request_session: nil, body: nil, document: nil,
-                              link_resolver: nil)
+                              link_resolver: nil, captured_responses: [])
         return unless opts.dig(scraper.options_key, :enabled)
         return if HEURISTIC_SCRAPERS.include?(scraper) && document.nil?
 
         scraper_opts = opts.fetch(scraper.options_key, {}).except(:enabled)
-        kwargs = construction_kwargs(scraper, request_session:, body:, document:, link_resolver:)
+        kwargs = construction_kwargs(
+          scraper,
+          request_session:,
+          body:,
+          document:,
+          link_resolver:,
+          captured_responses:
+        )
         scraper.new(parsed_body, url:, **kwargs, **scraper_opts)
       end
-      # rubocop:enable Metrics/ParameterLists
+      # rubocop:enable Metrics/ParameterLists, Metrics/MethodLength
 
       ##
       # @param instance [Object]
@@ -162,25 +179,33 @@ module Html2rss
         NoScraperFound.new(category: classify_no_scraper_surface(parsed_body, body:))
       end
 
-      def self.construction_kwargs(scraper, request_session:, body:, document:, link_resolver:)
+      def self.construction_kwargs(scraper, request_session:, body:, document:, link_resolver:, # rubocop:disable Metrics/ParameterLists -- scraper construction bag
+                                   captured_responses:)
         if HEURISTIC_SCRAPERS.include?(scraper)
           { document:, link_resolver: }.compact
         else
           {}.tap do |kwargs|
             kwargs[:request_session] = request_session if REQUEST_SESSION_SCRAPERS.include?(scraper)
             kwargs[:body] = body if scraper == Sitemap
+            kwargs[:captured_responses] = captured_responses if CAPTURED_RESPONSE_SCRAPERS.include?(scraper)
           end
         end
       end
       private_class_method :construction_kwargs
 
+      ##
+      # Classifies why scrapers could not extract from a parsed page.
+      #
+      # @param parsed_body [Nokogiri::HTML::Document]
+      # @param body [String, nil] raw body for blocked-surface detection
+      # @return [Symbol] one of {NoScraperFound::CATEGORY_MESSAGES} keys
       def self.classify_no_scraper_surface(parsed_body, body: nil)
         return :blocked_surface if blocked_surface?(parsed_body, body:)
         return :app_shell if app_shell_surface?(parsed_body)
+        return :high_entropy_surface if high_entropy_surface?(parsed_body)
 
         :unsupported_surface
       end
-      private_class_method :classify_no_scraper_surface
 
       def self.blocked_surface?(parsed_body, body: nil)
         Html2rss::RequestService::BlockedSurface.interstitial?(body || parsed_body.to_html)
@@ -197,6 +222,13 @@ module Html2rss
       end
       private_class_method :app_shell_surface?
 
+      def self.high_entropy_surface?(parsed_body)
+        return false if parsed_body.nil?
+
+        parsed_body.css('body a[href]').size >= HIGH_ENTROPY_MIN_ANCHORS
+      end
+      private_class_method :high_entropy_surface?
+
       def self.sparse_anchor_surface?(parsed_body)
         parsed_body.css('body a[href]').size <= APP_SHELL_MAX_ANCHORS
       end

data/lib/html2rss/auto_source/segmenter/list.rb

@@ -70,7 +70,9 @@ module Html2rss
           return false unless facts
 
           text = node.visible_text.to_s.strip
-          !segmenter.noise_policy.noise_anchor?(text:, destination_facts: facts, anchor: node)
+          !segmenter.noise_policy.noise_anchor?(
+            text:, destination_facts: facts, anchor: node, utility_landmark_ancestor: false
+          )
         end
         module_function :relevant_anchor?
         private_class_method :relevant_anchor?

data/lib/html2rss/auto_source/segmenter/primary_link.rb

@@ -51,16 +51,15 @@ module Html2rss
           heading_match = meaningful && heading_text.match?(/\p{Alnum}/) && heading_text == text
 
           return if @noise_policy.noise_anchor?(
-            text:, destination_facts: destination, anchor:, container:, heading_anchor:
+            text:, destination_facts: destination, anchor:, container:, heading_anchor:,
+            utility_landmark_ancestor: @segmenter.landmark_ancestor?(anchor, container)
           )
           return unless meaningful || content_like || heading_anchor
 
-          score = Scoring::AnchorScore.score(
-            heading_anchor:,
-            heading_text_match: heading_match,
-            meaningful_text: meaningful,
-            content_like_destination: content_like
-          )
+          score = (heading_anchor ? 100 : 0) +
+                  (heading_match ? 20 : 0) +
+                  (meaningful ? 10 : 0) +
+                  (content_like ? 10 : 0)
 
           { anchor:, destination: destination.destination, score: }
         end

data/lib/html2rss/auto_source/segmenter.rb

@@ -44,7 +44,7 @@ module Html2rss
         @minimum_selector_frequency = opts.fetch(:minimum_selector_frequency, 2)
         @use_top_selectors = opts.fetch(:use_top_selectors, 5)
         @link_resolver = opts[:link_resolver] || Scoring::LinkResolver.new(base_url)
-        @noise_policy = LinkDestination::NoisePolicy.new(link_resolver: @link_resolver, index: document.index)
+        @noise_policy = LinkDestination::NoisePolicy.new(link_resolver: @link_resolver)
       end
 
       # @return [Array<Segment>]
@@ -61,6 +61,35 @@ module Html2rss
 
       # @return [SST::Index]
       def index = document.index
+
+      ##
+      # Whether +anchor+ has a utility-landmark ancestor outside +container+.
+      #
+      # @param anchor [SST::Node]
+      # @param container [SST::Node]
+      # @return [Boolean]
+      def landmark_ancestor?(anchor, container)
+        cache = (@landmark_cache ||= {})
+        key = [anchor.object_id, container.object_id]
+        return cache[key] if cache.key?(key)
+
+        cache[key] = compute_landmark_ancestor?(anchor:, container:)
+      end
+
+      private
+
+      def compute_landmark_ancestor?(anchor:, container:) # rubocop:disable Metrics/CyclomaticComplexity
+        return false unless anchor && container
+
+        curr = index.parent_of(anchor)
+        while curr && curr.name != :html
+          return true if curr != container && curr.utility_landmark?
+          return false if curr.equal?(container)
+
+          curr = index.parent_of(curr)
+        end
+        false
+      end
     end
   end
 end

data/lib/html2rss/auto_source.rb

@@ -15,8 +15,8 @@ module Html2rss
   # @see Html2rss::AutoSource::Scraper::Schema
   # @see Html2rss::AutoSource::Scraper::SemanticHtml
   # @see Html2rss::AutoSource::Scraper::Html
-  # rubocop:disable Metrics/ClassLength -- defaults + tiered extract stay on the contributor entry type
-  class AutoSource
+  # {include:file:lib/html2rss/auto_source/README.md}
+  class AutoSource # rubocop:disable Metrics/ClassLength -- defaults + tiered extract stay on the contributor entry type
     # Default max articles to keep (also the short-circuit floor across scraper tiers).
     DEFAULT_LIMIT = 25
 
@@ -44,6 +44,9 @@ module Html2rss
         json_state: {
           enabled: true
         },
+        xhr_articles: {
+          enabled: true
+        },
         meta_oembed: {
           enabled: true
         },
@@ -93,6 +96,7 @@ module Html2rss
       @parsed_body = response.parsed_body
       @body = response.body
       @url = response.url
+      @captured_responses = response.captured_responses
       @opts = opts
       @request_session = request_session
     end
@@ -103,18 +107,29 @@ module Html2rss
     # Tiers: in-page structured → follow-up IO → SemanticHtml → Html.
     # SST is built only when a heuristic tier runs. Later tiers are skipped once
     # +limit+ articles with url+title remain after Cleanup; the result is capped to +limit+.
+    # Html is skipped when earlier tiers already admitted at least one clean article
+    # below +limit+ — quality over padding with weaker heuristic junk.
     #
     # @return [Array<Html2rss::Article>] extracted articles
     def articles
       @articles ||= extract_articles
     rescue Html2rss::AutoSource::Scraper::NoScraperFound => error
       Log.warn "#{self.class}: no scraper matched #{url} (#{error.message})"
+      @admission_drops = {}.freeze
       []
     end
 
+    ##
+    # Reason → count tallies from the final {Cleanup} pass (empty until {#articles} runs).
+    #
+    # @return [Hash{String => Integer}]
+    def admission_drops
+      @admission_drops || {}
+    end
+
     private
 
-    attr_reader :url, :parsed_body, :body, :request_session
+    attr_reader :url, :parsed_body, :body, :request_session, :captured_responses
 
     # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
     def extract_articles
@@ -126,6 +141,7 @@ module Html2rss
 
       Scraper::SCRAPER_TIERS.each do |tier|
         break if enough_articles?(articles)
+        break if skip_html_padding?(tier, articles)
 
         if Scraper.heuristic_tier?(tier)
           document ||= Scraper.normalize_sst(parsed_body)
@@ -143,7 +159,8 @@ module Html2rss
             request_session:,
             body:,
             document:,
-            link_resolver:
+            link_resolver:,
+            captured_responses:
           )
           next unless instance
           next unless Scraper.extractable_instance?(instance, parsed_body)
@@ -155,16 +172,31 @@ module Html2rss
 
       raise Scraper.no_scraper_found_for(parsed_body, body:) unless matched
 
-      Cleanup.call(articles, url:, **cleanup_options).first(article_limit)
+      result = Cleanup.call(articles, url:, **cleanup_options)
+      @admission_drops = result.drop_tallies
+      result.articles.first(article_limit)
     end
     # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
 
     def enough_articles?(articles)
-      threshold = article_limit
-      return false if articles.size < threshold
+      return false if articles.size < article_limit
 
-      cleaned = Cleanup.call(articles.dup, url:, **cleanup_options)
-      cleaned.count { |article| article.url && !article.title.to_s.empty? } >= threshold
+      admitted_articles(articles).size >= article_limit
+    end
+
+    # Prefer fewer clean items over refilling +limit+ from the Html heuristic tier.
+    def skip_html_padding?(tier, articles)
+      html_only_tier?(tier) && admitted_articles(articles).any?
+    end
+
+    def html_only_tier?(tier)
+      tier == [Scraper::Html]
+    end
+
+    def admitted_articles(articles)
+      Cleanup.call(articles.dup, url:, **cleanup_options).articles.select do |article|
+        article.url && !article.title.to_s.empty?
+      end
     end
 
     def article_limit
@@ -188,5 +220,4 @@ module Html2rss
       @opts.fetch(:cleanup, {})
     end
   end
-  # rubocop:enable Metrics/ClassLength
 end

data/lib/html2rss/capture/README.md

@@ -0,0 +1,61 @@
+# Capture
+
+`Html2rss.capture` (and CLI `html2rss capture`) analyzes a URL through the feed pipeline and produces a reusable config with **items selector + `enhance: true` only** — no title/url/description attribute-selector soup. At feed-build time, `enhance: true` fills missing article fields via `Html::ArticleExtractor` on each matched item.
+
+## When to use it
+
+Point `capture` at a listing URL when you want a first-draft YAML config instead of hand-writing selectors. Treat the output as a draft: selector quality depends on page structure.
+
+## Gem API
+
+```ruby
+config = Html2rss.capture('https://example.com/articles')
+
+# {
+#   channel: { url: "...", title: "...", time_zone: "UTC" },
+#   selectors: {
+#     items: { selector: "div.post", enhance: true }
+#   }
+# }
+
+File.write('my-feed.yml', YAML.dump(Html2rss::HashUtil.deep_stringify_keys(config)))
+feed = Html2rss.feed(config)
+```
+
+`Capture.build` returns a `CaptureResult` with quality meta (`has_selectors`, `segment_strategy`, `admission_drops`, `selected_strategy`). `Html2rss.capture` returns only the config hash.
+
+### Options
+
+```ruby
+Html2rss.capture('https://spa-site.com', strategy: :botasaurus)
+Html2rss.capture('https://example.com', items_selector: '.article-card')
+Html2rss.capture('https://example.com', strategy: :local_file, local_file_path: './page.html')
+```
+
+`strategy: :auto` uses the same AutoFallback chain as scrape (`faraday` → `botasaurus`). When AutoFallback selects a concrete strategy (or you pin one), Capture **stamps** `strategy:` into the emitted config so later `Html2rss.feed(config)` replays the same transport.
+
+## CLI
+
+```bash
+html2rss capture https://example.com/articles
+html2rss capture https://example.com --strategy botasaurus
+html2rss capture https://example.com/articles > my-feed.yml
+html2rss capture https://example.com --input ./page.html
+html2rss capture https://example.com --explain   # quality JSON on stderr; YAML on stdout
+```
+
+## How it works
+
+1. **Request** — `FeedPipeline` (AutoFallback when `:auto`)
+2. **Discover** — AutoSource extracts admitted articles
+3. **Segment** — try SST Segmenter strategies `:list` → `:cluster` → `:semantic`
+4. **Gate** — emit items selector only when ≥ `MIN_SELECTOR_MATCHES` (2) articles match
+5. **Assemble** — `{ items: { selector:, enhance: true } }` plus channel
+
+## Constraints
+
+- Selector quality depends on page structure; treat output as a first draft.
+- When the quality gate fails, selectors are omitted (`has_selectors: false`) rather than inventing attribute selectors.
+- Botasaurus hops need `BOTASAURUS_SCRAPER_URL`.
+
+See also {Html2rss::AutoSource} for article discovery and {Html2rss::FeedPipeline} for the `:auto` request chain.