html2rss 0.32.4 → 0.34.0

data/lib/html2rss/auto_source/segmenter/semantic.rb

@@ -9,24 +9,33 @@ module Html2rss
         # Tag names collected as leaf semantic containers.
         CANDIDATE_NAMES = %i[article section li tr div].to_set.freeze
 
-        module_function
-
         ##
         # @param segmenter [Segmenter]
         # @return [Array<Segment>]
-        def call(segmenter)
-          containers = collect_containers(segmenter.index)
-          primary = PrimaryLink.new(segmenter)
+        def self.call(segmenter)
+          picks = linked_containers(segmenter)
+          # A link wrapping its container is a whole card: it roots one segment, not one per div inside
+          # it, and none when a container already selected that link from inside.
+          claimed = picks.filter_map { |_, link, inside| link if inside }.to_set.compare_by_identity
+
+          picks.each_with_index.filter_map do |(container, link, inside), position|
+            root = inside ? container : (link if claimed.add?(link))
+            Segment.build(root_node: root, primary_link: link, strategy: :semantic, position:) if root
+          end
+        end
 
-          containers.each_with_index.filter_map do |container, position|
+        # @return [Array<Array(SST::Node, SST::Node, Boolean)>] container, its primary link, link inside it
+        def self.linked_containers(segmenter)
+          index = segmenter.index
+          primary = PrimaryLink.new(segmenter)
+          collect_containers(index).filter_map do |container|
             link = primary.select(container)
-            next unless link || segmenter.permit_unanchored
-
-            Segment.build(root_node: container, primary_link: link, strategy: :semantic, position:)
+            [container, link, index.descendant_of?(link, container)] if link
           end
         end
+        private_class_method :linked_containers
 
-        def collect_containers(index)
+        def self.collect_containers(index)
           non_leaf = mark_non_leaf_candidates(index.root)
           candidates = []
           index.each_node do |node|
@@ -40,14 +49,13 @@ module Html2rss
           candidates = filter_nested(candidates, index)
           sort_by_depth(candidates, index)
         end
-        module_function :collect_containers
         private_class_method :collect_containers
 
         TAG_MASKS = { article: 1, section: 2, li: 4, tr: 8, div: 16 }.freeze
         private_constant :TAG_MASKS
 
         # Single O(N) post-order pass: mark candidate nodes that contain a same-name candidate.
-        def mark_non_leaf_candidates(root) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        def self.mark_non_leaf_candidates(root) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
           non_leaf = Set.new.compare_by_identity
           visit = lambda do |node|
             seen_mask = 0
@@ -62,10 +70,9 @@ module Html2rss
           visit.call(root)
           non_leaf
         end
-        module_function :mark_non_leaf_candidates
         private_class_method :mark_non_leaf_candidates
 
-        def filter_nested(candidates, index)
+        def self.filter_nested(candidates, index)
           candidate_set = {}.compare_by_identity
           candidates.each { |c| candidate_set[c] = true }
           rejected = {}.compare_by_identity
@@ -78,25 +85,22 @@ module Html2rss
 
           candidates.reject { |c| rejected[c] }
         end
-        module_function :filter_nested
         private_class_method :filter_nested
 
-        def reject_ancestors(node, candidate_set, rejected, index)
+        def self.reject_ancestors(node, candidate_set, rejected, index)
           curr = index.parent_of(node)
           while curr && curr.name != :html
             rejected[curr] = true if candidate_set[curr]
             curr = index.parent_of(curr)
           end
         end
-        module_function :reject_ancestors
         private_class_method :reject_ancestors
 
-        def sort_by_depth(candidates, index)
+        def self.sort_by_depth(candidates, index)
           candidates.each_with_index
                     .sort_by { |node, i| [-index.depth_of(node), i] }
                     .map(&:first)
         end
-        module_function :sort_by_depth
         private_class_method :sort_by_depth
       end
     end

data/lib/html2rss/auto_source/segmenter.rb

@@ -14,11 +14,10 @@ module Html2rss
       # @param base_url [String, Html2rss::Url]
       # @param strategy [Symbol]
       # @param opts [Hash] segmentation options
-      # @option opts [Boolean] :permit_unanchored keep containers without a primary link
       # @option opts [Integer] :minimum_selector_frequency list strategy frequency floor
       # @option opts [Integer] :use_top_selectors list strategy selector budget
       # @option opts [Scoring::LinkResolver, nil] :link_resolver page-scoped destination facts cache
-      # @return [Array<Segment>]
+      # @return [Array<Segment>] linked segments only
       # rubocop:disable-next Style/ArgumentsForwarding -- keep named opts for YARD @option
       def self.call(document, base_url:, strategy:, **opts)
         new(document, base_url:, strategy:, **opts).call
@@ -28,18 +27,15 @@ module Html2rss
       # @param base_url [String, Html2rss::Url]
       # @param strategy [Symbol]
       # @param opts [Hash] segmentation options
-      # @option opts [Boolean] :permit_unanchored keep containers without a primary link
       # @option opts [Integer] :minimum_selector_frequency list strategy frequency floor
       # @option opts [Integer] :use_top_selectors list strategy selector budget
       # @option opts [Scoring::LinkResolver, nil] :link_resolver page-scoped destination facts cache
       def initialize(document, base_url:, strategy:, **opts)
-        raise ArgumentError, 'document must be SST::Document' unless document.is_a?(SST::Document)
-        raise ArgumentError, "unknown strategy: #{strategy.inspect}" unless Segment::STRATEGIES.include?(strategy)
+        raise Segment::InvalidError, 'document must be SST::Document' unless document.is_a?(SST::Document)
 
         @document = document
         @base_url = base_url
-        @strategy = strategy
-        @permit_unanchored = opts.fetch(:permit_unanchored, false)
+        @strategy = Segment.strategy!(strategy)
         @minimum_selector_frequency = opts.fetch(:minimum_selector_frequency, 2)
         @use_top_selectors = opts.fetch(:use_top_selectors, 5)
         @link_resolver = opts[:link_resolver] || Scoring::LinkResolver.new(base_url)
@@ -55,7 +51,7 @@ module Html2rss
         end
       end
 
-      attr_reader :document, :base_url, :permit_unanchored, :minimum_selector_frequency,
+      attr_reader :document, :base_url, :minimum_selector_frequency,
                   :use_top_selectors, :link_resolver, :noise_policy
 
       # @return [SST::Index]

data/lib/html2rss/auto_source.rb

@@ -22,7 +22,13 @@ module Html2rss
     # Default max articles to keep (also the short-circuit floor across scraper tiers).
     DEFAULT_LIMIT = 25
 
-    # Operational scraper failures quarantined within a tier; programmer errors propagate.
+    # Raised when a scraper yields an item that is neither an Article nor a Hash (a scraper bug).
+    class ScraperContractError < Html2rss::ContractError; end
+
+    # Operational scraper failures quarantined within a tier; programmer errors (including any
+    # {Html2rss::ContractError}) propagate.
+    # +ArgumentError+ stays for stdlib raises on publisher bytes (e.g. +invalid byte sequence+ after
+    # +CGI.unescapeHTML+ of a surrogate entity); lib raises are {Html2rss::Error} (CONTEXT.md § Error handling).
     OPERATIONAL_ERRORS = [
       Html2rss::Error,
       ArgumentError,
@@ -71,14 +77,12 @@ module Html2rss
           enabled: true
         },
         semantic_html: {
-          enabled: true,
-          fallback_anchorless: true
+          enabled: true
         },
         html: {
           enabled: true,
           minimum_selector_frequency: Scraper::Html::DEFAULT_MINIMUM_SELECTOR_FREQUENCY,
-          use_top_selectors: Scraper::Html::DEFAULT_USE_TOP_SELECTORS,
-          fallback_anchorless: true
+          use_top_selectors: Scraper::Html::DEFAULT_USE_TOP_SELECTORS
         }
       },
       cleanup: Cleanup::DEFAULT_CONFIG
@@ -120,14 +124,18 @@ module Html2rss
       @captured_responses = response.captured_responses
       @opts = opts
       @request_session = request_session
+      @cards_read = false
     end
 
     ##
     # Extracts articles by running scraper tiers until a sufficient set is found.
     #
     # Tiers: in-page structured → follow-up IO → SemanticHtml → Html.
-    # SST is built only when a heuristic tier runs. Later tiers are skipped once
-    # +limit+ articles with url+title remain after Cleanup; the result is capped to +limit+.
+    # SST is built at most once, lazily, by the first consumer: a heuristic tier or the
+    # {Corroboration} page check of structured-tier items. Later tiers are skipped once
+    # +limit+ counting articles with url+title remain after Cleanup ({#counting}); the result
+    # is capped to +limit+, counting articles first. A native feed's items come first, and
+    # Cleanup merges a page copy of the same article into them.
     # Html is skipped when earlier tiers already admitted at least one clean article
     # below +limit+ — quality over padding with weaker heuristic junk.
     #
@@ -153,6 +161,13 @@ module Html2rss
       @admission_drops || {}
     end
 
+    ##
+    # What the native feed this page advertises states about its channel ({Scraper::NativeFeed#channel}):
+    # nil until {#articles} runs, or when no feed was read.
+    #
+    # @return [Html2rss::Syndication::ChannelFacts, nil]
+    def feed_channel = @native_feed&.channel
+
     private
 
     attr_reader :url, :parsed_body, :body, :request_session, :captured_responses
@@ -163,21 +178,13 @@ module Html2rss
 
       articles = []
       matched = false
-      document = nil
-      link_resolver = nil
       scraper_opts = @opts.fetch(:scraper, {})
 
       Scraper::SCRAPER_TIERS.each do |tier|
         break if enough_articles?(articles)
         break if skip_html_padding?(tier, articles)
 
-        if Scraper.heuristic_tier?(tier)
-          document ||= Scraper.normalize_sst(parsed_body)
-          next unless document
-
-          link_resolver ||= ::Html2rss::Scoring::LinkResolver.new(url)
-        end
-
+        @cards_read = true if Scraper.heuristic_tier?(tier)
         tier.each do |scraper_class|
           instance = Scraper.build_instance(
             scraper_class,
@@ -186,7 +193,7 @@ module Html2rss
             url:,
             request_session:,
             body:,
-            document:,
+            document: (page_document if Scraper.heuristic_tier?(tier)),
             link_resolver:,
             captured_responses:
           )
@@ -195,6 +202,7 @@ module Html2rss
 
           matched = true
           articles.concat(run_scraper(instance))
+          @native_feed = instance if instance.is_a?(Scraper::NativeFeed)
         end
       end
 
@@ -231,10 +239,11 @@ module Html2rss
 
     def finish_articles(articles)
       captured_count = articles.size
-      result = Cleanup.call(articles, url:, **cleanup_options)
+      result = cleanup_result(articles)
       @admission_drops = result.drop_tallies
       log_admission_emptied(captured_count, result) if captured_count.positive? && result.articles.empty?
-      result.articles.first(article_limit)
+      counting = counting(result.articles)
+      (counting + (result.articles - counting)).first(article_limit)
     end
 
     def log_admission_emptied(captured_count, result)
@@ -252,7 +261,19 @@ module Html2rss
     def enough_articles?(articles)
       return false if articles.size < article_limit
 
-      admitted_articles(articles).size >= article_limit
+      counting(admitted_articles(articles)).size >= article_limit
+    end
+
+    # The articles that count toward +limit+, in order: those the page shows in its content
+    # ({Corroboration#shown?}), media items (audio, video) only on a media page
+    # ({Corroboration#media_page?}). The others stay admitted after the counting ones. A page is
+    # judged a media page only once a heuristic tier has read its cards: structured items alone
+    # cannot tell a podcast page from a news page with an audio list. A media page's counting
+    # items are in page order ({Corroboration#page_order}), not tier order.
+    def counting(articles)
+      media_page = @cards_read && corroboration.media_page?(articles)
+      counted = articles.select { (media_page || !_1.media?) && corroboration.shown?(_1) }
+      media_page ? corroboration.page_order(counted) : counted
     end
 
     # Prefer fewer clean items over refilling +limit+ from the Html heuristic tier.
@@ -264,8 +285,39 @@ module Html2rss
       tier == [Scraper::Html]
     end
 
+    # Cleanup is deterministic and +articles+ only grows, so one result per size is reused
+    # by the tier checks and by {#finish_articles}.
+    def cleanup_result(articles)
+      return @cleanup_memo.last if @cleanup_memo&.first == articles.size
+
+      Cleanup.call(articles.dup, url:, corroboration:, page_links: -> { page_links },
+                                 feed_url: @native_feed&.feed_url, **cleanup_options)
+             .tap { @cleanup_memo = [articles.size, _1] }
+    end
+
+    # Page SST shared by the heuristic tiers and the page check; nil for a JSON response
+    # or a page without SST nodes.
+    def page_document
+      return @page_document if defined?(@page_document)
+
+      @page_document = (SST::Normalizer.call(parsed_body) unless json_response?)
+    end
+
+    def link_resolver
+      @link_resolver ||= Scoring::LinkResolver.new(url)
+    end
+
+    def corroboration
+      @corroboration ||= Corroboration.new { page_links }
+    end
+
+    # Rendered links of the page, built once for the page check and sponsorship.
+    def page_links
+      @page_links ||= PageLinks.build(page_document, link_resolver:)
+    end
+
     def admitted_articles(articles)
-      Cleanup.call(articles.dup, url:, **cleanup_options).articles.select do |article|
+      cleanup_result(articles).articles.select do |article|
         article.url && !article.title.to_s.empty?
       end
     end
@@ -276,11 +328,11 @@ module Html2rss
 
     def run_scraper(instance)
       instance.map { |item| article_from_scraper_item(item, instance) }
+    rescue ContractError
+      raise
     rescue *OPERATIONAL_ERRORS => error
       Log.warn({
-        event_name: 'auto_source.quarantined',
-        host: url.host,
-        error_class: error.class.name,
+        event_name: 'auto_source.quarantined', host: url.host, error_class: error.class.name,
         message: "#{self.class}: #{instance.class} quarantined #{error.class}: #{error.message}"
       }.to_json)
       []
@@ -291,9 +343,12 @@ module Html2rss
       when Article
         item
       when Hash
-        Article.new(**item, scraper: instance.class)
+        # Hash items carry publisher-declared or structured titles: verbatim (collapse only), except an item
+        # with an explicit +kicker+, which composes +Kicker: Headline+. Heuristic scrapers yield Articles
+        # composed by Html::SstArticleExtractor.
+        Article.new(**item, compose_title: item.key?(:kicker), scraper: instance.class)
       else
-        raise TypeError, "#{instance.class} yielded #{item.class}; expected Article or Hash"
+        raise ScraperContractError, "#{instance.class} yielded #{item.class}; expected Article or Hash"
       end
     end
 

data/lib/html2rss/batch.rb

@@ -10,17 +10,9 @@ module Html2rss
     # Default worker threads for batch operations.
     DEFAULT_CONCURRENCY = 5
 
+    # Outcome of a batch operation; +to_h+ is provided by Data.
     BatchResult = Data.define(:total, :successful, :results)
 
-    # Value object representing the outcome of a batch operation.
-    class BatchResult
-      ##
-      # @return [Hash{Symbol => Object}]
-      def to_h
-        { total:, successful:, results: }
-      end
-    end
-
     class << self
       ##
       # Executes a block concurrently across items using a bounded worker pool.
@@ -31,11 +23,8 @@ module Html2rss
       # @yieldparam item [Object]
       # @return [Array<Object>]
       def map(items, concurrency: DEFAULT_CONCURRENCY, &)
-        list = Array(items)
-        return [] if list.empty?
-
-        worker_count = worker_pool_size(list.size, concurrency)
-        process_queue(list, worker_count, &)
+        Option.integer(:concurrency, concurrency, min: 1, max: MAX_CONCURRENCY)
+        process_queue(Option.list(:items, items), [concurrency, items.size].min, &)
       end
 
       ##
@@ -46,11 +35,10 @@ module Html2rss
       # @yieldparam item [Object]
       # @return [BatchResult]
       def run(items, concurrency: DEFAULT_CONCURRENCY, &)
-        list = Array(items)
-        results = map(list, concurrency:, &)
+        results = map(items, concurrency:, &)
         successful = results.count { |entry| entry.is_a?(Hash) ? entry[:ok] : !entry.nil? }
 
-        BatchResult.new(total: list.size, successful:, results:)
+        BatchResult.new(total: items.size, successful:, results:)
       end
 
       ##
@@ -62,7 +50,7 @@ module Html2rss
       # @param items_selector [String, nil]
       # @return [Hash{Symbol => Object}] +:strategy+, +:items+, +:channel_title+, +:admission_drops+
       def scrape_wire(url:, strategy:, limit:, items_selector: nil) # rubocop:disable Metrics/MethodLength
-        plan = (strategy || :auto).to_sym
+        plan = Option.strategy(strategy)
         feed_args = { strategy: plan, limit: }
         feed_args[:items_selector] = items_selector unless items_selector.nil?
         feed_result = Html2rss.auto_feed_result(url, **feed_args)
@@ -84,7 +72,9 @@ module Html2rss
       # @param concurrency [Integer] number of worker threads (default 5, max 10)
       # @return [BatchResult]
       def batch_scrape(urls:, strategy: :auto, limit: 10, concurrency: DEFAULT_CONCURRENCY)
-        run(urls, concurrency:) { |url| scrape_single_url(url:, strategy:, limit:) }
+        strategy = Option.strategy(strategy)
+        Option.integer(:limit, limit, min: 1)
+        run(Option.list(:urls, urls), concurrency:) { |url| scrape_single_url(url:, strategy:, limit:) }
       end
 
       ##
@@ -95,7 +85,8 @@ module Html2rss
       # @param concurrency [Integer] number of worker threads (default 5, max 10)
       # @return [BatchResult]
       def batch_inspect(urls:, strategy: :auto, concurrency: DEFAULT_CONCURRENCY)
-        reports = PageRecon::Diagnostics.batch(urls:, strategy:, concurrency:)
+        reports = PageRecon::Diagnostics.batch(urls: Option.list(:urls, urls), strategy: Option.strategy(strategy),
+                                               concurrency:)
         results = reports.map(&:to_wire_h)
         successful = reports.count { |report| diagnostic_success?(report) }
 
@@ -111,16 +102,21 @@ module Html2rss
       # @option options [String, nil] :cache_dir optional HTML cache directory
       # @return [BatchResult]
       def batch_recon(urls:, strategy: :auto, concurrency: DEFAULT_CONCURRENCY, **)
-        results = Recon.batch(urls, strategy: (strategy || :auto).to_sym, max_threads: concurrency, **)
-        successful = results.count { |r| r.status ? r.status < 400 : false }
+        Option.integer(:concurrency, concurrency, min: 1, max: MAX_CONCURRENCY)
+        results = Recon.batch(Option.list(:urls, urls), strategy: Option.strategy(strategy),
+                                                        max_threads: concurrency, **)
+        successful = results.count { |result| http_success?(result.status) }
         BatchResult.new(total: results.size, successful:, results: results.map(&:to_h))
       end
 
       private
 
       def diagnostic_success?(report)
-        status = report.to_wire_h[:status]
-        status ? status < 400 : false
+        http_success?(report.to_wire_h[:status])
+      end
+
+      def http_success?(status)
+        !status.nil? && status < 400
       end
 
       def scrape_single_url(url:, strategy:, limit:)
@@ -129,16 +125,12 @@ module Html2rss
           url: url.to_s, ok: true, items_count: wire[:items].size,
           items: wire[:items], channel_title: wire[:channel_title]
         }
-      rescue StandardError => error
+      rescue Html2rss::ContractError
+        raise
+      rescue Html2rss::Error => error
         { url: url.to_s, ok: false, error: error.message }
       end
 
-      def worker_pool_size(item_count, requested_concurrency)
-        return 1 if item_count <= 1
-
-        [[1, requested_concurrency.to_i].max, item_count, MAX_CONCURRENCY].min
-      end
-
       def process_queue(items, worker_count, &)
         queue = build_work_queue(items)
         results = Array.new(items.size)
@@ -157,7 +149,7 @@ module Html2rss
       end
 
       def drain_queue(queue, results)
-        until queue.empty?
+        loop do
           begin
             item, index = queue.pop(true)
           rescue ThreadError

data/lib/html2rss/capture/README.md

@@ -29,19 +29,22 @@ rss = Html2rss.apply(result.config)
 
 `Html2rss.capture` and `Capture.build` both return a `CaptureResult` with YAML (`#yaml`, includes the schema modeline) and quality meta (`has_selectors`, `segment_strategy`, `admission_drops`, `selected_strategy`, `native_feed`, `candidates`).
 
-`#candidates` is ranked discovery evidence: `{ items:, title:, link:, published: }` arrays (best-first, deduplicated). Items entries are `{ selector:, enhance:, sample: }`; field entries are `{ selector:, sample: }`. `sample` stays on `#candidates` only — emitted YAML is still `items` + `enhance`. Buckets may be empty when evidence is missing. The default `a[href]` YAML fallback is not listed as discovered evidence.
+`#candidates` is ranked discovery evidence: `{ items:, title:, link:, published: }` arrays (best-first, deduplicated). Items entries are `{ selector:, enhance:, sample: }`; field entries are `{ selector:, sample: }`. `sample` stays on `#candidates` only — emitted YAML is still `items` + `enhance`. `#candidates[:candidates_by_item]` is additive: for each of the top 5 item candidates (same order as `items`) it holds `{ selector:, title:, link:, published: }`, where the field buckets are computed from that item candidate's own matched nodes (reusing the discovery evidence, no refetch). Use it when the top-level `title` / `link` / `published` (which describe the first item candidate only) do not match the item selector you choose. Its first entry equals the top-level buckets. Buckets may be empty when evidence is missing. The default `a[href]` YAML fallback is not listed as discovered evidence.
 
 **Wire vs internal:** user-facing **`Html2rss.apply`** ships RSS from a config Hash. Pipeline internals **`Html2rss.feed` / `feed_result`** stay unchanged — use `apply` in CLI/MCP/docs examples, not `feed`.
 
 ### Options
 
 ```ruby
-Html2rss.capture('https://spa-site.com', strategy: :botasaurus)
+Html2rss.capture('https://example.com', strategy: :botasaurus)
 Html2rss.capture('https://example.com', items_selector: '.article-card')
 Html2rss.capture('https://example.com', strategy: :local_file, local_file_path: './page.html')
 Html2rss.capture('https://example.com', max_redirects: 8, max_requests: 4)
+Html2rss.capture('https://example.com', total_timeout_seconds: 20)
 ```
 
+`total_timeout_seconds:` (positive Integer, default 30) bounds the **whole** request chain: the `:auto` fallback hops (`default` → `botasaurus`) share one budget, so the bound is not per hop. When it elapses, capture raises `RequestService::RequestTimedOut`. Use it to stay under a host's own request deadline.
+
 `strategy: :auto` uses the same AutoFallback chain as scrape (`default` → `botasaurus`). When AutoFallback selects a concrete strategy (or you pin one), Capture **stamps** `strategy:` into the emitted config so later `Html2rss.apply(config)` replays the same transport.
 
 ## CLI
@@ -66,7 +69,7 @@ html2rss capture https://example.com --explain   # quality JSON on stderr; YAML
 ## Constraints
 
 - Selector quality depends on page structure; treat output as a first draft.
-- When HTML derivation misses (no articles, no SST document, fewer than `MIN_SELECTOR_MATCHES` pairs, or a rescued `ArgumentError`), Capture emits the default items selector with `segment_strategy: :default`. `enhance` still follows admission evidence (false when chrome drops are high). Non-HTML responses still omit selectors.
+- When HTML derivation misses (no articles, no SST document, fewer than `MIN_SELECTOR_MATCHES` pairs), Capture emits the default items selector with `segment_strategy: :default`. `enhance` still follows admission evidence (false when chrome drops are high). Non-HTML responses still omit selectors.
 - Botasaurus hops need `BOTASAURUS_SCRAPER_URL`.
 
 See also {Html2rss::AutoSource} for article discovery and {Html2rss::FeedPipeline} for the `:auto` request chain.