html2rss 0.33.0 → 0.34.0

data/lib/html2rss/auto_source/scraper/schema/thing.rb

@@ -19,11 +19,40 @@ module Html2rss
             ReviewNewsArticle SatiricalArticle ScholarlyArticle SocialMediaPosting TechArticle
           ].to_set.freeze
 
-          # Schema.org types emitted only as elements of an ItemList (never standalone).
-          ITEM_LIST_TYPES = Set['Event', 'JobPosting', 'VideoObject', 'PodcastEpisode'].freeze
+          # Schema.org media types and the file kind each proves (`contentUrl` is the file; `embedUrl`, the player,
+          # is never an enclosure nor the item URL), so an untyped `contentUrl` without a media extension is still
+          # audio or video.
+          MEDIA_KINDS = {
+            'AudioObject' => :audio, 'PodcastEpisode' => :audio, 'VideoObject' => :video, 'Clip' => :video
+          }.freeze
+
+          # Schema.org media types: items wherever they appear (standalone, in `@graph` or an array, as ItemList
+          # elements), marked `media: true`; their own `contentUrl` (when present) is a media enclosure.
+          MEDIA_TYPES = MEDIA_KINDS.keys.to_set.freeze
+
+          # Types emitted as items wherever they appear: articles and media.
+          ITEM_TYPES = (SUPPORTED_TYPES | MEDIA_TYPES).freeze
+
+          # Keys holding the media objects (object or array) whose `contentUrl` is an enclosure, in choice order.
+          MEDIA_KEYS = %i[video audio associatedMedia].freeze
+
+          # A MIME type (`type/subtype`) at the start of `encodingFormat`; a bare format name such as `mp3` maps
+          # through {Article::Enclosure::MEDIA_TYPES}.
+          MIME_TYPE = %r{\A[a-z]+/[a-z0-9][\w.+-]*}i
+
+          # Schema.org entity types that describe who or what something is, never a piece of content; a
+          # ListItem-wrapped object typed only with these is not a feed item. Organization and Place subtypes
+          # are listed because the wire form carries the subtype, not the parent.
+          ENTITY_TYPES = Set[
+            'Person',
+            'Organization', 'Corporation', 'NGO', 'LocalBusiness', 'GovernmentOrganization',
+            'EducationalOrganization', 'NewsMediaOrganization', 'Brand',
+            'Product', 'ProductGroup', 'Offer',
+            'Place', 'City', 'Country', 'PostalAddress'
+          ].freeze
 
-          # Schema.org types whose own `contentUrl` is the media enclosure.
-          MEDIA_TYPES = Set['VideoObject'].freeze
+          # Schema.org types emitted only as elements of an ItemList (never standalone).
+          ITEM_LIST_TYPES = Set['Event', 'JobPosting'].freeze
 
           # @param schema_object [Hash{Symbol => Object}] parsed schema.org object
           # @param url [String, Html2rss::Url, nil] base URL used for relative normalization
@@ -32,9 +61,10 @@ module Html2rss
             @base_url = normalized_base_url(url)
           end
 
-          # @return [Hash{Symbol => Object}] scraped article attributes
+          # @return [Hash{Symbol => Object}] scraped article attributes; a media object's carry `media: true`
           def call
-            { id:, title:, description:, author:, url:, image:, published_at:, categories:, enclosures: }
+            item = { id:, title:, description:, author:, url:, image:, published_at:, categories:, enclosures: }
+            own_media? ? item.merge(media: true) : item
           end
 
           # @return [String, nil] stable schema object identifier
@@ -94,9 +124,10 @@ module Html2rss
             @categories ||= CategoryExtractor.call(schema_object)
           end
 
-          # @return [Array<Hash{Symbol => Object}>, nil] enclosures from VideoObject/AudioObject `contentUrl`
+          # @return [Array<Hash{Symbol => Object}>, nil] enclosures from the `contentUrl` of nested media objects
+          #   ({MEDIA_KEYS}) and of a media object itself, with declared size and duration; `embedUrl` is never one
           def enclosures
-            nodes = Array(schema_object.values_at(:video, :audio)).flatten
+            nodes = schema_object.values_at(*MEDIA_KEYS).flatten
             nodes << schema_object if own_media?
             found = nodes.filter_map { media_enclosure(_1) }
             found unless found.empty?
@@ -111,28 +142,37 @@ module Html2rss
 
           private
 
-          # @return [Boolean] whether the object itself is a media object carrying its own `contentUrl`
-          def own_media?
-            Schema.normalize_types(schema_object[:@type]).intersect?(MEDIA_TYPES)
-          end
+          # @return [Boolean] whether the object itself is a media object (an item marked media)
+          def own_media? = Schema.normalize_types(schema_object[:@type]).intersect?(MEDIA_TYPES)
 
-          # @param node [Object] VideoObject/AudioObject node
-          # @return [Hash{Symbol => Object}, nil] enclosure attributes, or nil without a usable absolute contentUrl
+          # @param node [Object] media object (VideoObject, AudioObject, MediaObject) or the media item itself
+          # @return [Hash{Symbol => Object}, nil] enclosure attributes, or nil without an http(s) contentUrl
+          #   (an ImageObject is never one: images are {#image})
           def media_enclosure(node)
-            return unless node.is_a?(Hash) && node[:contentUrl].is_a?(String)
+            return unless node.is_a?(Hash) && (src = first_scalar(node[:contentUrl])).is_a?(String)
+            return if (types = Schema.normalize_types(node[:@type])).include?('ImageObject')
 
-            build_media_enclosure(node[:contentUrl], node[:encodingFormat])
+            kind = types.filter_map { MEDIA_KINDS[_1] }.first
+            type = mime_type(node[:encodingFormat])
+            Html2rss::Html::ArticleRules::Enclosure.from_media(src, type, base_url, kind:)&.merge(file_facts(node))
           end
 
-          # @param src [String] contentUrl
-          # @param type [Object] encodingFormat (used when a String)
-          # @return [Hash{Symbol => Object}, nil] enclosure attributes with an absolute url
-          def build_media_enclosure(src, type)
-            found = ::Html2rss::Html::ArticleRules::Enclosure.from_media(src, (type if type.is_a?(String)),
-                                                                         base_url || src)
-            found if found&.dig(:url)&.absolute?
-          rescue Url::InvalidUrlError
-            nil
+          # @param node [Hash] media object
+          # @return [Hash{Symbol => Integer, nil}] its byte length and playback seconds, the duration else that of
+          #   the item itself (a PodcastEpisode's duration)
+          def file_facts(node)
+            duration = Article::Enclosure.parse_duration(node[:duration]) ||
+                       Article::Enclosure.parse_duration(schema_object[:duration])
+            { bytes_length: Article::Enclosure.parse_size(node[:contentSize]), duration: }
+          end
+
+          # @param format [Object] encodingFormat
+          # @return [String, nil] its lowercase MIME type, or the playable type of a media format name (+mp3+,
+          #   +.m4a+); nil for any other name, so the file name decides
+          def mime_type(format)
+            return unless format.is_a?(String)
+
+            format[MIME_TYPE]&.downcase || Article::Enclosure::MEDIA_TYPES[format.strip.downcase.delete_prefix('.')]
           end
 
           # @param value [Object] url candidate
@@ -147,11 +187,12 @@ module Html2rss
             nil
           end
 
-          # @return [Html2rss::Url, nil] first absolute http(s) mainEntityOfPage or @id, without fragment
+          # @return [Html2rss::Url, nil] first absolute http(s) mainEntityOfPage or @id, without fragment; never
+          #   `contentUrl` (a file) or `embedUrl` (a player page)
           def fallback_url
             main = schema_object[:mainEntityOfPage]
             candidates = main.is_a?(Hash) ? [main[:@id], main[:url]] : [main]
-            (candidates + [schema_object[:@id]]).lazy.filter_map { web_url(_1) }.first
+            (candidates << schema_object[:@id]).filter_map { web_url(_1) }.first
           end
 
           # @param value [Object] url candidate

data/lib/html2rss/auto_source/scraper/schema.rb

@@ -25,7 +25,7 @@ module Html2rss
 
         # Canonical Schema.org type names keyed by folded wire forms (case-insensitive lookup).
         CANONICAL_BY_DOWNCASE = begin
-          canonical_types = Thing::SUPPORTED_TYPES | ItemList::SUPPORTED_TYPES | Thing::ITEM_LIST_TYPES |
+          canonical_types = Thing::ITEM_TYPES | ItemList::SUPPORTED_TYPES | Thing::ITEM_LIST_TYPES |
                             DENIED_CONTAINER_TYPES | Set['Product']
           canonical_types.to_h { |type| [::Html2rss::Html::Probe.fold(type), type] }.freeze
         end.freeze
@@ -33,7 +33,7 @@ module Html2rss
         # Pre-compiled regex for supported schema types (short name or schema.org URL; string or array @type).
         # Allows preceding entries in a JSON @type array (e.g. ["WebPage","NewsArticle"]).
         SUPPORTED_TYPES_RE = begin
-          types = Thing::SUPPORTED_TYPES | ItemList::SUPPORTED_TYPES
+          types = Thing::ITEM_TYPES | ItemList::SUPPORTED_TYPES
           type_re = Regexp.union(types.to_a)
           %r{(?i)"@type"\s*:\s*(?:\[\s*(?:"[^"]*"\s*,\s*)*)?"(?:https?://schema\.org/)?(?:#{type_re.source})"}
         end.freeze
@@ -45,7 +45,7 @@ module Html2rss
         EMPTY_TYPES = Set.new.freeze
 
         # Keys whose `{"@id": ...}` stubs are resolved against sibling nodes of the same JSON-LD document.
-        REFERENCE_KEYS = %i[author publisher image].freeze
+        REFERENCE_KEYS = (%i[author publisher image] + Thing::MEDIA_KEYS).freeze
 
         # @return [Symbol] scraper config key
         def self.options_key = :schema
@@ -115,7 +115,7 @@ module Html2rss
             # Prefer article/list extractors over denied containers for multi-type @type arrays
             # (e.g. ["NewsArticle","WebPage"]).
             return ItemList if types.intersect?(ItemList::SUPPORTED_TYPES)
-            return Thing if types.intersect?(Thing::SUPPORTED_TYPES)
+            return Thing if types.intersect?(Thing::ITEM_TYPES)
             return nil if types.intersect?(DENIED_CONTAINER_TYPES)
 
             Log.debug("#{name}: unsupported schema object @type=#{schema_object[:@type].inspect}")
@@ -214,12 +214,9 @@ module Html2rss
 
         # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
         # @param url [String, Html2rss::Url] base page URL
-        # @param opts [Hash] scraper-specific options
-        # @option opts [Object] :_reserved reserved for future scraper-specific options
-        def initialize(parsed_body, url:, **opts)
+        def initialize(parsed_body, url:)
           @parsed_body = parsed_body
           @url = url
-          @opts = opts
         end
 
         # @return [Boolean]

data/lib/html2rss/auto_source/scraper/semantic_html/entry_deduplicator.rb

@@ -33,12 +33,8 @@ module Html2rss
           # @return [Html2rss::Article, nil]
           def article_for(entry)
             @article_cache.fetch(entry) do
-              @article_cache[entry] = ::Html2rss::Html::SstArticleExtractor.call(
-                entry,
-                base_url: @url,
-                scraper: @scraper,
-                fallback_anchorless: true
-              )
+              @article_cache[entry] =
+                ::Html2rss::Html::SstArticleExtractor.call(entry, base_url: @url, scraper: @scraper)
             end
           end
 
@@ -83,9 +79,7 @@ module Html2rss
             ]
           end
 
-          def word_count(text)
-            (@word_counts ||= {})[text] ||= text.to_s.scan(/\p{Alnum}+/).size
-          end
+          def word_count(text) = (@word_counts ||= {})[text] ||= WordCount.count(text)
         end
       end
     end

data/lib/html2rss/auto_source/scraper/semantic_html.rb

@@ -32,14 +32,11 @@ module Html2rss
         # @param url [String, Html2rss::Url] base url
         # @param document [SST::Document, nil] memoized SST document from AutoSource
         # @param link_resolver [Scoring::LinkResolver, nil] shared page-scoped resolver
-        # @param opts [Hash] scraper-specific options
-        # @option opts [Boolean] :fallback_anchorless whether to keep containers without a primary anchor
-        def initialize(parsed_body = nil, url:, document: nil, link_resolver: nil, **opts)
+        def initialize(parsed_body = nil, url:, document: nil, link_resolver: nil)
           @parsed_body = parsed_body
           @provided_document = document
           @provided_link_resolver = link_resolver
           @url = url
-          @permit_unanchored = opts.fetch(:fallback_anchorless, false)
         end
 
         ##
@@ -76,7 +73,6 @@ module Html2rss
             document,
             base_url: @url,
             strategy: :semantic,
-            permit_unanchored: @permit_unanchored,
             link_resolver:
           )
           Scoring::Engine.new(link_resolver:).rank_top(segments, limit: TOP_K)

data/lib/html2rss/auto_source/scraper/sitemap.rb

@@ -43,8 +43,9 @@ module Html2rss
         # @param url [String, Html2rss::Url] canonical page URL
         # @param request_session [Html2rss::RequestSession, nil] shared request session for follow-up fetches
         # @param body [String, nil] raw response body for direct sitemap XML parsing
-        # @option _opts [Float] :min_priority minimum priority score (default: 0.3)
-        # @option _opts [Integer] :max_age_days maximum entry age in days (default: 30)
+        # @param opts [Hash] sitemap options
+        # @option opts [Float] :min_priority minimum priority score (default: 0.3)
+        # @option opts [Integer] :max_age_days maximum entry age in days (default: 30)
         def initialize(parsed_body, url:, request_session: nil, body: nil, **opts)
           @parsed_body = parsed_body
           @body = body
@@ -135,7 +136,7 @@ module Html2rss
 
           response = request_session.follow_up(url: target_url, relation: :auto_source, origin_url: url)
           response&.body
-        rescue Html2rss::RequestService::RequestBudgetExceeded
+        rescue Html2rss::RequestService::RequestBudgetExceeded, Html2rss::ContractError
           raise
         rescue Html2rss::Error => error
           Log.warn({

data/lib/html2rss/auto_source/scraper/wordpress_api.rb

@@ -46,10 +46,8 @@ module Html2rss
         # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
         # @param url [String, Html2rss::Url] canonical page URL
         # @param request_session [Html2rss::RequestSession, nil] shared request session for follow-up fetches
-        # @param _opts [Hash] unused scraper-specific options
-        # @option _opts [Object] :_reserved reserved for future scraper-specific options
         # @return [void]
-        def initialize(parsed_body, url:, request_session: nil, **_opts)
+        def initialize(parsed_body, url:, request_session: nil)
           @parsed_body = parsed_body
           @url = Html2rss::Url.from_absolute(url)
           @request_session = request_session
@@ -109,6 +107,8 @@ module Html2rss
             relation: :auto_source,
             origin_url: url
           )
+        rescue Html2rss::ContractError
+          raise
         rescue Html2rss::Error => error
           Log.warn({
             event_name: 'wordpress_api.fetch_failed',

data/lib/html2rss/auto_source/scraper/xhr_articles.rb

@@ -26,9 +26,7 @@ module Html2rss
         # @param _parsed_body [Nokogiri::HTML::Document, nil] unused HTML document
         # @param url [String, Html2rss::Url] page URL used to resolve relative links
         # @param captured_responses [Array<Hash>] JSON bodies from Response#captured_responses
-        # @param _opts [Hash] scraper-specific options
-        # @option _opts [Object] :_reserved reserved for future scraper-specific options
-        def initialize(_parsed_body, url:, captured_responses: [], **_opts)
+        def initialize(_parsed_body, url:, captured_responses: [])
           @url = url
           @captured_responses = captured_responses
         end

data/lib/html2rss/auto_source/scraper.rb

@@ -38,8 +38,16 @@ module Html2rss
       # Flat ordered list (request slot accounting, detection helpers).
       SCRAPERS = (DOCUMENT_SCRAPERS + SCRAPER_TIERS.flatten).freeze
 
-      # Heuristic scrapers that share one memoized SST::Document per page.
+      # Heuristic scrapers that share one memoized SST::Document per page. The only titles composed
+      # (+Kicker: Headline+, noise stripped): their cards are read by Html::SstArticleExtractor. Every
+      # other scraper yields publisher-declared or structured headlines, kept verbatim (collapse only).
       HEURISTIC_SCRAPERS = Set[SemanticHtml, Html].freeze
+      # In-page structured scrapers whose items {Corroboration} checks against the rendered page.
+      # Publisher-declared sources (NativeFeed, WordpressApi, Sitemap, MetaOembed) stay exempt.
+      CORROBORATED_SCRAPERS = Set[Schema, Microdata, Microformats2, JsonState, XhrArticles, JsonDocument].freeze
+      # Scrapers whose items are read off the rendered page: such an item needs a title ({Cleanup}
+      # drops it as +blank_title+), while a publisher-declared item (feed, API) may be untitled.
+      PAGE_SCRAPERS = (CORROBORATED_SCRAPERS | HEURISTIC_SCRAPERS).freeze
       # Scrapers that accept a shared follow-up +request_session+.
       REQUEST_SESSION_SCRAPERS = Set[NativeFeed, WordpressApi, Sitemap, MetaOembed].freeze
       # Scrapers that consume browser-captured XHR/fetch JSON bodies.
@@ -80,7 +88,7 @@ module Html2rss
           return if CATEGORY_MESSAGES.key?(category)
 
           valid_categories = CATEGORY_MESSAGES.keys.join(', ')
-          raise ArgumentError, "Unknown category: #{category.inspect}. Valid categories are: #{valid_categories}"
+          raise InvalidOption, "Unknown category: #{category.inspect}. Valid categories are: #{valid_categories}"
         end
       end
 

data/lib/html2rss/auto_source/segment.rb

@@ -7,6 +7,9 @@ module Html2rss
     ##
     # One candidate content block discovered by {Segmenter}.
     class Segment
+      # Raised when a segment (or the Segmenter producing segments) gets invalid input.
+      class InvalidError < Html2rss::ContractError; end
+
       # Allowed Segmenter strategies.
       STRATEGIES = Set[:semantic, :list, :cluster].freeze
 
@@ -16,17 +19,28 @@ module Html2rss
       # @param strategy [Symbol]
       # @param position [Integer]
       # @return [Segment]
-      # @raise [ArgumentError] on invalid construction
+      # @raise [InvalidError] on invalid construction
       def self.build(root_node:, strategy:, position:, primary_link: nil)
-        raise ArgumentError, 'root_node must be SST::Node' unless root_node.is_a?(SST::Node)
+        raise InvalidError, 'root_node must be SST::Node' unless root_node.is_a?(SST::Node)
         unless primary_link.nil? || primary_link.is_a?(SST::Node)
-          raise ArgumentError, 'primary_link must be SST::Node or nil'
+          raise InvalidError, 'primary_link must be SST::Node or nil'
         end
-        raise ArgumentError, "unknown strategy: #{strategy.inspect}" unless STRATEGIES.include?(strategy)
-        raise ArgumentError, 'position must be Integer' unless position.is_a?(Integer)
+
+        strategy!(strategy)
+        raise InvalidError, 'position must be Integer' unless position.is_a?(Integer)
 
         new(root_node:, primary_link:, strategy:, position:)
       end
+
+      ##
+      # @param strategy [Symbol]
+      # @return [Symbol] +strategy+ when it is one of {STRATEGIES}
+      # @raise [InvalidError] for any other value
+      def self.strategy!(strategy)
+        return strategy if STRATEGIES.include?(strategy)
+
+        raise InvalidError, "unknown strategy: #{strategy.inspect}"
+      end
     end
   end
 end

data/lib/html2rss/auto_source/segmenter/cluster.rb

@@ -15,7 +15,7 @@ module Html2rss
           nodes = best_group_nodes(segmenter)
           nodes.each_with_index.filter_map do |node, position|
             link = primary.select(node)
-            next unless link || segmenter.permit_unanchored
+            next unless link
 
             Segment.build(root_node: node, primary_link: link, strategy: :cluster, position:)
           end
@@ -115,21 +115,23 @@ module Html2rss
             groups.sort_by { |_key, nodes| -nodes.size }.first(MAX_GROUPS).to_h
           end
 
-          # rubocop:disable-next Metrics/MethodLength
+          # A group is a container of +nodes_b+ when one of its nodes strictly encloses two or more of them.
           def container_of?(nodes_a, nodes_b)
             return false unless @layout_tags.include?(nodes_b.first.name)
 
-            nodes_a.any? do |node_a|
-              count = 0
-              nodes_b.each do |node_b|
-                next if node_a.equal?(node_b)
+            counts = enclosed_counts(nodes_b)
+            nodes_a.any? { |node_a| counts[node_a] > 1 }
+          end
 
-                if @index.descendant_of?(node_b, node_a)
-                  count += 1
-                  break if count > 1
-                end
+          # Number of +nodes+ strictly beneath each ancestor, tallied once per group instead of per pair.
+          def enclosed_counts(nodes)
+            @enclosed_counts ||= {}.compare_by_identity
+            @enclosed_counts[nodes] ||= nodes.each_with_object(Hash.new(0).compare_by_identity) do |node, tally|
+              ancestor = @index.parent_of(node)
+              while ancestor
+                tally[ancestor] += 1
+                ancestor = @index.parent_of(ancestor)
               end
-              count > 1
             end
           end
 

data/lib/html2rss/auto_source/segmenter/list.rb

@@ -33,9 +33,10 @@ module Html2rss
           index = segmenter.index
           main = main_landmark(index)
           groups = {}.compare_by_identity
+          link_counts = relevant_link_counts(segmenter)
 
           index.each_node.select { |node| card_anchor?(segmenter, index, main, node) }.each do |node|
-            card = parent_until_boundary(segmenter, node)
+            card = parent_until_boundary(segmenter, node, link_counts)
             (groups[index.parent_of(card)] ||= []) << [card, node]
           end
           groups
@@ -69,9 +70,8 @@ module Html2rss
         BOUNDARY_TAGS = Set[:body, :html].freeze
         private_constant :BOUNDARY_TAGS
 
-        def self.parent_until_boundary(segmenter, node)
+        def self.parent_until_boundary(segmenter, node, link_counts)
           index = segmenter.index
-          link_counts = Hash.new { |hash, curr| hash[curr] = count_relevant_links(segmenter, curr) }
 
           index.parent_until(node, lambda { |curr|
             return true if BOUNDARY_TAGS.include?(curr.name)
@@ -81,11 +81,19 @@ module Html2rss
         end
         private_class_method :parent_until_boundary
 
+        # Node-keyed counts shared across every anchor of one run, so shared ancestors are counted once.
+        # Keyed by identity: SST nodes are Data, whose structural +#hash+ walks the whole subtree.
+        def self.relevant_link_counts(segmenter)
+          relevant = Hash.new { |hash, node| hash[node] = relevant_anchor?(segmenter, node) }.compare_by_identity
+          Hash.new { |hash, node| hash[node] = count_relevant_links(node, relevant) }.compare_by_identity
+        end
+        private_class_method :relevant_link_counts
+
         # Walk nodes are relevant anchors or their non-link ancestors, so a link counts as one.
-        def self.count_relevant_links(segmenter, node)
+        def self.count_relevant_links(node, relevant)
           return 1 if node.link?
 
-          node.count_descendants { |child| child.link? && relevant_anchor?(segmenter, child) }
+          node.count_descendants { |child| child.link? && relevant[child] }
         end
         private_class_method :count_relevant_links
       end

data/lib/html2rss/auto_source/segmenter/primary_link.rb

@@ -4,8 +4,9 @@ module Html2rss
   class AutoSource
     class Segmenter
       ##
-      # Selects the strongest content-like primary link inside a container
-      # (port of Discovery::SemanticAnchorCandidates).
+      # Selects the strongest content-like primary link of a container
+      # (port of Discovery::SemanticAnchorCandidates). A card without a qualifying link of its own
+      # falls back to the anchor that wraps the whole card.
       class PrimaryLink
         # @param segmenter [Segmenter]
         def initialize(segmenter)
@@ -32,11 +33,28 @@ module Html2rss
             winners[dest] = facts if current.nil? || facts[:score] > current[:score]
           end
 
-          winners.values.max_by { |f| f[:score] }&.fetch(:anchor)
+          winners.values.max_by { |f| f[:score] }&.fetch(:anchor) || wrapping_link(container)
         end
 
         private
 
+        # The nearest ancestor link when it wraps a whole card.
+        def wrapping_link(container)
+          link = @index.parent_until(@index.parent_of(container), :link?.to_proc)
+          return unless link && !@index.ignored_chrome?(link) && card_link?(link)
+
+          link if candidate_facts(link, container)
+        end
+
+        # A card link holds the card's heading and no other link. A link holding other links is a
+        # parser-recovered unclosed anchor spanning a region; one without a heading wraps a fragment
+        # of a story (such as one live-blog entry), not a card.
+        def card_link?(link)
+          (@card_links ||= {}.compare_by_identity).fetch(link) do
+            @card_links[link] = link.find(&:heading?) && link.find { |node| node.link? && !node.equal?(link) }.nil?
+          end
+        end
+
         # rubocop:disable-next Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
         def candidate_facts(anchor, container)
           destination = @link_resolver.destination_facts(anchor)
@@ -45,8 +63,8 @@ module Html2rss
           text = anchor.visible_text.to_s.strip
           heading = first_heading(container)
           heading_text = heading ? heading.visible_text.to_s.strip : ''
-          heading_anchor = heading_anchor?(anchor, heading)
           meaningful = text.match?(/\p{Alnum}/)
+          heading_anchor = heading_anchor?(anchor, heading) || (!meaningful && overlay_link?(anchor, heading))
           content_like = destination.content_path
           heading_match = meaningful && heading_text.match?(/\p{Alnum}/) && heading_text == text
 
@@ -73,6 +91,13 @@ module Html2rss
 
           anchor.equal?(heading) || @index.descendant_of?(anchor, heading)
         end
+
+        # An empty overlay link beside the heading (or a wrapper of it) stands in for the heading's link.
+        def overlay_link?(anchor, heading)
+          return false unless heading&.visible_text.to_s.match?(/\p{Alnum}/)
+
+          @index.descendant_of?(heading, @index.parent_of(anchor))
+        end
       end
     end
   end

data/lib/html2rss/auto_source/segmenter/semantic.rb

@@ -13,16 +13,27 @@ module Html2rss
         # @param segmenter [Segmenter]
         # @return [Array<Segment>]
         def self.call(segmenter)
-          containers = collect_containers(segmenter.index)
-          primary = PrimaryLink.new(segmenter)
+          picks = linked_containers(segmenter)
+          # A link wrapping its container is a whole card: it roots one segment, not one per div inside
+          # it, and none when a container already selected that link from inside.
+          claimed = picks.filter_map { |_, link, inside| link if inside }.to_set.compare_by_identity
 
-          containers.each_with_index.filter_map do |container, position|
-            link = primary.select(container)
-            next unless link || segmenter.permit_unanchored
+          picks.each_with_index.filter_map do |(container, link, inside), position|
+            root = inside ? container : (link if claimed.add?(link))
+            Segment.build(root_node: root, primary_link: link, strategy: :semantic, position:) if root
+          end
+        end
 
-            Segment.build(root_node: container, primary_link: link, strategy: :semantic, position:)
+        # @return [Array<Array(SST::Node, SST::Node, Boolean)>] container, its primary link, link inside it
+        def self.linked_containers(segmenter)
+          index = segmenter.index
+          primary = PrimaryLink.new(segmenter)
+          collect_containers(index).filter_map do |container|
+            link = primary.select(container)
+            [container, link, index.descendant_of?(link, container)] if link
           end
         end
+        private_class_method :linked_containers
 
         def self.collect_containers(index)
           non_leaf = mark_non_leaf_candidates(index.root)

data/lib/html2rss/auto_source/segmenter.rb

@@ -14,11 +14,10 @@ module Html2rss
       # @param base_url [String, Html2rss::Url]
       # @param strategy [Symbol]
       # @param opts [Hash] segmentation options
-      # @option opts [Boolean] :permit_unanchored keep containers without a primary link
       # @option opts [Integer] :minimum_selector_frequency list strategy frequency floor
       # @option opts [Integer] :use_top_selectors list strategy selector budget
       # @option opts [Scoring::LinkResolver, nil] :link_resolver page-scoped destination facts cache
-      # @return [Array<Segment>]
+      # @return [Array<Segment>] linked segments only
       # rubocop:disable-next Style/ArgumentsForwarding -- keep named opts for YARD @option
       def self.call(document, base_url:, strategy:, **opts)
         new(document, base_url:, strategy:, **opts).call
@@ -28,18 +27,15 @@ module Html2rss
       # @param base_url [String, Html2rss::Url]
       # @param strategy [Symbol]
       # @param opts [Hash] segmentation options
-      # @option opts [Boolean] :permit_unanchored keep containers without a primary link
       # @option opts [Integer] :minimum_selector_frequency list strategy frequency floor
       # @option opts [Integer] :use_top_selectors list strategy selector budget
       # @option opts [Scoring::LinkResolver, nil] :link_resolver page-scoped destination facts cache
       def initialize(document, base_url:, strategy:, **opts)
-        raise ArgumentError, 'document must be SST::Document' unless document.is_a?(SST::Document)
-        raise ArgumentError, "unknown strategy: #{strategy.inspect}" unless Segment::STRATEGIES.include?(strategy)
+        raise Segment::InvalidError, 'document must be SST::Document' unless document.is_a?(SST::Document)
 
         @document = document
         @base_url = base_url
-        @strategy = strategy
-        @permit_unanchored = opts.fetch(:permit_unanchored, false)
+        @strategy = Segment.strategy!(strategy)
         @minimum_selector_frequency = opts.fetch(:minimum_selector_frequency, 2)
         @use_top_selectors = opts.fetch(:use_top_selectors, 5)
         @link_resolver = opts[:link_resolver] || Scoring::LinkResolver.new(base_url)
@@ -55,7 +51,7 @@ module Html2rss
         end
       end
 
-      attr_reader :document, :base_url, :permit_unanchored, :minimum_selector_frequency,
+      attr_reader :document, :base_url, :minimum_selector_frequency,
                   :use_top_selectors, :link_resolver, :noise_policy
 
       # @return [SST::Index]