html2rss 0.32.4 → 0.34.0

data/lib/html2rss/auto_source/scraper/sitemap.rb

@@ -7,7 +7,7 @@ module Html2rss
     module Scraper
       ##
       # Scrapes articles from XML sitemaps explicitly linked in HTML or discovered via standard sitemap paths.
-      class Sitemap # rubocop:disable Metrics/ClassLength -- scraper + structured warn paths
+      class Sitemap
         include Enumerable
 
         # Selector for sitemap link tags in HTML head
@@ -43,8 +43,9 @@ module Html2rss
         # @param url [String, Html2rss::Url] canonical page URL
         # @param request_session [Html2rss::RequestSession, nil] shared request session for follow-up fetches
         # @param body [String, nil] raw response body for direct sitemap XML parsing
-        # @option _opts [Float] :min_priority minimum priority score (default: 0.3)
-        # @option _opts [Integer] :max_age_days maximum entry age in days (default: 30)
+        # @param opts [Hash] sitemap options
+        # @option opts [Float] :min_priority minimum priority score (default: 0.3)
+        # @option opts [Integer] :max_age_days maximum entry age in days (default: 30)
         def initialize(parsed_body, url:, request_session: nil, body: nil, **opts)
           @parsed_body = parsed_body
           @body = body
@@ -65,13 +66,7 @@ module Html2rss
         def each
           return enum_for(:each) unless block_given?
 
-          entries = fetch_and_parse_entries
-          return if entries.empty?
-
-          entries.each do |entry|
-            article_hash = article_from_entry(entry)
-            yield article_hash if article_hash
-          end
+          fetch_and_parse_entries.each { |entry| yield article_from_entry(entry) }
         end
 
         private
@@ -137,11 +132,11 @@ module Html2rss
 
         # rubocop:disable-next Metrics/MethodLength -- structured fetch-failed warn
         def fetch_xml(target_url)
-          return unless request_session && target_url
+          return unless request_session
 
           response = request_session.follow_up(url: target_url, relation: :auto_source, origin_url: url)
           response&.body
-        rescue Html2rss::RequestService::RequestBudgetExceeded
+        rescue Html2rss::RequestService::RequestBudgetExceeded, Html2rss::ContractError
           raise
         rescue Html2rss::Error => error
           Log.warn({
@@ -160,8 +155,6 @@ module Html2rss
         end
 
         def article_from_entry(entry)
-          return unless entry.url
-
           {
             url: entry.url,
             title: entry.title,

data/lib/html2rss/auto_source/scraper/wordpress_api/page_scope.rb

@@ -154,7 +154,7 @@ module Html2rss
             end
 
             def archive_like?
-              category_archive? || tag_archive? || author_archive? || date_archive? || body_classes.include?('archive')
+              body_classes.include?('archive')
             end
 
             def singular_like?
@@ -178,7 +178,7 @@ module Html2rss
 
               canonical_url = Html2rss::Url.from_relative(href, url)
               same_origin_url?(canonical_url, url) ? canonical_url : url
-            rescue ArgumentError
+            rescue Html2rss::Url::InvalidUrlError
               url
             end
 

data/lib/html2rss/auto_source/scraper/wordpress_api/posts_endpoint.rb

@@ -54,7 +54,7 @@ module Html2rss
             return log_missing_api_root if href.empty?
 
             Html2rss::Url.from_relative(href, page_url)
-          rescue ArgumentError => error
+          rescue Html2rss::Url::InvalidUrlError => error
             logger.warn("#{WordpressApi}: invalid WordPress API endpoint #{href.inspect} (#{error.message})")
             nil
           end
@@ -110,13 +110,11 @@ module Html2rss
           def normalized_api_path(path)
             segments = path.to_s.split('/').reject(&:empty?)
             normalized_path = "/#{segments.join('/')}"
-            normalized_path = '/' if normalized_path == '/'
             normalized_path.end_with?('/') ? normalized_path : "#{normalized_path}/"
           end
 
           def normalized_rest_route(route)
             value = route.to_s
-            value = '/' if value.empty?
             value = "/#{value}" unless value.start_with?('/')
             trim_trailing_slashes(value)
           end

data/lib/html2rss/auto_source/scraper/wordpress_api.rb

@@ -46,10 +46,8 @@ module Html2rss
         # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
         # @param url [String, Html2rss::Url] canonical page URL
         # @param request_session [Html2rss::RequestSession, nil] shared request session for follow-up fetches
-        # @param _opts [Hash] unused scraper-specific options
-        # @option _opts [Object] :_reserved reserved for future scraper-specific options
         # @return [void]
-        def initialize(parsed_body, url:, request_session: nil, **_opts)
+        def initialize(parsed_body, url:, request_session: nil)
           @parsed_body = parsed_body
           @url = Html2rss::Url.from_absolute(url)
           @request_session = request_session
@@ -109,6 +107,8 @@ module Html2rss
             relation: :auto_source,
             origin_url: url
           )
+        rescue Html2rss::ContractError
+          raise
         rescue Html2rss::Error => error
           Log.warn({
             event_name: 'wordpress_api.fetch_failed',
@@ -132,8 +132,8 @@ module Html2rss
           absolute_link(post[:link])
         end
 
-        def article_id(_post, article_url)
-          root_path_query_id(article_url) || present_string(article_url.path) || article_url.to_s
+        def article_id(article_url)
+          root_path_query_id(article_url) || present_string(article_url.path)
         end
 
         def article_title(post)
@@ -154,7 +154,7 @@ module Html2rss
 
         def article_attributes(post, article_url)
           {
-            id: article_id(post, article_url),
+            id: article_id(article_url),
             title: article_title(post),
             description: article_description(post),
             url: article_url,
@@ -168,7 +168,7 @@ module Html2rss
           return unless value
 
           Html2rss::Url.from_relative(value, url)
-        rescue ArgumentError
+        rescue Html2rss::Url::InvalidUrlError
           nil
         end
 

data/lib/html2rss/auto_source/scraper/xhr_articles.rb

@@ -26,16 +26,14 @@ module Html2rss
         # @param _parsed_body [Nokogiri::HTML::Document, nil] unused HTML document
         # @param url [String, Html2rss::Url] page URL used to resolve relative links
         # @param captured_responses [Array<Hash>] JSON bodies from Response#captured_responses
-        # @param _opts [Hash] scraper-specific options
-        # @option _opts [Object] :_reserved reserved for future scraper-specific options
-        def initialize(_parsed_body, url:, captured_responses: [], **_opts)
+        def initialize(_parsed_body, url:, captured_responses: [])
           @url = url
           @captured_responses = captured_responses
         end
 
         # @return [Boolean] true when any captured body contains article-like arrays
         def extractable?
-          parsed_bodies.any? { |doc| JsonState::CandidateDetector.candidate_array?(doc) }
+          parsed_bodies.any?
         end
 
         # @yield [Hash{Symbol => Object}] normalized article hash

data/lib/html2rss/auto_source/scraper.rb

@@ -38,8 +38,16 @@ module Html2rss
       # Flat ordered list (request slot accounting, detection helpers).
       SCRAPERS = (DOCUMENT_SCRAPERS + SCRAPER_TIERS.flatten).freeze
 
-      # Heuristic scrapers that share one memoized SST::Document per page.
+      # Heuristic scrapers that share one memoized SST::Document per page. The only titles composed
+      # (+Kicker: Headline+, noise stripped): their cards are read by Html::SstArticleExtractor. Every
+      # other scraper yields publisher-declared or structured headlines, kept verbatim (collapse only).
       HEURISTIC_SCRAPERS = Set[SemanticHtml, Html].freeze
+      # In-page structured scrapers whose items {Corroboration} checks against the rendered page.
+      # Publisher-declared sources (NativeFeed, WordpressApi, Sitemap, MetaOembed) stay exempt.
+      CORROBORATED_SCRAPERS = Set[Schema, Microdata, Microformats2, JsonState, XhrArticles, JsonDocument].freeze
+      # Scrapers whose items are read off the rendered page: such an item needs a title ({Cleanup}
+      # drops it as +blank_title+), while a publisher-declared item (feed, API) may be untitled.
+      PAGE_SCRAPERS = (CORROBORATED_SCRAPERS | HEURISTIC_SCRAPERS).freeze
       # Scrapers that accept a shared follow-up +request_session+.
       REQUEST_SESSION_SCRAPERS = Set[NativeFeed, WordpressApi, Sitemap, MetaOembed].freeze
       # Scrapers that consume browser-captured XHR/fetch JSON bodies.
@@ -80,7 +88,7 @@ module Html2rss
           return if CATEGORY_MESSAGES.key?(category)
 
           valid_categories = CATEGORY_MESSAGES.keys.join(', ')
-          raise ArgumentError, "Unknown category: #{category.inspect}. Valid categories are: #{valid_categories}"
+          raise InvalidOption, "Unknown category: #{category.inspect}. Valid categories are: #{valid_categories}"
         end
       end
 
@@ -117,15 +125,6 @@ module Html2rss
         HEURISTIC_SCRAPERS.intersect?(tier)
       end
 
-      ##
-      # @param parsed_body [Nokogiri::HTML::Document]
-      # @return [SST::Document, nil]
-      def self.normalize_sst(parsed_body)
-        SST::Normalizer.call(parsed_body)
-      rescue SST::Normalizer::EmptyTree
-        nil
-      end
-
       ##
       # Builds a scraper when enabled; returns nil when disabled.
       #
@@ -220,8 +219,6 @@ module Html2rss
       private_class_method :app_shell_surface?
 
       def self.high_entropy_surface?(parsed_body)
-        return false if parsed_body.nil?
-
         parsed_body.css('body a[href]').size >= HIGH_ENTROPY_MIN_ANCHORS
       end
       private_class_method :high_entropy_surface?

data/lib/html2rss/auto_source/segment.rb

@@ -7,6 +7,9 @@ module Html2rss
     ##
     # One candidate content block discovered by {Segmenter}.
     class Segment
+      # Raised when a segment (or the Segmenter producing segments) gets invalid input.
+      class InvalidError < Html2rss::ContractError; end
+
       # Allowed Segmenter strategies.
       STRATEGIES = Set[:semantic, :list, :cluster].freeze
 
@@ -16,17 +19,28 @@ module Html2rss
       # @param strategy [Symbol]
       # @param position [Integer]
       # @return [Segment]
-      # @raise [ArgumentError] on invalid construction
+      # @raise [InvalidError] on invalid construction
       def self.build(root_node:, strategy:, position:, primary_link: nil)
-        raise ArgumentError, 'root_node must be SST::Node' unless root_node.is_a?(SST::Node)
+        raise InvalidError, 'root_node must be SST::Node' unless root_node.is_a?(SST::Node)
         unless primary_link.nil? || primary_link.is_a?(SST::Node)
-          raise ArgumentError, 'primary_link must be SST::Node or nil'
+          raise InvalidError, 'primary_link must be SST::Node or nil'
         end
-        raise ArgumentError, "unknown strategy: #{strategy.inspect}" unless STRATEGIES.include?(strategy)
-        raise ArgumentError, 'position must be Integer' unless position.is_a?(Integer)
+
+        strategy!(strategy)
+        raise InvalidError, 'position must be Integer' unless position.is_a?(Integer)
 
         new(root_node:, primary_link:, strategy:, position:)
       end
+
+      ##
+      # @param strategy [Symbol]
+      # @return [Symbol] +strategy+ when it is one of {STRATEGIES}
+      # @raise [InvalidError] for any other value
+      def self.strategy!(strategy)
+        return strategy if STRATEGIES.include?(strategy)
+
+        raise InvalidError, "unknown strategy: #{strategy.inspect}"
+      end
     end
   end
 end

data/lib/html2rss/auto_source/segmenter/cluster.rb

@@ -7,23 +7,21 @@ module Html2rss
       # Class / structure clustering for anchorless pages
       # (port of Discovery::DomClustering + OverlapResolver; group scoring owned by Scoring::ClusterScorer).
       module Cluster
-        module_function
-
         ##
         # @param segmenter [Segmenter]
         # @return [Array<Segment>]
-        def call(segmenter)
+        def self.call(segmenter)
           primary = PrimaryLink.new(segmenter)
           nodes = best_group_nodes(segmenter)
           nodes.each_with_index.filter_map do |node, position|
             link = primary.select(node)
-            next unless link || segmenter.permit_unanchored
+            next unless link
 
             Segment.build(root_node: node, primary_link: link, strategy: :cluster, position:)
           end
         end
 
-        def best_group_nodes(segmenter)
+        def self.best_group_nodes(segmenter)
           groups = collect_groups(segmenter)
           return [] if groups.empty?
 
@@ -33,10 +31,9 @@ module Html2rss
           final_groups = resolver.filter_1_to_1_overlap(non_containers)
           scorer.select_best_group(final_groups)
         end
-        module_function :best_group_nodes
         private_class_method :best_group_nodes
 
-        def collect_groups(segmenter) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
+        def self.collect_groups(segmenter) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
           class_groups = Hash.new { |hash, key| hash[key] = [] }
           structure_groups = Hash.new { |hash, key| hash[key] = [] }
           min = segmenter.minimum_selector_frequency
@@ -57,24 +54,21 @@ module Html2rss
 
           structure_groups.select { |_, nodes| nodes.size >= min }
         end
-        module_function :collect_groups
         private_class_method :collect_groups
 
-        def normalize_class(class_names)
+        def self.normalize_class(class_names)
           return '' if class_names.empty?
 
           class_names.map { ::Html2rss::Html::Probe.fold(_1) }.sort.join(' ')
         end
-        module_function :normalize_class
         private_class_method :normalize_class
 
-        def structure_signature(node)
+        def self.structure_signature(node)
           kids = node.children
           return '' if kids.empty?
 
           kids.map { |child| child.name.to_s }.join('>')
         end
-        module_function :structure_signature
         private_class_method :structure_signature
 
         ##
@@ -121,21 +115,23 @@ module Html2rss
             groups.sort_by { |_key, nodes| -nodes.size }.first(MAX_GROUPS).to_h
           end
 
-          # rubocop:disable-next Metrics/MethodLength
+          # A group is a container of +nodes_b+ when one of its nodes strictly encloses two or more of them.
           def container_of?(nodes_a, nodes_b)
             return false unless @layout_tags.include?(nodes_b.first.name)
 
-            nodes_a.any? do |node_a|
-              count = 0
-              nodes_b.each do |node_b|
-                next if node_a.equal?(node_b)
+            counts = enclosed_counts(nodes_b)
+            nodes_a.any? { |node_a| counts[node_a] > 1 }
+          end
 
-                if @index.descendant_of?(node_b, node_a)
-                  count += 1
-                  break if count > 1
-                end
+          # Number of +nodes+ strictly beneath each ancestor, tallied once per group instead of per pair.
+          def enclosed_counts(nodes)
+            @enclosed_counts ||= {}.compare_by_identity
+            @enclosed_counts[nodes] ||= nodes.each_with_object(Hash.new(0).compare_by_identity) do |node, tally|
+              ancestor = @index.parent_of(node)
+              while ancestor
+                tally[ancestor] += 1
+                ancestor = @index.parent_of(ancestor)
               end
-              count > 1
             end
           end
 

data/lib/html2rss/auto_source/segmenter/list.rb

@@ -7,118 +7,94 @@ module Html2rss
       # Builds repeated-list article segments by grouping cards under a shared parent.
       # When the document has a +main+ or +[role=main]+, anchors outside it are dropped.
       module List
-        module_function
-
         ##
         # @param segmenter [Segmenter]
         # @return [Array<Segment>]
-        def call(segmenter)
-          primary = PrimaryLink.new(segmenter)
-          pairs = article_pairs(segmenter)
-          seen = {}.compare_by_identity
-
-          pairs.each_with_index.filter_map do |(article_tag, selected_anchor), position|
-            next unless seen[article_tag].nil?
-
-            seen[article_tag] = true
-
-            link = selected_anchor || primary.select(article_tag)
-            next unless link || segmenter.permit_unanchored
-
+        def self.call(segmenter)
+          article_pairs(segmenter).each_with_index.map do |(article_tag, link), position|
             Segment.build(root_node: article_tag, primary_link: link, strategy: :list, position:)
           end
         end
 
-        def article_pairs(segmenter)
+        def self.article_pairs(segmenter)
           top_groups(segmenter).flatten(1)
         end
-        module_function :article_pairs
         private_class_method :article_pairs
 
-        def top_groups(segmenter)
+        def self.top_groups(segmenter)
           cards_by_parent(segmenter)
             .select { |_parent, cards| cards.size >= segmenter.minimum_selector_frequency }
             .max_by(segmenter.use_top_selectors) { |_parent, cards| cards.size }
             .map(&:last)
         end
-        module_function :top_groups
         private_class_method :top_groups
 
-        def cards_by_parent(segmenter) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+        def self.cards_by_parent(segmenter)
           index = segmenter.index
           main = main_landmark(index)
           groups = {}.compare_by_identity
+          link_counts = relevant_link_counts(segmenter)
 
-          index.each_node do |node|
-            next unless node.link?
-            next if index.ignored_chrome?(node)
-            next unless relevant_anchor?(segmenter, node)
-            next if main && !inside_main?(index, node, main)
-
-            card = parent_until_boundary(segmenter, node)
-            next unless card
-
-            parent = index.parent_of(card)
-            next unless parent
-
-            (groups[parent] ||= []) << [card, node]
+          index.each_node.select { |node| card_anchor?(segmenter, index, main, node) }.each do |node|
+            card = parent_until_boundary(segmenter, node, link_counts)
+            (groups[index.parent_of(card)] ||= []) << [card, node]
           end
           groups
         end
-        module_function :cards_by_parent
         private_class_method :cards_by_parent
 
-        def main_landmark(index)
+        def self.card_anchor?(segmenter, index, main, node)
+          node.link? && !index.ignored_chrome?(node) && relevant_anchor?(segmenter, node) &&
+            (main.nil? || inside_main?(index, node, main))
+        end
+        private_class_method :card_anchor?
+
+        def self.main_landmark(index)
           index.each_node.find { |node| node.name == :main || node.attrs.raw['role'] == 'main' }
         end
-        module_function :main_landmark
         private_class_method :main_landmark
 
-        def inside_main?(index, node, main)
+        def self.inside_main?(index, node, main)
           node.equal?(main) || index.descendant_of?(node, main)
         end
-        module_function :inside_main?
         private_class_method :inside_main?
 
-        def relevant_anchor?(segmenter, node)
-          facts = segmenter.link_resolver.destination_facts(node)
-          return false unless facts
-
-          text = node.visible_text.to_s.strip
+        def self.relevant_anchor?(segmenter, node)
           !segmenter.noise_policy.noise_anchor?(
-            text:, destination_facts: facts, anchor: node, utility_landmark_ancestor: false
+            text: node.visible_text, destination_facts: segmenter.link_resolver.destination_facts(node),
+            anchor: node
           )
         end
-        module_function :relevant_anchor?
         private_class_method :relevant_anchor?
 
         BOUNDARY_TAGS = Set[:body, :html].freeze
         private_constant :BOUNDARY_TAGS
 
-        def parent_until_boundary(segmenter, node)
+        def self.parent_until_boundary(segmenter, node, link_counts)
           index = segmenter.index
-          link_counts = Hash.new { |hash, curr| hash[curr] = count_relevant_links(segmenter, curr) }
 
           index.parent_until(node, lambda { |curr|
             return true if BOUNDARY_TAGS.include?(curr.name)
-            return false if index.ignored_chrome?(curr)
 
-            parent = index.parent_of(curr)
-            parent && link_counts[parent] > link_counts[curr]
+            link_counts[index.parent_of(curr)] > link_counts[curr]
           })
         end
-        module_function :parent_until_boundary
         private_class_method :parent_until_boundary
 
-        def count_relevant_links(segmenter, node)
-          # Chrome inside a card must not stop the walk before the shared list parent.
-          if node.link?
-            relevant_anchor?(segmenter, node) ? 1 : 0
-          else
-            node.count_descendants { |child| child.link? && relevant_anchor?(segmenter, child) }
-          end
+        # Node-keyed counts shared across every anchor of one run, so shared ancestors are counted once.
+        # Keyed by identity: SST nodes are Data, whose structural +#hash+ walks the whole subtree.
+        def self.relevant_link_counts(segmenter)
+          relevant = Hash.new { |hash, node| hash[node] = relevant_anchor?(segmenter, node) }.compare_by_identity
+          Hash.new { |hash, node| hash[node] = count_relevant_links(node, relevant) }.compare_by_identity
+        end
+        private_class_method :relevant_link_counts
+
+        # Walk nodes are relevant anchors or their non-link ancestors, so a link counts as one.
+        def self.count_relevant_links(node, relevant)
+          return 1 if node.link?
+
+          node.count_descendants { |child| child.link? && relevant[child] }
         end
-        module_function :count_relevant_links
         private_class_method :count_relevant_links
       end
     end

data/lib/html2rss/auto_source/segmenter/primary_link.rb

@@ -4,8 +4,9 @@ module Html2rss
   class AutoSource
     class Segmenter
       ##
-      # Selects the strongest content-like primary link inside a container
-      # (port of Discovery::SemanticAnchorCandidates).
+      # Selects the strongest content-like primary link of a container
+      # (port of Discovery::SemanticAnchorCandidates). A card without a qualifying link of its own
+      # falls back to the anchor that wraps the whole card.
       class PrimaryLink
         # @param segmenter [Segmenter]
         def initialize(segmenter)
@@ -32,11 +33,28 @@ module Html2rss
             winners[dest] = facts if current.nil? || facts[:score] > current[:score]
           end
 
-          winners.values.max_by { |f| f[:score] }&.fetch(:anchor)
+          winners.values.max_by { |f| f[:score] }&.fetch(:anchor) || wrapping_link(container)
         end
 
         private
 
+        # The nearest ancestor link when it wraps a whole card.
+        def wrapping_link(container)
+          link = @index.parent_until(@index.parent_of(container), :link?.to_proc)
+          return unless link && !@index.ignored_chrome?(link) && card_link?(link)
+
+          link if candidate_facts(link, container)
+        end
+
+        # A card link holds the card's heading and no other link. A link holding other links is a
+        # parser-recovered unclosed anchor spanning a region; one without a heading wraps a fragment
+        # of a story (such as one live-blog entry), not a card.
+        def card_link?(link)
+          (@card_links ||= {}.compare_by_identity).fetch(link) do
+            @card_links[link] = link.find(&:heading?) && link.find { |node| node.link? && !node.equal?(link) }.nil?
+          end
+        end
+
         # rubocop:disable-next Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
         def candidate_facts(anchor, container)
           destination = @link_resolver.destination_facts(anchor)
@@ -45,8 +63,8 @@ module Html2rss
           text = anchor.visible_text.to_s.strip
           heading = first_heading(container)
           heading_text = heading ? heading.visible_text.to_s.strip : ''
-          heading_anchor = heading_anchor?(anchor, heading)
           meaningful = text.match?(/\p{Alnum}/)
+          heading_anchor = heading_anchor?(anchor, heading) || (!meaningful && overlay_link?(anchor, heading))
           content_like = destination.content_path
           heading_match = meaningful && heading_text.match?(/\p{Alnum}/) && heading_text == text
 
@@ -73,6 +91,13 @@ module Html2rss
 
           anchor.equal?(heading) || @index.descendant_of?(anchor, heading)
         end
+
+        # An empty overlay link beside the heading (or a wrapper of it) stands in for the heading's link.
+        def overlay_link?(anchor, heading)
+          return false unless heading&.visible_text.to_s.match?(/\p{Alnum}/)
+
+          @index.descendant_of?(heading, @index.parent_of(anchor))
+        end
       end
     end
   end