html2rss 0.23.0 → 0.25.0

data/lib/html2rss/auto_source/segmenter/cluster.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    class Segmenter
+      ##
+      # Class / structure clustering for anchorless pages
+      # (port of Discovery::DomClustering + OverlapResolver; group scoring owned by Scoring::ClusterScorer).
+      module Cluster
+        module_function
+
+        ##
+        # @param segmenter [Segmenter]
+        # @return [Array<Segment>]
+        def call(segmenter)
+          nodes = best_group_nodes(segmenter)
+          nodes.each_with_index.map do |node, position|
+            Segment.build(root_node: node, primary_link: nil, strategy: :cluster, position:)
+          end
+        end
+
+        def best_group_nodes(segmenter)
+          class_groups = collect_class_groups(segmenter)
+          groups = class_groups.empty? ? collect_structure_groups(segmenter) : class_groups
+          return [] if groups.empty?
+
+          resolver = OverlapResolver.new(index: segmenter.index)
+          scorer = Scoring::ClusterScorer.new
+          non_containers = resolver.filter_containers(groups)
+          final_groups = resolver.filter_1_to_1_overlap(non_containers)
+          scorer.select_best_group(final_groups)
+        end
+        module_function :best_group_nodes
+        private_class_method :best_group_nodes
+
+        def collect_class_groups(segmenter) # rubocop:disable Metrics/AbcSize
+          groups = Hash.new { |h, k| h[k] = [] }
+          segmenter.index.each_node do |node|
+            next if SST::Tags::CLUSTER_EXCLUDED_NAMES.include?(node.name)
+            next if segmenter.index.ignored_chrome?(node)
+
+            cls = normalize_class(node.attrs.class_names)
+            groups[cls] << node unless cls.empty?
+          end
+          groups.select { |_, nodes| nodes.size >= segmenter.minimum_selector_frequency }
+        end
+        module_function :collect_class_groups
+        private_class_method :collect_class_groups
+
+        def collect_structure_groups(segmenter) # rubocop:disable Metrics/AbcSize
+          groups = Hash.new { |h, k| h[k] = [] }
+          segmenter.index.each_node do |node|
+            next if SST::Tags::CLUSTER_EXCLUDED_NAMES.include?(node.name)
+            next if segmenter.index.ignored_chrome?(node)
+
+            sig = structure_signature(node)
+            groups[sig] << node unless sig.empty?
+          end
+          groups.select { |_, nodes| nodes.size >= segmenter.minimum_selector_frequency }
+        end
+        module_function :collect_structure_groups
+        private_class_method :collect_structure_groups
+
+        def normalize_class(class_names)
+          return '' if class_names.empty?
+
+          class_names.sort.join(' ')
+        end
+        module_function :normalize_class
+        private_class_method :normalize_class
+
+        def structure_signature(node)
+          kids = node.children
+          return '' if kids.empty?
+
+          kids.map { |c| c.name.to_s }.join('>')
+        end
+        module_function :structure_signature
+        private_class_method :structure_signature
+
+        ##
+        # Filters layout wrapper groups and resolves 1-to-1 nested card wrappers.
+        class OverlapResolver
+          # Cap pairwise work on dense class grids.
+          MAX_GROUPS = 40
+
+          # @param index [SST::Index]
+          def initialize(index:)
+            @index = index
+            @layout_tags = SST::Tags::LAYOUT_NAMES
+            @scorer = Scoring::ClusterScorer.new
+          end
+
+          # @param groups [Hash{String => Array<SST::Node>}]
+          # @return [Hash{String => Array<SST::Node>}]
+          def filter_containers(groups)
+            capped = cap_groups(groups)
+            capped.reject do |cls_a, nodes_a|
+              capped.any? { |cls_b, nodes_b| cls_a != cls_b && container_of?(nodes_a, nodes_b) }
+            end
+          end
+
+          # @param groups [Hash{String => Array<SST::Node>}]
+          # @return [Hash{String => Array<SST::Node>}]
+          def filter_1_to_1_overlap(groups)
+            discarded = {}
+            groups.each_key do |cls_a|
+              groups.each_key do |cls_b|
+                next if cls_a == cls_b || discarded[cls_a] || discarded[cls_b]
+
+                resolve_1_to_1_overlap(cls_a, cls_b, groups, discarded)
+              end
+            end
+            groups.reject { |cls, _| discarded[cls] }
+          end
+
+          private
+
+          def cap_groups(groups)
+            return groups if groups.size <= MAX_GROUPS
+
+            groups.sort_by { |_key, nodes| -nodes.size }.first(MAX_GROUPS).to_h
+          end
+
+          # rubocop:disable Metrics/MethodLength
+          def container_of?(nodes_a, nodes_b)
+            return false unless @layout_tags.include?(nodes_b.first.name)
+
+            nodes_a.any? do |node_a|
+              count = 0
+              nodes_b.each do |node_b|
+                next if node_a.equal?(node_b)
+
+                if @index.descendant_of?(node_b, node_a)
+                  count += 1
+                  break if count > 1
+                end
+              end
+              count > 1
+            end
+          end
+          # rubocop:enable Metrics/MethodLength
+
+          def resolve_1_to_1_overlap(cls_a, cls_b, groups, discarded)
+            nodes_a = groups[cls_a]
+            nodes_b = groups[cls_b]
+            return if nodes_a.size != nodes_b.size
+
+            nested = nodes_a.zip(nodes_b).all? { |a, b| !a.equal?(b) && @index.descendant_of?(b, a) }
+            return unless nested
+
+            discarded[keep_descendant?(nodes_a, nodes_b) ? cls_a : cls_b] = true
+          end
+
+          def keep_descendant?(nodes_a, nodes_b)
+            @scorer.avg_words(nodes_b) >= 0.8 * @scorer.avg_words(nodes_a) &&
+              @layout_tags.include?(nodes_b.first.name)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/segmenter/list.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    class Segmenter
+      ##
+      # Builds repeated-list article segments from tag_path frequency
+      # (port of Discovery::ListCandidates using tag_path instead of xpath).
+      module List
+        module_function
+
+        ##
+        # @param segmenter [Segmenter]
+        # @return [Array<Segment>]
+        def call(segmenter)
+          primary = PrimaryLink.new(segmenter)
+          pairs = article_pairs(segmenter)
+          seen = {}.compare_by_identity
+
+          pairs.each_with_index.filter_map do |(article_tag, selected_anchor), position|
+            next unless seen[article_tag].nil?
+
+            seen[article_tag] = true
+
+            link = selected_anchor || primary.select(article_tag)
+            next unless link || segmenter.permit_unanchored
+
+            Segment.build(root_node: article_tag, primary_link: link, strategy: :list, position:)
+          end
+        end
+
+        def article_pairs(segmenter)
+          by_path = relevant_links_by_path(segmenter)
+          top_paths(segmenter, by_path).flat_map do |path|
+            by_path.fetch(path, []).filter_map do |node|
+              article_tag = parent_until_boundary(segmenter, node)
+              next unless article_tag
+
+              [article_tag, node]
+            end
+          end
+        end
+        module_function :article_pairs
+        private_class_method :article_pairs
+
+        def relevant_links_by_path(segmenter)
+          by_path = Hash.new { |hash, path| hash[path] = [] }
+          segmenter.index.each_node do |node|
+            next unless node.link?
+            next if segmenter.index.ignored_chrome?(node)
+            next unless relevant_anchor?(segmenter, node)
+
+            by_path[node.tag_path] << node
+          end
+          by_path
+        end
+        module_function :relevant_links_by_path
+        private_class_method :relevant_links_by_path
+
+        def top_paths(segmenter, by_path)
+          by_path.select { |_path, nodes| nodes.size >= segmenter.minimum_selector_frequency }
+                 .max_by(segmenter.use_top_selectors) { |_path, nodes| nodes.size }
+                 .map(&:first)
+        end
+        module_function :top_paths
+        private_class_method :top_paths
+
+        def relevant_anchor?(segmenter, node)
+          facts = segmenter.link_resolver.destination_facts(node)
+          return false unless facts
+
+          text = node.visible_text.to_s.strip
+          !segmenter.noise_policy.noise_anchor?(text:, destination_facts: facts, anchor: node)
+        end
+        module_function :relevant_anchor?
+        private_class_method :relevant_anchor?
+
+        def parent_until_boundary(segmenter, node)
+          index = segmenter.index
+          link_counts = Hash.new { |hash, curr| hash[curr] = count_links(curr) }
+
+          index.parent_until(node, lambda { |curr|
+            return true if %i[body html].include?(curr.name)
+            return false if index.ignored_chrome?(curr)
+
+            parent = index.parent_of(curr)
+            parent && link_counts[parent] > link_counts[curr]
+          })
+        end
+        module_function :parent_until_boundary
+        private_class_method :parent_until_boundary
+
+        def count_links(node)
+          node.link? ? 1 : node.descendants.count(&:link?)
+        end
+        module_function :count_links
+        private_class_method :count_links
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/segmenter/primary_link.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    class Segmenter
+      ##
+      # Selects the strongest content-like primary link inside a container
+      # (port of Discovery::SemanticAnchorCandidates).
+      class PrimaryLink
+        # @param segmenter [Segmenter]
+        def initialize(segmenter)
+          @segmenter = segmenter
+          @index = segmenter.index
+          @link_resolver = segmenter.link_resolver
+          @noise_policy = segmenter.noise_policy
+        end
+
+        ##
+        # @param container [SST::Node]
+        # @return [SST::Node, nil]
+        def select(container) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+          winners = {}
+
+          container.find_all(&:link?).each do |anchor|
+            next if @index.ignored_chrome?(anchor)
+
+            facts = candidate_facts(anchor, container)
+            next unless facts
+
+            dest = facts[:destination]
+            current = winners[dest]
+            winners[dest] = facts if current.nil? || facts[:score] > current[:score]
+          end
+
+          winners.values.max_by { |f| f[:score] }&.fetch(:anchor)
+        end
+
+        private
+
+        # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+        def candidate_facts(anchor, container)
+          destination = @link_resolver.destination_facts(anchor)
+          return unless destination
+
+          text = anchor.visible_text.to_s.strip
+          heading = first_heading(container)
+          heading_text = heading ? heading.visible_text.to_s.strip : ''
+          heading_anchor = heading_anchor?(anchor, heading)
+          meaningful = text.match?(/\p{Alnum}/)
+          content_like = destination.content_path
+          heading_match = meaningful && heading_text.match?(/\p{Alnum}/) && heading_text == text
+
+          return if @noise_policy.noise_anchor?(
+            text:, destination_facts: destination, anchor:, container:, heading_anchor:
+          )
+          return unless meaningful || content_like || heading_anchor
+
+          score = Scoring::AnchorScore.score(
+            heading_anchor:,
+            heading_text_match: heading_match,
+            meaningful_text: meaningful,
+            content_like_destination: content_like
+          )
+
+          { anchor:, destination: destination.destination, score: }
+        end
+        # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+
+        def first_heading(container)
+          (@headings ||= {}.compare_by_identity)[container] ||= container.find(&:heading?)
+        end
+
+        def heading_anchor?(anchor, heading)
+          return false unless heading
+
+          anchor.equal?(heading) || @index.descendant_of?(anchor, heading)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/segmenter/semantic.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    class Segmenter
+      ##
+      # Collects leaf semantic containers (port of Discovery::SemanticContainers).
+      module Semantic
+        # Tag names collected as leaf semantic containers.
+        CANDIDATE_NAMES = %i[article section li tr div].to_set.freeze
+
+        module_function
+
+        ##
+        # @param segmenter [Segmenter]
+        # @return [Array<Segment>]
+        def call(segmenter)
+          containers = collect_containers(segmenter.index)
+          primary = PrimaryLink.new(segmenter)
+
+          containers.each_with_index.filter_map do |container, position|
+            link = primary.select(container)
+            next unless link || segmenter.permit_unanchored
+
+            Segment.build(root_node: container, primary_link: link, strategy: :semantic, position:)
+          end
+        end
+
+        def collect_containers(index)
+          non_leaf = mark_non_leaf_candidates(index.root)
+          candidates = []
+          index.each_node do |node|
+            next unless CANDIDATE_NAMES.include?(node.name)
+            next if non_leaf.include?(node)
+            next if index.ignored_chrome?(node)
+
+            candidates << node
+          end
+
+          candidates = filter_nested(candidates, index)
+          sort_by_depth(candidates, index)
+        end
+        module_function :collect_containers
+        private_class_method :collect_containers
+
+        TAG_MASKS = { article: 1, section: 2, li: 4, tr: 8, div: 16 }.freeze
+        private_constant :TAG_MASKS
+
+        # Single O(N) post-order pass: mark candidate nodes that contain a same-name candidate.
+        def mark_non_leaf_candidates(root) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+          non_leaf = Set.new.compare_by_identity
+          visit = lambda do |node|
+            seen_mask = 0
+            node.children.each do |child|
+              child_mask = visit.call(child)
+              seen_mask |= child_mask | (TAG_MASKS[child.name] || 0)
+            end
+            node_mask = TAG_MASKS[node.name]
+            non_leaf.add(node) if node_mask && seen_mask.anybits?(node_mask)
+            seen_mask
+          end
+          visit.call(root)
+          non_leaf
+        end
+        module_function :mark_non_leaf_candidates
+        private_class_method :mark_non_leaf_candidates
+
+        def filter_nested(candidates, index)
+          candidate_set = {}.compare_by_identity
+          candidates.each { |c| candidate_set[c] = true }
+          rejected = {}.compare_by_identity
+
+          candidates.each do |candidate|
+            next if candidate.name == :div
+
+            reject_ancestors(candidate, candidate_set, rejected, index)
+          end
+
+          candidates.reject { |c| rejected[c] }
+        end
+        module_function :filter_nested
+        private_class_method :filter_nested
+
+        def reject_ancestors(node, candidate_set, rejected, index)
+          curr = index.parent_of(node)
+          while curr && curr.name != :html
+            rejected[curr] = true if candidate_set[curr]
+            curr = index.parent_of(curr)
+          end
+        end
+        module_function :reject_ancestors
+        private_class_method :reject_ancestors
+
+        def sort_by_depth(candidates, index)
+          candidates.each_with_index
+                    .sort_by { |node, i| [-index.depth_of(node), i] }
+                    .map(&:first)
+        end
+        module_function :sort_by_depth
+        private_class_method :sort_by_depth
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/segmenter.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    ##
+    # Discovers candidate content segments from an SST::Document.
+    #
+    # Strategies:
+    # - +:semantic+ — leaf article/section/li/tr/div containers with a primary link
+    # - +:list+ — repeated tag_path anchors walked to a shared container boundary
+    # - +:cluster+ — class / structure clustering for anchorless card grids
+    class Segmenter
+      # @param document [SST::Document]
+      # @param base_url [String, Html2rss::Url]
+      # @param strategy [Symbol]
+      # @param opts [Hash] segmentation options
+      # @option opts [Boolean] :permit_unanchored keep containers without a primary link
+      # @option opts [Integer] :minimum_selector_frequency list strategy frequency floor
+      # @option opts [Integer] :use_top_selectors list strategy selector budget
+      # @option opts [Scoring::LinkResolver, nil] :link_resolver page-scoped destination facts cache
+      # @return [Array<Segment>]
+      # rubocop:disable Style/ArgumentsForwarding -- keep named opts for YARD @option
+      def self.call(document, base_url:, strategy:, **opts)
+        new(document, base_url:, strategy:, **opts).call
+      end
+      # rubocop:enable Style/ArgumentsForwarding
+
+      # @param document [SST::Document]
+      # @param base_url [String, Html2rss::Url]
+      # @param strategy [Symbol]
+      # @param opts [Hash] segmentation options
+      # @option opts [Boolean] :permit_unanchored keep containers without a primary link
+      # @option opts [Integer] :minimum_selector_frequency list strategy frequency floor
+      # @option opts [Integer] :use_top_selectors list strategy selector budget
+      # @option opts [Scoring::LinkResolver, nil] :link_resolver page-scoped destination facts cache
+      def initialize(document, base_url:, strategy:, **opts)
+        raise ArgumentError, 'document must be SST::Document' unless document.is_a?(SST::Document)
+        raise ArgumentError, "unknown strategy: #{strategy.inspect}" unless Segment::STRATEGIES.include?(strategy)
+
+        @document = document
+        @base_url = base_url
+        @strategy = strategy
+        @permit_unanchored = opts.fetch(:permit_unanchored, false)
+        @minimum_selector_frequency = opts.fetch(:minimum_selector_frequency, 2)
+        @use_top_selectors = opts.fetch(:use_top_selectors, 5)
+        @link_resolver = opts[:link_resolver] || Scoring::LinkResolver.new(base_url)
+        @noise_policy = LinkDestination::NoisePolicy.new(link_resolver: @link_resolver, index: document.index)
+      end
+
+      # @return [Array<Segment>]
+      def call
+        case @strategy
+        when :semantic then Semantic.call(self)
+        when :list then List.call(self)
+        when :cluster then Cluster.call(self)
+        end
+      end
+
+      attr_reader :document, :base_url, :permit_unanchored, :minimum_selector_frequency,
+                  :use_top_selectors, :link_resolver, :noise_policy
+
+      # @return [SST::Index]
+      def index = document.index
+    end
+  end
+end

data/lib/html2rss/auto_source.rb

@@ -15,17 +15,22 @@ module Html2rss
   # @see Html2rss::AutoSource::Scraper::Schema
   # @see Html2rss::AutoSource::Scraper::SemanticHtml
   # @see Html2rss::AutoSource::Scraper::Html
+  # rubocop:disable Metrics/ClassLength -- defaults + tiered extract stay on the contributor entry type
   class AutoSource
+    # Default max articles to keep (also the short-circuit floor across scraper tiers).
+    DEFAULT_LIMIT = 25
+
     # Default auto-source configuration shipped for scraper and cleanup behavior.
     DEFAULT_CONFIG = {
+      limit: DEFAULT_LIMIT,
       scraper: {
         wordpress_api: {
           enabled: true
         },
         sitemap: {
           enabled: true,
-          min_priority: Discovery::Sitemap::DEFAULT_MIN_PRIORITY,
-          max_age_days: Discovery::Sitemap::DEFAULT_MAX_AGE_DAYS
+          min_priority: Scraper::Sitemap::Parser::DEFAULT_MIN_PRIORITY,
+          max_age_days: Scraper::Sitemap::Parser::DEFAULT_MAX_AGE_DAYS
         },
         schema: {
           enabled: true
@@ -80,28 +85,24 @@ module Html2rss
     # @param response [Html2rss::RequestService::Response] initial page response
     # @param opts [Hash] validated auto-source options
     # @param request_session [Html2rss::RequestSession, nil] shared request session for follow-up fetches
+    # @option opts [Integer] :limit max articles to keep; later tiers stop once this many survive Cleanup
     # @option opts [Hash] :scraper scraper configuration map
     # @option opts [Hash] :cleanup cleanup configuration map
     # @return [void]
     def initialize(response, opts = DEFAULT_CONFIG, request_session: nil)
       @parsed_body = response.parsed_body
+      @body = response.body
       @url = response.url
       @opts = opts
       @request_session = request_session
     end
 
     ##
-    # Extracts article candidates by selecting every scraper that can explain the
-    # page shape, running those scrapers, and normalizing the resulting hashes
-    # into `Article` objects.
-    #
-    # The contributor-facing flow is:
-    # 1. choose scraper instances that match the page
-    # 2. let each scraper collect its own candidates
-    # 3. clean and deduplicate the merged article list
+    # Extracts articles by running scraper tiers until a sufficient set is found.
     #
-    # Scrapers with expensive precomputation, such as `SemanticHtml`, keep that
-    # state on the instance so detection and extraction can reuse the same work.
+    # Tiers: in-page structured → follow-up IO → SemanticHtml → Html.
+    # SST is built only when a heuristic tier runs. Later tiers are skipped once
+    # +limit+ articles with url+title remain after Cleanup; the result is capped to +limit+.
     #
     # @return [Array<Html2rss::Article>] extracted articles
     def articles
@@ -113,22 +114,73 @@ module Html2rss
 
     private
 
-    attr_reader :url, :parsed_body, :request_session
+    attr_reader :url, :parsed_body, :body, :request_session
 
+    # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
     def extract_articles
-      scraper_instances = Scraper.instances_for(parsed_body, url:, request_session:, opts: @opts[:scraper])
-      return [] if scraper_instances.empty?
+      articles = []
+      matched = false
+      document = nil
+      link_resolver = nil
+      scraper_opts = @opts.fetch(:scraper, {})
+
+      Scraper::SCRAPER_TIERS.each do |tier|
+        break if enough_articles?(articles)
+
+        if Scraper.heuristic_tier?(tier)
+          document ||= Scraper.normalize_sst(parsed_body)
+          next unless document
 
-      # Scrapers are run sequentially.
-      articles = scraper_instances.flat_map do |instance|
-        run_scraper(instance)
+          link_resolver ||= ::Html2rss::Scoring::LinkResolver.new(url)
+        end
+
+        tier.each do |scraper_class|
+          instance = Scraper.build_instance(
+            scraper_class,
+            parsed_body,
+            opts: scraper_opts,
+            url:,
+            request_session:,
+            body:,
+            document:,
+            link_resolver:
+          )
+          next unless instance
+          next unless Scraper.extractable_instance?(instance, parsed_body)
+
+          matched = true
+          articles.concat(run_scraper(instance))
+        end
       end
-      Cleanup.call(articles, url:, **cleanup_options)
+
+      raise Scraper.no_scraper_found_for(parsed_body, body:) unless matched
+
+      Cleanup.call(articles, url:, **cleanup_options).first(article_limit)
+    end
+    # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+
+    def enough_articles?(articles)
+      threshold = article_limit
+      return false if articles.size < threshold
+
+      cleaned = Cleanup.call(articles.dup, url:, **cleanup_options)
+      cleaned.count { |article| article.url && !article.title.to_s.empty? } >= threshold
+    end
+
+    def article_limit
+      @opts.fetch(:limit, DEFAULT_LIMIT)
     end
 
     def run_scraper(instance)
-      instance.each.map do |article_hash|
-        Article.new(**article_hash, scraper: instance.class)
+      instance.each.map do |item|
+        case item
+        when Article
+          item
+        when Hash
+          Article.new(**item, scraper: instance.class)
+        else
+          raise TypeError, "#{instance.class} yielded #{item.class}; expected Article or Hash"
+        end
       end
     end
 
@@ -136,4 +188,5 @@ module Html2rss
       @opts.fetch(:cleanup, {})
     end
   end
+  # rubocop:enable Metrics/ClassLength
 end