html2rss 0.24.0 → 0.26.0

data/lib/html2rss/auto_source/scraper.rb

@@ -11,7 +11,7 @@ module Html2rss
     # Detection is intentionally shallow for most scrapers, but instance-based
     # matching is available for scrapers that need to carry expensive selection
     # state forward into extraction.
-    module Scraper
+    module Scraper # rubocop:disable Metrics/ModuleLength -- tier registry + construction helpers
       # Root markers indicating likely app-shell/client-rendered surfaces.
       APP_SHELL_ROOT_SELECTORS = '#app, #root, #__next, [data-reactroot], [ng-app], [id*="app-shell"]'
       # Maximum anchors tolerated before app-shell detection is considered unlikely.
@@ -19,31 +19,37 @@ module Html2rss
       # Maximum visible text length tolerated for app-shell classification.
       APP_SHELL_MAX_VISIBLE_TEXT_LENGTH = 220
 
-      # Ordered scraper classes considered during auto-source extraction.
-      SCRAPERS = [
-        WordpressApi,
-        Sitemap,
-        Schema,
-        Microdata,
-        Microformats2,
-        JsonState,
-        MetaOembed,
-        SemanticHtml,
-        Html
+      # Extraction tiers: merge within a tier, then stop when AutoSource has enough articles.
+      # Heuristic scrapers are separate tiers so SemanticHtml can satisfy before Html runs.
+      SCRAPER_TIERS = [
+        [Schema, Microdata, Microformats2, JsonState, XhrArticles].freeze,
+        [WordpressApi, Sitemap, MetaOembed].freeze,
+        [SemanticHtml].freeze,
+        [Html].freeze
       ].freeze
 
+      # Flat ordered list (request slot accounting, detection helpers).
+      SCRAPERS = SCRAPER_TIERS.flatten.freeze
+
+      # Heuristic scrapers that share one memoized SST::Document per page.
+      HEURISTIC_SCRAPERS = [SemanticHtml, Html].freeze
+      # Scrapers that accept a shared follow-up +request_session+.
+      REQUEST_SESSION_SCRAPERS = [WordpressApi, Sitemap, MetaOembed].freeze
+      # Scrapers that consume browser-captured XHR/fetch JSON bodies.
+      CAPTURED_RESPONSE_SCRAPERS = [XhrArticles].freeze
+
       ##
       # Error raised when no suitable scraper is found.
       class NoScraperFound < Html2rss::Error
-        # User-facing messages grouped by no-scraper surface category.
+        # Surface diagnostics shared with {Html2rss::NoFeedItemsExtracted} (one string home).
         CATEGORY_MESSAGES = {
-          blocked_surface: 'No scrapers found: blocked surface likely (anti-bot or interstitial). ' \
-                           'Retry with --strategy browserless, try a more specific public listing URL, ' \
+          blocked_surface: 'blocked surface likely (anti-bot or interstitial). ' \
+                           'Target a direct listing URL, configure BOTASAURUS_SCRAPER_URL, ' \
                            'or run from an environment that can complete anti-bot checks.',
-          app_shell: 'No scrapers found: app-shell surface detected (client-rendered page with little or no ' \
-                     'server-rendered article HTML). Retry with --strategy browserless, or target a direct ' \
+          app_shell: 'app-shell surface detected (client-rendered page with little or no ' \
+                     'server-rendered article HTML). Configure BOTASAURUS_SCRAPER_URL or target a direct ' \
                      'listing/update URL instead of a homepage or shell entrypoint.',
-          unsupported_surface: 'No scrapers found: unsupported extraction surface for auto mode. ' \
+          unsupported_surface: 'unsupported extraction surface for auto mode. ' \
                                'Try a direct listing/changelog/category URL, ' \
                                'or use explicit selectors in a feed config.'
         }.freeze
@@ -53,7 +59,7 @@ module Html2rss
         def initialize(message = nil, category: :unsupported_surface)
           validate_category!(category)
           @category = category
-          super(message || CATEGORY_MESSAGES.fetch(@category))
+          super(message || "No scrapers found: #{CATEGORY_MESSAGES.fetch(@category)}")
         end
 
         attr_reader :category
@@ -77,9 +83,11 @@ module Html2rss
       # @option opts [Hash] :microdata scraper toggle and configuration
       # @option opts [Hash] :microformats2 scraper toggle and configuration
       # @option opts [Hash] :json_state scraper toggle and configuration
+      # @option opts [Hash] :xhr_articles scraper toggle and configuration
       # @option opts [Hash] :meta_oembed scraper toggle and configuration
       # @option opts [Hash] :semantic_html scraper toggle and configuration
       # @option opts [Hash] :html scraper toggle and configuration
+      # @option opts [Hash] :sitemap scraper toggle and configuration
       # @return [Array<Class>] An array of scraper classes that can handle the parsed body.
       def self.from(parsed_body, opts = Html2rss::AutoSource::DEFAULT_CONFIG[:scraper])
         scrapers = SCRAPERS.select { |scraper| opts.dig(scraper.options_key, :enabled) }
@@ -90,63 +98,111 @@ module Html2rss
         scrapers
       end
 
-      # Returns scraper instances ready for extraction.
-      # @param parsed_body [Nokogiri::HTML::Document] The parsed HTML document.
-      # @param url [String, Html2rss::Url] The page url.
-      # @param request_session [Html2rss::RequestSession, nil] Shared follow-up session.
-      # @param opts [Hash] The options hash.
+      ##
+      # @param tier [Array<Class>]
+      # @return [Boolean]
+      def self.heuristic_tier?(tier)
+        tier.intersect?(HEURISTIC_SCRAPERS)
+      end
+
+      ##
+      # @param parsed_body [Nokogiri::HTML::Document]
+      # @return [SST::Document, nil]
+      def self.normalize_sst(parsed_body)
+        SST::Normalizer.call(parsed_body)
+      rescue ArgumentError
+        nil
+      end
+
+      ##
+      # Builds a scraper when enabled; returns nil when disabled.
+      #
+      # @param scraper [Class]
+      # @param parsed_body [Nokogiri::HTML::Document]
+      # @param opts [Hash] full scraper options map
+      # @param url [Html2rss::Url, String]
+      # @param request_session [Html2rss::RequestSession, nil]
+      # @param body [String, nil]
+      # @param document [SST::Document, nil]
+      # @param link_resolver [Scoring::LinkResolver, nil]
+      # @param captured_responses [Array<Hash>] browser-captured JSON XHR/fetch bodies
       # @option opts [Hash] :wordpress_api scraper toggle and configuration
       # @option opts [Hash] :schema scraper toggle and configuration
       # @option opts [Hash] :microdata scraper toggle and configuration
       # @option opts [Hash] :microformats2 scraper toggle and configuration
       # @option opts [Hash] :json_state scraper toggle and configuration
+      # @option opts [Hash] :xhr_articles scraper toggle and configuration
       # @option opts [Hash] :meta_oembed scraper toggle and configuration
       # @option opts [Hash] :semantic_html scraper toggle and configuration
       # @option opts [Hash] :html scraper toggle and configuration
-      # @return [Array<Object>] An array of scraper instances that can handle the parsed body.
-      #
-      # `instances_for` is the main entrypoint for extraction. It lets a scraper
-      # decide whether it matches using the same instance that will later yield
-      # article hashes, which keeps precomputed state close to the scraper that
-      # owns it.
-      def self.instances_for(parsed_body, url:, request_session: nil,
-                             opts: Html2rss::AutoSource::DEFAULT_CONFIG[:scraper])
-        instances = SCRAPERS.filter_map do |scraper|
-          next unless opts.dig(scraper.options_key, :enabled)
-
-          instance = scraper.new(parsed_body, url:, request_session:, **opts.fetch(scraper.options_key, {}))
-          next unless extractable_instance?(instance, parsed_body)
-
-          instance
-        end
-
-        raise no_scraper_found_for(parsed_body) if instances.empty?
-
-        instances
+      # @option opts [Hash] :sitemap scraper toggle and configuration
+      # @return [Object, nil]
+      # rubocop:disable Metrics/ParameterLists, Metrics/MethodLength -- construction context for structured and heuristic scrapers
+      def self.build_instance(scraper, parsed_body, opts:, url:, request_session: nil, body: nil, document: nil,
+                              link_resolver: nil, captured_responses: [])
+        return unless opts.dig(scraper.options_key, :enabled)
+        return if HEURISTIC_SCRAPERS.include?(scraper) && document.nil?
+
+        scraper_opts = opts.fetch(scraper.options_key, {}).except(:enabled)
+        kwargs = construction_kwargs(
+          scraper,
+          request_session:,
+          body:,
+          document:,
+          link_resolver:,
+          captured_responses:
+        )
+        scraper.new(parsed_body, url:, **kwargs, **scraper_opts)
       end
+      # rubocop:enable Metrics/ParameterLists, Metrics/MethodLength
 
+      ##
+      # @param instance [Object]
+      # @param parsed_body [Nokogiri::HTML::Document]
+      # @return [Boolean]
       def self.extractable_instance?(instance, parsed_body)
         return instance.extractable? if instance.respond_to?(:extractable?)
 
         instance.class.articles?(parsed_body)
       end
-      private_class_method :extractable_instance?
 
-      def self.no_scraper_found_for(parsed_body)
-        NoScraperFound.new(category: classify_no_scraper_surface(parsed_body))
+      ##
+      # @param parsed_body [Nokogiri::HTML::Document]
+      # @param body [String, nil] raw response body (preferred for blocked-surface checks)
+      # @return [NoScraperFound]
+      def self.no_scraper_found_for(parsed_body, body: nil)
+        NoScraperFound.new(category: classify_no_scraper_surface(parsed_body, body:))
       end
-      private_class_method :no_scraper_found_for
 
-      def self.classify_no_scraper_surface(parsed_body)
-        return :blocked_surface if blocked_surface?(parsed_body)
+      def self.construction_kwargs(scraper, request_session:, body:, document:, link_resolver:, # rubocop:disable Metrics/ParameterLists -- scraper construction bag
+                                   captured_responses:)
+        if HEURISTIC_SCRAPERS.include?(scraper)
+          { document:, link_resolver: }.compact
+        else
+          {}.tap do |kwargs|
+            kwargs[:request_session] = request_session if REQUEST_SESSION_SCRAPERS.include?(scraper)
+            kwargs[:body] = body if scraper == Sitemap
+            kwargs[:captured_responses] = captured_responses if CAPTURED_RESPONSE_SCRAPERS.include?(scraper)
+          end
+        end
+      end
+      private_class_method :construction_kwargs
+
+      ##
+      # Classifies why scrapers could not extract from a parsed page.
+      #
+      # @param parsed_body [Nokogiri::HTML::Document]
+      # @param body [String, nil] raw body for blocked-surface detection
+      # @return [Symbol] one of {NoScraperFound::CATEGORY_MESSAGES} keys
+      def self.classify_no_scraper_surface(parsed_body, body: nil)
+        return :blocked_surface if blocked_surface?(parsed_body, body:)
         return :app_shell if app_shell_surface?(parsed_body)
 
         :unsupported_surface
       end
-      private_class_method :classify_no_scraper_surface
 
-      def self.blocked_surface?(parsed_body)
-        Html2rss::RequestService::BlockedSurface.interstitial?(parsed_body.to_html)
+      def self.blocked_surface?(parsed_body, body: nil)
+        Html2rss::RequestService::BlockedSurface.interstitial?(body || parsed_body.to_html)
       end
       private_class_method :blocked_surface?
 

data/lib/html2rss/auto_source/segment.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    ##
+    # One candidate content block discovered by {Segmenter}.
+    Segment = Data.define(:root_node, :primary_link, :strategy, :position) do
+      ##
+      # @param root_node [SST::Node]
+      # @param primary_link [SST::Node, nil]
+      # @param strategy [Symbol]
+      # @param position [Integer]
+      # @return [Segment]
+      # @raise [ArgumentError] on invalid construction
+      def self.build(root_node:, strategy:, position:, primary_link: nil)
+        raise ArgumentError, 'root_node must be SST::Node' unless root_node.is_a?(SST::Node)
+        unless primary_link.nil? || primary_link.is_a?(SST::Node)
+          raise ArgumentError, 'primary_link must be SST::Node or nil'
+        end
+        raise ArgumentError, "unknown strategy: #{strategy.inspect}" unless Segment::STRATEGIES.include?(strategy)
+        raise ArgumentError, 'position must be Integer' unless position.is_a?(Integer)
+
+        new(root_node:, primary_link:, strategy:, position:)
+      end
+    end
+    # Allowed Segmenter strategies.
+    Segment::STRATEGIES = %i[semantic list cluster].to_set.freeze
+  end
+end

data/lib/html2rss/auto_source/segmenter/cluster.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    class Segmenter
+      ##
+      # Class / structure clustering for anchorless pages
+      # (port of Discovery::DomClustering + OverlapResolver; group scoring owned by Scoring::ClusterScorer).
+      module Cluster
+        module_function
+
+        ##
+        # @param segmenter [Segmenter]
+        # @return [Array<Segment>]
+        def call(segmenter)
+          nodes = best_group_nodes(segmenter)
+          nodes.each_with_index.map do |node, position|
+            Segment.build(root_node: node, primary_link: nil, strategy: :cluster, position:)
+          end
+        end
+
+        def best_group_nodes(segmenter)
+          class_groups = collect_class_groups(segmenter)
+          groups = class_groups.empty? ? collect_structure_groups(segmenter) : class_groups
+          return [] if groups.empty?
+
+          resolver = OverlapResolver.new(index: segmenter.index)
+          scorer = Scoring::ClusterScorer.new
+          non_containers = resolver.filter_containers(groups)
+          final_groups = resolver.filter_1_to_1_overlap(non_containers)
+          scorer.select_best_group(final_groups)
+        end
+        module_function :best_group_nodes
+        private_class_method :best_group_nodes
+
+        def collect_class_groups(segmenter) # rubocop:disable Metrics/AbcSize
+          groups = Hash.new { |h, k| h[k] = [] }
+          segmenter.index.each_node do |node|
+            next if SST::Tags::CLUSTER_EXCLUDED_NAMES.include?(node.name)
+            next if segmenter.index.ignored_chrome?(node)
+
+            cls = normalize_class(node.attrs.class_names)
+            groups[cls] << node unless cls.empty?
+          end
+          groups.select { |_, nodes| nodes.size >= segmenter.minimum_selector_frequency }
+        end
+        module_function :collect_class_groups
+        private_class_method :collect_class_groups
+
+        def collect_structure_groups(segmenter) # rubocop:disable Metrics/AbcSize
+          groups = Hash.new { |h, k| h[k] = [] }
+          segmenter.index.each_node do |node|
+            next if SST::Tags::CLUSTER_EXCLUDED_NAMES.include?(node.name)
+            next if segmenter.index.ignored_chrome?(node)
+
+            sig = structure_signature(node)
+            groups[sig] << node unless sig.empty?
+          end
+          groups.select { |_, nodes| nodes.size >= segmenter.minimum_selector_frequency }
+        end
+        module_function :collect_structure_groups
+        private_class_method :collect_structure_groups
+
+        def normalize_class(class_names)
+          return '' if class_names.empty?
+
+          class_names.sort.join(' ')
+        end
+        module_function :normalize_class
+        private_class_method :normalize_class
+
+        def structure_signature(node)
+          kids = node.children
+          return '' if kids.empty?
+
+          kids.map { |c| c.name.to_s }.join('>')
+        end
+        module_function :structure_signature
+        private_class_method :structure_signature
+
+        ##
+        # Filters layout wrapper groups and resolves 1-to-1 nested card wrappers.
+        class OverlapResolver
+          # Cap pairwise work on dense class grids.
+          MAX_GROUPS = 40
+
+          # @param index [SST::Index]
+          def initialize(index:)
+            @index = index
+            @layout_tags = SST::Tags::LAYOUT_NAMES
+            @scorer = Scoring::ClusterScorer.new
+          end
+
+          # @param groups [Hash{String => Array<SST::Node>}]
+          # @return [Hash{String => Array<SST::Node>}]
+          def filter_containers(groups)
+            capped = cap_groups(groups)
+            capped.reject do |cls_a, nodes_a|
+              capped.any? { |cls_b, nodes_b| cls_a != cls_b && container_of?(nodes_a, nodes_b) }
+            end
+          end
+
+          # @param groups [Hash{String => Array<SST::Node>}]
+          # @return [Hash{String => Array<SST::Node>}]
+          def filter_1_to_1_overlap(groups)
+            discarded = {}
+            groups.each_key do |cls_a|
+              groups.each_key do |cls_b|
+                next if cls_a == cls_b || discarded[cls_a] || discarded[cls_b]
+
+                resolve_1_to_1_overlap(cls_a, cls_b, groups, discarded)
+              end
+            end
+            groups.reject { |cls, _| discarded[cls] }
+          end
+
+          private
+
+          def cap_groups(groups)
+            return groups if groups.size <= MAX_GROUPS
+
+            groups.sort_by { |_key, nodes| -nodes.size }.first(MAX_GROUPS).to_h
+          end
+
+          # rubocop:disable Metrics/MethodLength
+          def container_of?(nodes_a, nodes_b)
+            return false unless @layout_tags.include?(nodes_b.first.name)
+
+            nodes_a.any? do |node_a|
+              count = 0
+              nodes_b.each do |node_b|
+                next if node_a.equal?(node_b)
+
+                if @index.descendant_of?(node_b, node_a)
+                  count += 1
+                  break if count > 1
+                end
+              end
+              count > 1
+            end
+          end
+          # rubocop:enable Metrics/MethodLength
+
+          def resolve_1_to_1_overlap(cls_a, cls_b, groups, discarded)
+            nodes_a = groups[cls_a]
+            nodes_b = groups[cls_b]
+            return if nodes_a.size != nodes_b.size
+
+            nested = nodes_a.zip(nodes_b).all? { |a, b| !a.equal?(b) && @index.descendant_of?(b, a) }
+            return unless nested
+
+            discarded[keep_descendant?(nodes_a, nodes_b) ? cls_a : cls_b] = true
+          end
+
+          def keep_descendant?(nodes_a, nodes_b)
+            @scorer.avg_words(nodes_b) >= 0.8 * @scorer.avg_words(nodes_a) &&
+              @layout_tags.include?(nodes_b.first.name)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/segmenter/list.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    class Segmenter
+      ##
+      # Builds repeated-list article segments from tag_path frequency
+      # (port of Discovery::ListCandidates using tag_path instead of xpath).
+      module List
+        module_function
+
+        ##
+        # @param segmenter [Segmenter]
+        # @return [Array<Segment>]
+        def call(segmenter)
+          primary = PrimaryLink.new(segmenter)
+          pairs = article_pairs(segmenter)
+          seen = {}.compare_by_identity
+
+          pairs.each_with_index.filter_map do |(article_tag, selected_anchor), position|
+            next unless seen[article_tag].nil?
+
+            seen[article_tag] = true
+
+            link = selected_anchor || primary.select(article_tag)
+            next unless link || segmenter.permit_unanchored
+
+            Segment.build(root_node: article_tag, primary_link: link, strategy: :list, position:)
+          end
+        end
+
+        def article_pairs(segmenter)
+          by_path = relevant_links_by_path(segmenter)
+          top_paths(segmenter, by_path).flat_map do |path|
+            by_path.fetch(path, []).filter_map do |node|
+              article_tag = parent_until_boundary(segmenter, node)
+              next unless article_tag
+
+              [article_tag, node]
+            end
+          end
+        end
+        module_function :article_pairs
+        private_class_method :article_pairs
+
+        def relevant_links_by_path(segmenter)
+          by_path = Hash.new { |hash, path| hash[path] = [] }
+          segmenter.index.each_node do |node|
+            next unless node.link?
+            next if segmenter.index.ignored_chrome?(node)
+            next unless relevant_anchor?(segmenter, node)
+
+            by_path[node.tag_path] << node
+          end
+          by_path
+        end
+        module_function :relevant_links_by_path
+        private_class_method :relevant_links_by_path
+
+        def top_paths(segmenter, by_path)
+          by_path.select { |_path, nodes| nodes.size >= segmenter.minimum_selector_frequency }
+                 .max_by(segmenter.use_top_selectors) { |_path, nodes| nodes.size }
+                 .map(&:first)
+        end
+        module_function :top_paths
+        private_class_method :top_paths
+
+        def relevant_anchor?(segmenter, node)
+          facts = segmenter.link_resolver.destination_facts(node)
+          return false unless facts
+
+          text = node.visible_text.to_s.strip
+          !segmenter.noise_policy.noise_anchor?(
+            text:, destination_facts: facts, anchor: node, utility_landmark_ancestor: false
+          )
+        end
+        module_function :relevant_anchor?
+        private_class_method :relevant_anchor?
+
+        def parent_until_boundary(segmenter, node)
+          index = segmenter.index
+          link_counts = Hash.new { |hash, curr| hash[curr] = count_links(curr) }
+
+          index.parent_until(node, lambda { |curr|
+            return true if %i[body html].include?(curr.name)
+            return false if index.ignored_chrome?(curr)
+
+            parent = index.parent_of(curr)
+            parent && link_counts[parent] > link_counts[curr]
+          })
+        end
+        module_function :parent_until_boundary
+        private_class_method :parent_until_boundary
+
+        def count_links(node)
+          node.link? ? 1 : node.descendants.count(&:link?)
+        end
+        module_function :count_links
+        private_class_method :count_links
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/segmenter/primary_link.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    class Segmenter
+      ##
+      # Selects the strongest content-like primary link inside a container
+      # (port of Discovery::SemanticAnchorCandidates).
+      class PrimaryLink
+        # @param segmenter [Segmenter]
+        def initialize(segmenter)
+          @segmenter = segmenter
+          @index = segmenter.index
+          @link_resolver = segmenter.link_resolver
+          @noise_policy = segmenter.noise_policy
+        end
+
+        ##
+        # @param container [SST::Node]
+        # @return [SST::Node, nil]
+        def select(container) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+          winners = {}
+
+          container.find_all(&:link?).each do |anchor|
+            next if @index.ignored_chrome?(anchor)
+
+            facts = candidate_facts(anchor, container)
+            next unless facts
+
+            dest = facts[:destination]
+            current = winners[dest]
+            winners[dest] = facts if current.nil? || facts[:score] > current[:score]
+          end
+
+          winners.values.max_by { |f| f[:score] }&.fetch(:anchor)
+        end
+
+        private
+
+        # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+        def candidate_facts(anchor, container)
+          destination = @link_resolver.destination_facts(anchor)
+          return unless destination
+
+          text = anchor.visible_text.to_s.strip
+          heading = first_heading(container)
+          heading_text = heading ? heading.visible_text.to_s.strip : ''
+          heading_anchor = heading_anchor?(anchor, heading)
+          meaningful = text.match?(/\p{Alnum}/)
+          content_like = destination.content_path
+          heading_match = meaningful && heading_text.match?(/\p{Alnum}/) && heading_text == text
+
+          return if @noise_policy.noise_anchor?(
+            text:, destination_facts: destination, anchor:, container:, heading_anchor:,
+            utility_landmark_ancestor: @segmenter.landmark_ancestor?(anchor, container)
+          )
+          return unless meaningful || content_like || heading_anchor
+
+          score = (heading_anchor ? 100 : 0) +
+                  (heading_match ? 20 : 0) +
+                  (meaningful ? 10 : 0) +
+                  (content_like ? 10 : 0)
+
+          { anchor:, destination: destination.destination, score: }
+        end
+        # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+
+        def first_heading(container)
+          (@headings ||= {}.compare_by_identity)[container] ||= container.find(&:heading?)
+        end
+
+        def heading_anchor?(anchor, heading)
+          return false unless heading
+
+          anchor.equal?(heading) || @index.descendant_of?(anchor, heading)
+        end
+      end
+    end
+  end
+end