html2rss 0.24.0 → 0.26.0

data/lib/html2rss/auto_source/segmenter/semantic.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    class Segmenter
+      ##
+      # Collects leaf semantic containers (port of Discovery::SemanticContainers).
+      module Semantic
+        # Tag names collected as leaf semantic containers.
+        CANDIDATE_NAMES = %i[article section li tr div].to_set.freeze
+
+        module_function
+
+        ##
+        # @param segmenter [Segmenter]
+        # @return [Array<Segment>]
+        def call(segmenter)
+          containers = collect_containers(segmenter.index)
+          primary = PrimaryLink.new(segmenter)
+
+          containers.each_with_index.filter_map do |container, position|
+            link = primary.select(container)
+            next unless link || segmenter.permit_unanchored
+
+            Segment.build(root_node: container, primary_link: link, strategy: :semantic, position:)
+          end
+        end
+
+        def collect_containers(index)
+          non_leaf = mark_non_leaf_candidates(index.root)
+          candidates = []
+          index.each_node do |node|
+            next unless CANDIDATE_NAMES.include?(node.name)
+            next if non_leaf.include?(node)
+            next if index.ignored_chrome?(node)
+
+            candidates << node
+          end
+
+          candidates = filter_nested(candidates, index)
+          sort_by_depth(candidates, index)
+        end
+        module_function :collect_containers
+        private_class_method :collect_containers
+
+        TAG_MASKS = { article: 1, section: 2, li: 4, tr: 8, div: 16 }.freeze
+        private_constant :TAG_MASKS
+
+        # Single O(N) post-order pass: mark candidate nodes that contain a same-name candidate.
+        def mark_non_leaf_candidates(root) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+          non_leaf = Set.new.compare_by_identity
+          visit = lambda do |node|
+            seen_mask = 0
+            node.children.each do |child|
+              child_mask = visit.call(child)
+              seen_mask |= child_mask | (TAG_MASKS[child.name] || 0)
+            end
+            node_mask = TAG_MASKS[node.name]
+            non_leaf.add(node) if node_mask && seen_mask.anybits?(node_mask)
+            seen_mask
+          end
+          visit.call(root)
+          non_leaf
+        end
+        module_function :mark_non_leaf_candidates
+        private_class_method :mark_non_leaf_candidates
+
+        def filter_nested(candidates, index)
+          candidate_set = {}.compare_by_identity
+          candidates.each { |c| candidate_set[c] = true }
+          rejected = {}.compare_by_identity
+
+          candidates.each do |candidate|
+            next if candidate.name == :div
+
+            reject_ancestors(candidate, candidate_set, rejected, index)
+          end
+
+          candidates.reject { |c| rejected[c] }
+        end
+        module_function :filter_nested
+        private_class_method :filter_nested
+
+        def reject_ancestors(node, candidate_set, rejected, index)
+          curr = index.parent_of(node)
+          while curr && curr.name != :html
+            rejected[curr] = true if candidate_set[curr]
+            curr = index.parent_of(curr)
+          end
+        end
+        module_function :reject_ancestors
+        private_class_method :reject_ancestors
+
+        def sort_by_depth(candidates, index)
+          candidates.each_with_index
+                    .sort_by { |node, i| [-index.depth_of(node), i] }
+                    .map(&:first)
+        end
+        module_function :sort_by_depth
+        private_class_method :sort_by_depth
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/segmenter.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    ##
+    # Discovers candidate content segments from an SST::Document.
+    #
+    # Strategies:
+    # - +:semantic+ — leaf article/section/li/tr/div containers with a primary link
+    # - +:list+ — repeated tag_path anchors walked to a shared container boundary
+    # - +:cluster+ — class / structure clustering for anchorless card grids
+    class Segmenter
+      # @param document [SST::Document]
+      # @param base_url [String, Html2rss::Url]
+      # @param strategy [Symbol]
+      # @param opts [Hash] segmentation options
+      # @option opts [Boolean] :permit_unanchored keep containers without a primary link
+      # @option opts [Integer] :minimum_selector_frequency list strategy frequency floor
+      # @option opts [Integer] :use_top_selectors list strategy selector budget
+      # @option opts [Scoring::LinkResolver, nil] :link_resolver page-scoped destination facts cache
+      # @return [Array<Segment>]
+      # rubocop:disable Style/ArgumentsForwarding -- keep named opts for YARD @option
+      def self.call(document, base_url:, strategy:, **opts)
+        new(document, base_url:, strategy:, **opts).call
+      end
+      # rubocop:enable Style/ArgumentsForwarding
+
+      # @param document [SST::Document]
+      # @param base_url [String, Html2rss::Url]
+      # @param strategy [Symbol]
+      # @param opts [Hash] segmentation options
+      # @option opts [Boolean] :permit_unanchored keep containers without a primary link
+      # @option opts [Integer] :minimum_selector_frequency list strategy frequency floor
+      # @option opts [Integer] :use_top_selectors list strategy selector budget
+      # @option opts [Scoring::LinkResolver, nil] :link_resolver page-scoped destination facts cache
+      def initialize(document, base_url:, strategy:, **opts)
+        raise ArgumentError, 'document must be SST::Document' unless document.is_a?(SST::Document)
+        raise ArgumentError, "unknown strategy: #{strategy.inspect}" unless Segment::STRATEGIES.include?(strategy)
+
+        @document = document
+        @base_url = base_url
+        @strategy = strategy
+        @permit_unanchored = opts.fetch(:permit_unanchored, false)
+        @minimum_selector_frequency = opts.fetch(:minimum_selector_frequency, 2)
+        @use_top_selectors = opts.fetch(:use_top_selectors, 5)
+        @link_resolver = opts[:link_resolver] || Scoring::LinkResolver.new(base_url)
+        @noise_policy = LinkDestination::NoisePolicy.new(link_resolver: @link_resolver)
+      end
+
+      # @return [Array<Segment>]
+      def call
+        case @strategy
+        when :semantic then Semantic.call(self)
+        when :list then List.call(self)
+        when :cluster then Cluster.call(self)
+        end
+      end
+
+      attr_reader :document, :base_url, :permit_unanchored, :minimum_selector_frequency,
+                  :use_top_selectors, :link_resolver, :noise_policy
+
+      # @return [SST::Index]
+      def index = document.index
+
+      ##
+      # Whether +anchor+ has a utility-landmark ancestor outside +container+.
+      #
+      # @param anchor [SST::Node]
+      # @param container [SST::Node]
+      # @return [Boolean]
+      def landmark_ancestor?(anchor, container)
+        cache = (@landmark_cache ||= {})
+        key = [anchor.object_id, container.object_id]
+        return cache[key] if cache.key?(key)
+
+        cache[key] = compute_landmark_ancestor?(anchor:, container:)
+      end
+
+      private
+
+      def compute_landmark_ancestor?(anchor:, container:) # rubocop:disable Metrics/CyclomaticComplexity
+        return false unless anchor && container
+
+        curr = index.parent_of(anchor)
+        while curr && curr.name != :html
+          return true if curr != container && curr.utility_landmark?
+          return false if curr.equal?(container)
+
+          curr = index.parent_of(curr)
+        end
+        false
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source.rb

@@ -15,17 +15,22 @@ module Html2rss
   # @see Html2rss::AutoSource::Scraper::Schema
   # @see Html2rss::AutoSource::Scraper::SemanticHtml
   # @see Html2rss::AutoSource::Scraper::Html
+  # rubocop:disable Metrics/ClassLength -- defaults + tiered extract stay on the contributor entry type
   class AutoSource
+    # Default max articles to keep (also the short-circuit floor across scraper tiers).
+    DEFAULT_LIMIT = 25
+
     # Default auto-source configuration shipped for scraper and cleanup behavior.
     DEFAULT_CONFIG = {
+      limit: DEFAULT_LIMIT,
       scraper: {
         wordpress_api: {
           enabled: true
         },
         sitemap: {
           enabled: true,
-          min_priority: Discovery::Sitemap::DEFAULT_MIN_PRIORITY,
-          max_age_days: Discovery::Sitemap::DEFAULT_MAX_AGE_DAYS
+          min_priority: Scraper::Sitemap::Parser::DEFAULT_MIN_PRIORITY,
+          max_age_days: Scraper::Sitemap::Parser::DEFAULT_MAX_AGE_DAYS
         },
         schema: {
           enabled: true
@@ -39,6 +44,9 @@ module Html2rss
         json_state: {
           enabled: true
         },
+        xhr_articles: {
+          enabled: true
+        },
         meta_oembed: {
           enabled: true
         },
@@ -80,28 +88,25 @@ module Html2rss
     # @param response [Html2rss::RequestService::Response] initial page response
     # @param opts [Hash] validated auto-source options
     # @param request_session [Html2rss::RequestSession, nil] shared request session for follow-up fetches
+    # @option opts [Integer] :limit max articles to keep; later tiers stop once this many survive Cleanup
     # @option opts [Hash] :scraper scraper configuration map
     # @option opts [Hash] :cleanup cleanup configuration map
     # @return [void]
     def initialize(response, opts = DEFAULT_CONFIG, request_session: nil)
       @parsed_body = response.parsed_body
+      @body = response.body
       @url = response.url
+      @captured_responses = response.captured_responses
       @opts = opts
       @request_session = request_session
     end
 
     ##
-    # Extracts article candidates by selecting every scraper that can explain the
-    # page shape, running those scrapers, and normalizing the resulting hashes
-    # into `Article` objects.
+    # Extracts articles by running scraper tiers until a sufficient set is found.
     #
-    # The contributor-facing flow is:
-    # 1. choose scraper instances that match the page
-    # 2. let each scraper collect its own candidates
-    # 3. clean and deduplicate the merged article list
-    #
-    # Scrapers with expensive precomputation, such as `SemanticHtml`, keep that
-    # state on the instance so detection and extraction can reuse the same work.
+    # Tiers: in-page structured → follow-up IO → SemanticHtml → Html.
+    # SST is built only when a heuristic tier runs. Later tiers are skipped once
+    # +limit+ articles with url+title remain after Cleanup; the result is capped to +limit+.
     #
     # @return [Array<Html2rss::Article>] extracted articles
     def articles
@@ -113,22 +118,74 @@ module Html2rss
 
     private
 
-    attr_reader :url, :parsed_body, :request_session
+    attr_reader :url, :parsed_body, :body, :request_session, :captured_responses
 
+    # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
     def extract_articles
-      scraper_instances = Scraper.instances_for(parsed_body, url:, request_session:, opts: @opts[:scraper])
-      return [] if scraper_instances.empty?
+      articles = []
+      matched = false
+      document = nil
+      link_resolver = nil
+      scraper_opts = @opts.fetch(:scraper, {})
+
+      Scraper::SCRAPER_TIERS.each do |tier|
+        break if enough_articles?(articles)
+
+        if Scraper.heuristic_tier?(tier)
+          document ||= Scraper.normalize_sst(parsed_body)
+          next unless document
+
+          link_resolver ||= ::Html2rss::Scoring::LinkResolver.new(url)
+        end
+
+        tier.each do |scraper_class|
+          instance = Scraper.build_instance(
+            scraper_class,
+            parsed_body,
+            opts: scraper_opts,
+            url:,
+            request_session:,
+            body:,
+            document:,
+            link_resolver:,
+            captured_responses:
+          )
+          next unless instance
+          next unless Scraper.extractable_instance?(instance, parsed_body)
 
-      # Scrapers are run sequentially.
-      articles = scraper_instances.flat_map do |instance|
-        run_scraper(instance)
+          matched = true
+          articles.concat(run_scraper(instance))
+        end
       end
-      Cleanup.call(articles, url:, **cleanup_options)
+
+      raise Scraper.no_scraper_found_for(parsed_body, body:) unless matched
+
+      Cleanup.call(articles, url:, **cleanup_options).first(article_limit)
+    end
+    # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+
+    def enough_articles?(articles)
+      threshold = article_limit
+      return false if articles.size < threshold
+
+      cleaned = Cleanup.call(articles.dup, url:, **cleanup_options)
+      cleaned.count { |article| article.url && !article.title.to_s.empty? } >= threshold
+    end
+
+    def article_limit
+      @opts.fetch(:limit, DEFAULT_LIMIT)
     end
 
     def run_scraper(instance)
-      instance.each.map do |article_hash|
-        Article.new(**article_hash, scraper: instance.class)
+      instance.each.map do |item|
+        case item
+        when Article
+          item
+        when Hash
+          Article.new(**item, scraper: instance.class)
+        else
+          raise TypeError, "#{instance.class} yielded #{item.class}; expected Article or Hash"
+        end
       end
     end
 
@@ -136,4 +193,5 @@ module Html2rss
       @opts.fetch(:cleanup, {})
     end
   end
+  # rubocop:enable Metrics/ClassLength
 end

data/lib/html2rss/capture.rb

@@ -0,0 +1,315 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Analyzes a URL and produces a reusable feed config hash with derived CSS selectors.
+  #
+  # Uses the auto-source pipeline to extract articles, then traces back through
+  # SST segments to build items + attribute selectors suitable for a static feed config.
+  class Capture # rubocop:disable Metrics/ClassLength
+    LEADING_TRIM_TAGS = %w[html body].freeze
+    private_constant :LEADING_TRIM_TAGS
+
+    ##
+    # Result of a capture operation.
+    # @!attribute config [Hash] feed config hash with +:channel+ and +:selectors+
+    # @!attribute articles_count [Integer] number of articles extracted
+    # @!attribute channel_title [String] derived channel title
+    CaptureResult = Data.define(:config, :articles_count, :channel_title)
+
+    class << self
+      ##
+      # Analyzes a URL and builds a reusable feed config.
+      #
+      # @param url [String] source page URL
+      # @param strategy [Symbol] request strategy (+:auto+, +:faraday+, +:botasaurus+)
+      # @option options [String, nil] :items_selector optional selector hint
+      # @option options [Integer, nil] :max_redirects optional redirect limit override
+      # @option options [Integer, nil] :max_requests optional request budget override
+      # @option options [Integer, nil] :limit max articles to keep
+      # @option options [String, nil] :local_file_path optional local HTML file path
+      # @return [CaptureResult]
+      def build(url, strategy: :auto, **)
+        new(url, strategy:, **).build
+      end
+    end
+
+    ##
+    # @param url [String] source page URL
+    # @param strategy [Symbol] request strategy
+    # @param options [Hash] additional options
+    # @option options [String, nil] :items_selector optional selector hint
+    # @option options [Integer, nil] :max_redirects optional redirect limit override
+    # @option options [Integer, nil] :max_requests optional request budget override
+    # @option options [Integer, nil] :limit max articles to keep
+    # @option options [String, nil] :local_file_path optional local HTML file path
+    def initialize(url, strategy: :auto, **options)
+      @url = url
+      @strategy = strategy
+      @items_selector_hint = options.delete(:items_selector)
+      @max_redirects = options.delete(:max_redirects)
+      @max_requests = options.delete(:max_requests)
+      @limit = options.delete(:limit)
+      @local_file_path = options.delete(:local_file_path)
+    end
+
+    ##
+    # Runs the capture pipeline.
+    #
+    # @return [CaptureResult]
+    def build # rubocop:disable Metrics/MethodLength
+      response = fetch_response
+      articles = extract_articles(response)
+      selectors = derive_selectors(response, articles)
+
+      config = {
+        channel: build_channel(response),
+        selectors: selectors.empty? ? nil : selectors,
+        **local_file_request_overlay
+      }.compact
+
+      CaptureResult.new(
+        config:,
+        articles_count: articles.size,
+        channel_title: channel_title_from(response)
+      )
+    end
+
+    private
+
+    def raw_config
+      @raw_config ||= build_raw_config
+    end
+
+    def build_raw_config # rubocop:disable Metrics/MethodLength
+      Config.auto_source_config(
+        url: @url,
+        items_selector: @items_selector_hint,
+        request_controls: Config::RequestControls.from_shortcut(
+          strategy: @strategy,
+          max_redirects: @max_redirects,
+          max_requests: @max_requests
+        ),
+        limit: @limit
+      ).tap do |config|
+        config[:strategy] = resolve_strategy(config[:strategy] || @strategy)
+        apply_local_file_path!(config)
+      end
+    end
+
+    def apply_local_file_path!(config)
+      return unless @local_file_path
+
+      config[:strategy] = :local_file
+      config[:request] ||= {}
+      config[:request][:local_file_path] = @local_file_path
+    end
+
+    def local_file_request_overlay
+      return {} unless @local_file_path
+
+      { strategy: :local_file, request: { local_file_path: @local_file_path } }
+    end
+
+    def resolve_strategy(strategy)
+      plan = FeedPipeline::StrategyPlan.resolve(strategy)
+      plan.is_a?(FeedPipeline::StrategyPlan::Auto) ? :faraday : plan.strategy
+    end
+
+    def fetch_response
+      config = Config.from_hash(raw_config)
+      resources = FeedPipeline::RuntimePolicy.resources_for(config)
+      session = RequestSession.build(
+        config:,
+        strategy: config.strategy,
+        budget: resources.budget,
+        policy: resources.policy
+      )
+      session.fetch_initial_response
+    end
+
+    def extract_articles(response)
+      auto_source_opts = raw_config[:auto_source] || AutoSource::DEFAULT_CONFIG
+      AutoSource.new(response, auto_source_opts).articles
+    rescue AutoSource::Scraper::NoScraperFound
+      []
+    end
+
+    def derive_selectors(response, articles) # rubocop:disable Metrics/MethodLength
+      return {} if articles.empty? || !response.html_response?
+
+      sst = normalize_sst(response)
+      return {} unless sst
+
+      segments = discover_segments(sst)
+      return {} if segments.empty?
+
+      matched = match_segments_to_articles(segments, articles)
+      return {} if matched.empty?
+
+      build_selector_hash(matched, sst)
+    rescue ArgumentError => error
+      Log.warn("Capture selector derivation failed: #{error.message}")
+      {}
+    end
+
+    def normalize_sst(response)
+      SST::Normalizer.call(response.body)
+    end
+
+    def discover_segments(sst)
+      link_resolver = Scoring::LinkResolver.new(@url)
+
+      AutoSource::Segmenter.call(
+        sst,
+        base_url: @url,
+        strategy: :list,
+        permit_unanchored: false,
+        link_resolver:
+      )
+    end
+
+    def match_segments_to_articles(segments, articles)
+      articles_by_url = articles.each_with_object({}) do |article, hash|
+        url_str = article.url.to_s
+        hash[url_str] = article unless url_str.empty?
+      end
+
+      segments.filter_map do |segment|
+        next unless segment.primary_link
+
+        article = find_matching_article(segment, articles_by_url)
+        { segment:, article: } if article
+      end
+    end
+
+    def find_matching_article(segment, articles_by_url) # rubocop:disable Metrics/AbcSize
+      href = segment.primary_link.attrs.href.to_s
+      return articles_by_url.values.find { |a| a.title == title_from_segment(segment) } if href.empty?
+
+      resolved = Url.from_relative(href, @url)
+      articles_by_url[resolved.to_s] || articles_by_url.values.find { |a| fuzzy_url_match?(a.url, resolved) }
+    end
+
+    def fuzzy_url_match?(article_url, resolved)
+      article_url.to_s.end_with?(resolved.path.to_s)
+    rescue StandardError
+      false
+    end
+
+    def title_from_segment(segment)
+      segment.root_node.visible_text.to_s.strip
+    end
+
+    def build_selector_hash(matched, _sst)
+      items_sel = items_selector(matched)
+      return {} unless items_sel
+
+      first = matched.first
+      root = first[:segment].root_node
+
+      attrs = {}.tap do |a|
+        a[:title] = title_selector(root)
+        a[:url] = url_selector(first[:segment])
+        a[:description] = description_selector(root, first[:segment])
+      end.compact
+
+      { items: { selector: items_sel }, **attrs }
+    end
+
+    def items_selector(matched)
+      roots = matched.map { |m| m[:segment].root_node }
+      shared = shared_class_items_selector(roots)
+      return shared if shared
+
+      paths = roots.map(&:tag_path)
+      common = common_path_prefix(paths)
+      tag_path = common.empty? ? paths.first.to_s : common
+      return nil if tag_path.empty?
+
+      css_from_trimmed_tag_path(tag_path)
+    end
+
+    def shared_class_items_selector(roots)
+      shared = roots.map { |root| root.attrs.class_names }.reduce { |left, right| left & right }
+      return nil if shared.nil? || shared.empty?
+
+      "#{roots.first.name}.#{shared.min}"
+    end
+
+    def css_from_trimmed_tag_path(tag_path)
+      segments = tag_path.split('/').reject(&:empty?)
+      trimmed = trim_leading_items_segments(segments)
+      trimmed.join(' > ')
+    end
+
+    def trim_leading_items_segments(segments)
+      return segments if segments.empty?
+
+      trimmed = segments.dup
+      trimmed.shift while trimmed.any? && LEADING_TRIM_TAGS.include?(trimmed.first)
+      trimmed.shift while trimmed.length > 1 && trimmed.first == 'div'
+      trimmed.empty? ? [segments.last] : trimmed
+    end
+
+    def common_path_prefix(paths) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity
+      return '' if paths.empty?
+
+      segments = paths.map { |p| p.split('/').reject(&:empty?) }
+      prefix = segments.first
+      segments.each do |seg|
+        i = 0
+        i += 1 while i < prefix.length && i < seg.length && prefix[i] == seg[i]
+        prefix = prefix[0...i]
+      end
+      "/#{prefix.join('/')}"
+    end
+
+    def title_selector(root) # rubocop:disable Metrics/CyclomaticComplexity
+      heading = root.find(&:heading?)
+      relative = css_for_path(heading.tag_path, root.tag_path) if heading
+      relative ||= begin
+        link = root.find { |n| n.link? && n.visible_text.to_s.strip.length > 3 }
+        css_for_path(link.tag_path, root.tag_path) if link
+      end
+      return nil unless relative
+
+      { selector: relative }
+    end
+
+    def url_selector(segment)
+      return nil unless segment.primary_link
+
+      relative = css_for_path(segment.primary_link.tag_path, segment.root_node.tag_path)
+      { selector: relative, extractor: 'href' }
+    end
+
+    def description_selector(root, segment)
+      relative = css_for_path(root.tag_path, segment.root_node.tag_path)
+      return nil if relative.nil? || relative.empty? || relative == '.'
+
+      { selector: relative }
+    end
+
+    def css_for_path(full_path, root_path)
+      relative = full_path.delete_prefix(root_path)
+      return '.' if relative.empty?
+
+      relative.split('/').reject(&:empty?).join(' > ')
+    end
+
+    def build_channel(response)
+      {
+        url: @url,
+        title: channel_title_from(response),
+        time_zone: 'UTC'
+      }.compact
+    end
+
+    def channel_title_from(response)
+      Channel.from_response(response).title
+    rescue StandardError
+      nil
+    end
+  end
+end