html2rss 0.22.2 → 0.24.0

data/lib/html2rss/auto_source/discovery/dom_clustering.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    module Discovery
+      ##
+      # Discovers repeated content card nodes in anchorless or classless DOM trees
+      # by evaluating class groups first and falling back to 1-level tag structure signatures.
+      class DomClustering
+        # Node tags considered layout containers
+        LAYOUT_TAG_NAMES = Set['div', 'section', 'article', 'li', 'ul', 'ol'].freeze
+        # HTML/layout tags excluded from candidate nodes (owned by Html::Navigator).
+        EXCLUDED_TAGS = Html2rss::Html::Navigator::CLUSTER_EXCLUDED_TAGS
+
+        class << self
+          ##
+          # Clusters elements in parsed_body and returns the best set of content card nodes.
+          #
+          # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
+          # @param minimum_selector_frequency [Integer] minimum frequency for candidate groups
+          # @return [Array<Nokogiri::XML::Node>] candidate nodes of the top-scoring cluster
+          def call(parsed_body, minimum_selector_frequency:)
+            new(parsed_body, minimum_frequency: minimum_selector_frequency).call
+          end
+        end
+
+        # @param parsed_body [Nokogiri::HTML::Document]
+        # @param minimum_frequency [Integer]
+        def initialize(parsed_body, minimum_frequency:)
+          @parsed_body = parsed_body
+          @minimum_frequency = minimum_frequency
+          @cache = {}.compare_by_identity
+        end
+
+        # @return [Array<Nokogiri::XML::Node>]
+        def call
+          class_result = cluster_by_class
+          return class_result unless class_result.empty?
+
+          cluster_by_structure
+        end
+
+        private
+
+        attr_reader :parsed_body, :minimum_frequency, :cache
+
+        def cluster_by_class
+          class_groups = collect_class_groups
+          return [] if class_groups.empty?
+
+          score_groups(class_groups)
+        end
+
+        def cluster_by_structure
+          structure_groups = collect_structure_groups
+          return [] if structure_groups.empty?
+
+          score_groups(structure_groups)
+        end
+
+        def score_groups(candidate_groups)
+          scorer = GroupScorer.new
+          resolver = OverlapResolver.new(layout_tags: LAYOUT_TAG_NAMES, word_counter: scorer)
+
+          non_containers = resolver.filter_containers(candidate_groups)
+          final_groups = resolver.filter_1_to_1_overlap(non_containers)
+
+          scorer.select_best_group(final_groups)
+        end
+
+        def collect_class_groups
+          groups = Hash.new { |h, k| h[k] = [] }
+          parsed_body.css('[class]').each { |node| add_class_node(node, groups) }
+          groups.select { |_, nodes| nodes.size >= minimum_frequency }
+        end
+
+        def add_class_node(node, groups)
+          return if EXCLUDED_TAGS.include?(node.name)
+          return if Html2rss::Html::Navigator.ignored_container_path?(node, cache)
+
+          cls = normalize_class(node['class'])
+          groups[cls] << node unless cls.empty?
+        end
+
+        def collect_structure_groups
+          groups = Hash.new { |h, k| h[k] = [] }
+          parsed_body.xpath('//*').each { |node| add_structure_node(node, groups) }
+          groups.select { |_, nodes| nodes.size >= minimum_frequency }
+        end
+
+        def add_structure_node(node, groups)
+          return if EXCLUDED_TAGS.include?(node.name)
+          return if Html2rss::Html::Navigator.ignored_container_path?(node, cache)
+
+          sig = structure_signature(node)
+          groups[sig] << node unless sig.empty?
+        end
+
+        def normalize_class(class_attr)
+          class_str = class_attr.to_s.strip
+          return '' if class_str.empty?
+
+          if class_str.include?(' ')
+            class_str.split(/\s+/).sort.join(' ')
+          else
+            class_str
+          end
+        end
+
+        def structure_signature(node)
+          child_elements = node.element_children
+          return '' if child_elements.empty?
+
+          child_elements.map(&:name).join('>')
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/discovery/list_candidates.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    module Discovery
+      ##
+      # Builds repeated-list article container candidates from generic HTML.
+      class ListCandidates
+        ##
+        # Simplify an XPath selector by removing index notation.
+        #
+        # @param xpath [String] original XPath
+        # @return [String] XPath without positional indexes
+        def self.simplify_xpath(xpath)
+          xpath.gsub(/\[\d+\]/, '')
+        end
+
+        # @param parsed_body [Nokogiri::HTML::Document] parsed document
+        # @param minimum_selector_frequency [Integer] minimum repeated anchor path count
+        # @param use_top_selectors [Integer] number of frequent anchor paths to inspect
+        def initialize(parsed_body, minimum_selector_frequency:, use_top_selectors:)
+          @parsed_body = parsed_body
+          @minimum_selector_frequency = minimum_selector_frequency
+          @use_top_selectors = use_top_selectors
+        end
+
+        ##
+        # @param anchor_filter [#call] predicate for scraper-specific anchor eligibility
+        # @param boundary_condition [#call] predicate for article container boundary
+        # @yieldparam article_tag [Nokogiri::XML::Node] candidate article container
+        # @yieldparam selected_anchor [Nokogiri::XML::Node] anchor that made the container eligible
+        # @return [Enumerator]
+        def each_article_tag(anchor_filter:, boundary_condition:)
+          return enum_for(:each_article_tag, anchor_filter:, boundary_condition:) unless block_given?
+
+          article_tags(anchor_filter:, boundary_condition:).each { yield _1[:article_tag], _1[:selected_anchor] }
+        end
+
+        private
+
+        attr_reader :parsed_body, :minimum_selector_frequency, :use_top_selectors
+
+        def article_tags(anchor_filter:, boundary_condition:)
+          cache = {}.compare_by_identity
+          selectors(anchor_filter:).flat_map do |selector|
+            article_tags_for_selector(selector, boundary_condition, cache)
+          end
+        end
+
+        def article_tags_for_selector(selector, boundary_condition, cache = {}.compare_by_identity)
+          parsed_body.xpath(selector).filter_map do |selected_tag|
+            next if Html2rss::Html::Navigator.ignored_container_path?(selected_tag, cache)
+
+            article_tag = Html2rss::Html::Navigator.parent_until_condition(selected_tag, boundary_condition)
+            next unless article_tag
+
+            { article_tag:, selected_anchor: selected_tag }
+          end
+        end
+
+        def selectors(anchor_filter:)
+          anchor_counts(anchor_filter:)
+            .select { |_selector, count| count >= minimum_selector_frequency }
+            .max_by(use_top_selectors, &:last)
+            .map(&:first)
+        end
+
+        def anchor_counts(anchor_filter:)
+          cache = {}.compare_by_identity
+          Hash.new(0).tap do |counts|
+            each_anchor(anchor_filter:) do |node|
+              next if Html2rss::Html::Navigator.ignored_container_path?(node, cache)
+
+              path = self.class.simplify_xpath(node.path)
+              counts[path] += 1
+            end
+          end
+        end
+
+        def each_anchor(anchor_filter:)
+          return enum_for(:each_anchor, anchor_filter:) unless block_given?
+
+          traversal_root&.css(Html2rss::Html::Navigator::MAIN_ANCHOR_SELECTOR)&.each do |node|
+            yield node if anchor_filter.call(node)
+          end
+        end
+
+        def traversal_root
+          parsed_body.at_css('body, html') || parsed_body.root
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/discovery/semantic_anchor_candidates.rb

@@ -0,0 +1,219 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    module Discovery
+      ##
+      # Builds ranked anchor facts for one semantic content container.
+      class SemanticAnchorCandidates
+        # Anchor candidate plus scoring signals used by semantic anchor ranking.
+        AnchorFacts = Data.define(
+          :anchor,
+          :text,
+          :url,
+          :destination,
+          :segments,
+          :meaningful_text,
+          :content_like_destination,
+          :heading_anchor,
+          :heading_text_match,
+          :score
+        ) do
+          # @param candidate [Candidate] eligible semantic anchor candidate
+          # @return [AnchorFacts] serializable facts used for ranking and dedupe
+          def self.from_candidate(candidate)
+            signals = candidate.anchor_signals
+
+            new(
+              **candidate.anchor_identity_attributes,
+              meaningful_text: signals.meaningful_text,
+              content_like_destination: signals.content_like_destination,
+              heading_anchor: signals.heading_anchor,
+              heading_text_match: signals.heading_text_match,
+              score: signals.score
+            )
+          end
+        end
+
+        # Shared context for all anchors in one semantic container.
+        class Context
+          attr_reader :container, :link_heuristics
+
+          # @param container [Nokogiri::XML::Node] semantic container
+          # @param link_heuristics [Html2rss::AutoSource::LinkHeuristics] destination/text heuristics
+          def initialize(container, link_heuristics:)
+            @container = container
+            @link_heuristics = link_heuristics
+          end
+
+          # @return [Nokogiri::XML::Node, nil] heading used to identify title anchors
+          def heading
+            @heading ||= @container.at_css(Html2rss::Html::Navigator::HEADING_TAGS.join(','))
+          end
+
+          # @return [String] visible heading text
+          def heading_text
+            @heading_text ||= visible_text(heading)
+          end
+
+          # @param node [Nokogiri::XML::Node, nil] node to extract text from
+          # @return [String] visible text for the node
+          def visible_text(node)
+            return '' unless node
+
+            (@visible_texts ||= {}.compare_by_identity)[node] ||= Html2rss::Html::Navigator.extract_visible_text(node).to_s.strip
+          end
+
+          # @param anchor [Nokogiri::XML::Node] anchor candidate
+          # @return [Html2rss::AutoSource::LinkHeuristics::DestinationFacts, nil] destination facts
+          def destination_facts(anchor)
+            @link_heuristics.destination_facts(anchor)
+          end
+        end
+
+        # One anchor plus the facts needed to decide whether it represents content.
+        class Candidate
+          attr_reader :anchor
+
+          # @param anchor [Nokogiri::XML::Node] anchor candidate
+          # @param context [Context] semantic container context
+          def initialize(anchor, context)
+            @anchor = anchor
+            @context = context
+          end
+
+          # @return [AnchorFacts, nil] ranked anchor facts when the anchor is eligible
+          def facts
+            return unless destination_facts
+            return if noise_anchor?
+            return unless representative_content_anchor?
+
+            AnchorFacts.from_candidate(self)
+          end
+
+          # @return [Html2rss::AutoSource::LinkHeuristics::DestinationFacts, nil] destination facts
+          def destination_facts
+            @destination_facts ||= @context.destination_facts(@anchor)
+          end
+
+          # @return [String] visible anchor text
+          def text
+            @text ||= @context.visible_text(@anchor)
+          end
+
+          # @return [Hash] anchor identity attributes used to build AnchorFacts
+          def anchor_identity_attributes
+            {
+              anchor:,
+              text:,
+              url: destination_facts.url,
+              destination: destination_facts.destination,
+              segments: destination_facts.segments
+            }
+          end
+
+          # @return [Html2rss::AutoSource::LinkHeuristics::AnchorSignals] ranking signals
+          def anchor_signals
+            LinkHeuristics::AnchorSignals.new(
+              meaningful_text: meaningful_text?,
+              content_like_destination: content_like_destination?,
+              heading_anchor: heading_anchor?,
+              heading_text_match: heading_text_match?
+            )
+          end
+
+          # @return [Boolean] true when visible anchor text has words
+          def meaningful_text?
+            @meaningful_text ||= text.match?(/\p{Alnum}/)
+          end
+
+          # @return [Boolean] true when the destination route has content signals
+          def content_like_destination?
+            destination_facts.content_path
+          end
+
+          # @return [Boolean] true when the anchor is inside the selected heading
+          def heading_anchor?
+            heading = @context.heading
+            heading && (@anchor == heading || Html2rss::Html::Navigator.descendant_of?(@anchor, heading))
+          end
+
+          # @return [Boolean] true when anchor text exactly matches heading text
+          def heading_text_match?
+            heading_text = @context.heading_text
+
+            meaningful_text? &&
+              heading_text.match?(/\p{Alnum}/) &&
+              heading_text == text
+          end
+
+          private
+
+          def representative_content_anchor?
+            meaningful_text? || content_like_destination? || heading_anchor?
+          end
+
+          def noise_anchor?
+            @context.link_heuristics.noise_anchor?(
+              text:,
+              destination_facts:,
+              anchor: @anchor,
+              container: @context.container,
+              heading_anchor: heading_anchor?
+            )
+          end
+        end
+
+        # Keeps the strongest semantic anchor fact for each destination.
+        class DestinationWinners
+          def initialize
+            @winners = {}
+          end
+
+          # @return [Array<AnchorFacts>] strongest candidate per destination
+          def to_a
+            @winners.values
+          end
+
+          # @param facts [AnchorFacts] candidate anchor facts
+          # @return [void]
+          def add(facts)
+            destination = facts.destination
+            @winners[destination] = stronger_fact(@winners[destination], facts)
+          end
+
+          private
+
+          def stronger_fact(current, candidate)
+            return candidate unless current
+
+            current.score >= candidate.score ? current : candidate
+          end
+        end
+
+        # @param container [Nokogiri::XML::Node] semantic container
+        # @param link_heuristics [Html2rss::AutoSource::LinkHeuristics] destination/text heuristics
+        def initialize(container, link_heuristics:)
+          @container = container
+          @context = Context.new(container, link_heuristics:)
+        end
+
+        # @return [Array<AnchorFacts>] strongest candidate per destination
+        def to_a
+          @container.css(Html2rss::Html::Navigator::MAIN_ANCHOR_SELECTOR)
+                    .each_with_object(DestinationWinners.new) { |anchor, winners| add_anchor(anchor, winners) }
+                    .to_a
+        end
+
+        private
+
+        def add_anchor(anchor, winners)
+          return if Html2rss::Html::Navigator.ignored_container_path?(anchor)
+
+          facts = Candidate.new(anchor, @context).facts
+          winners.add(facts) if facts
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/discovery/semantic_containers.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    module Discovery
+      ##
+      # Collects semantic content containers from a parsed HTML document.
+      class SemanticContainers
+        # Candidate selectors used to locate extractable semantic content blocks.
+        SELECTORS = [
+          'article:not(:has(article))',
+          'section:not(:has(section))',
+          'li:not(:has(li))',
+          'tr:not(:has(tr))',
+          'div:not(:has(div))'
+        ].freeze
+
+        # @param parsed_body [Nokogiri::HTML::Document] parsed document
+        # @return [Array<Nokogiri::XML::Node>] candidate semantic containers
+        def self.call(parsed_body)
+          new(parsed_body).call
+        end
+
+        # @param parsed_body [Nokogiri::HTML::Document] parsed document
+        def initialize(parsed_body)
+          @parsed_body = parsed_body
+        end
+
+        # @return [Array<Nokogiri::XML::Node>] candidate semantic containers
+        def call
+          cache = {}.compare_by_identity
+          candidates = @parsed_body.css(SELECTORS.join(',')).reject do |node|
+            Html2rss::Html::Navigator.ignored_container_path?(node, cache)
+          end
+
+          candidates = filter_nested_containers(candidates)
+          sort_by_depth(candidates)
+        end
+
+        private
+
+        def filter_nested_containers(candidates)
+          candidate_set = Set.new(candidates)
+          rejected = Set.new
+
+          candidates.each do |candidate_b|
+            next if candidate_b.name == 'div'
+
+            find_and_reject_ancestors(candidate_b, candidate_set, rejected)
+          end
+
+          candidates.reject { |c| rejected.include?(c) }
+        end
+
+        def find_and_reject_ancestors(node, candidate_set, rejected)
+          curr = node.parent
+          while curr && !curr.document? && curr.name != 'html'
+            rejected << curr if candidate_set.include?(curr)
+            curr = curr.parent
+          end
+        end
+
+        def sort_by_depth(candidates)
+          candidates.each_with_index
+                    .sort_by { |node, index| [-node.ancestors.size, index] }
+                    .map!(&:first)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/discovery/sitemap.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+require 'date'
+require 'nokogiri'
+
+module Html2rss
+  class AutoSource
+    module Discovery
+      ##
+      # Parses XML sitemap documents and extracts prioritized, recent article entries.
+      class Sitemap
+        # Default minimum priority threshold (0.0 to 1.0)
+        DEFAULT_MIN_PRIORITY = 0.3
+        # Default maximum age in days for sitemap entries
+        DEFAULT_MAX_AGE_DAYS = 30
+        # Google News sitemap XML namespace
+        NEWS_NS = 'http://www.google.com/schemas/sitemap-news/0.9'
+
+        # Result value object returned by .call
+        # @!attribute entries [Array<SitemapEntry>] parsed entries (empty for sitemapindex docs)
+        # @!attribute sub_sitemap_urls [Array<String>] child sitemap URLs (empty for urlset docs)
+        Result = Data.define(:entries, :sub_sitemap_urls)
+
+        # Immutable facts for one sitemap entry
+        SitemapEntry = Data.define(:url, :title, :published_at, :priority, :changefreq)
+
+        class << self
+          ##
+          # Parses sitemap XML body and returns a Result.
+          #
+          # @param xml_content [String] sitemap XML body string
+          # @param min_priority [Float] minimum priority score (0.0..1.0)
+          # @param max_age_days [Integer] maximum age in days for entries
+          # @return [Result] result with entries and/or sub_sitemap_urls
+          def call(xml_content, min_priority: DEFAULT_MIN_PRIORITY, max_age_days: DEFAULT_MAX_AGE_DAYS)
+            new(xml_content, min_priority:, max_age_days:).call
+          end
+        end
+
+        ##
+        # @param xml_content [String]
+        # @param min_priority [Float]
+        # @param max_age_days [Integer]
+        def initialize(xml_content, min_priority: DEFAULT_MIN_PRIORITY, max_age_days: DEFAULT_MAX_AGE_DAYS)
+          @xml_content = xml_content
+          @min_priority = min_priority
+          @max_age_days = max_age_days
+        end
+
+        ##
+        # @return [Result]
+        def call
+          document = Nokogiri::XML(xml_content)
+          return Result.new(entries: [], sub_sitemap_urls: []) if invalid_xml?(document)
+
+          if sitemapindex?(document)
+            Result.new(entries: [], sub_sitemap_urls: extract_sub_sitemap_urls(document))
+          else
+            Result.new(entries: extract_entries(document), sub_sitemap_urls: [])
+          end
+        end
+
+        private
+
+        attr_reader :xml_content, :min_priority, :max_age_days
+
+        def invalid_xml?(document)
+          document.errors.any? && document.at_css('urlset, sitemapindex').nil?
+        end
+
+        def sitemapindex?(document)
+          !document.at_xpath('//*[local-name()="sitemapindex"]').nil?
+        end
+
+        def extract_sub_sitemap_urls(document)
+          document.xpath('//*[local-name()="sitemap"]/*[local-name()="loc"]').filter_map do |node|
+            text = node.text.strip
+            text unless text.empty?
+          end
+        end
+
+        def extract_entries(document)
+          document.xpath('//*[local-name()="url"]').filter_map { build_entry(_1) }
+        end
+
+        def build_entry(url_node)
+          loc = parse_loc(url_node)
+          return unless loc
+          return if invalid_entry_signals?(url_node)
+
+          published_at = parse_date(url_node)
+          return if stale_date?(published_at)
+
+          build_entry_facts(url_node, loc, published_at)
+        end
+
+        def build_entry_facts(url_node, loc, published_at)
+          SitemapEntry.new(
+            url: loc,
+            title: parse_title(url_node),
+            published_at:,
+            priority: parse_priority(url_node),
+            changefreq: parse_changefreq(url_node)
+          )
+        end
+
+        def parse_loc(url_node)
+          node = url_node.at_xpath('./*[local-name()="loc"]')
+          text = node&.text&.strip
+          text unless text.to_s.empty?
+        end
+
+        def parse_changefreq(url_node)
+          node = url_node.at_xpath('./*[local-name()="changefreq"]')
+          node ? node.text.strip.downcase : nil
+        end
+
+        def invalid_entry_signals?(url_node)
+          parse_priority(url_node) < min_priority || parse_changefreq(url_node) == 'never'
+        end
+
+        def parse_priority(url_node)
+          node = url_node.at_xpath('./*[local-name()="priority"]')
+          text = node&.text&.strip
+          text ? text.to_f : 0.5
+        end
+
+        def parse_date(url_node)
+          raw_date = raw_date_from(url_node)
+          return unless raw_date
+
+          Time.parse(raw_date).utc.iso8601
+        rescue ArgumentError
+          nil
+        end
+
+        def raw_date_from(url_node)
+          news_node = url_node.at_xpath('.//news:publication_date', 'news' => NEWS_NS)
+          lastmod_node = url_node.at_xpath('./*[local-name()="lastmod"]')
+          (news_node || lastmod_node)&.text&.strip
+        end
+
+        def parse_title(url_node)
+          node = url_node.at_xpath('.//news:title', 'news' => NEWS_NS)
+          node&.text&.strip
+        end
+
+        def stale_date?(published_at)
+          return false unless published_at
+
+          time = Time.parse(published_at)
+          (Time.now.utc - time) > (max_age_days * 86_400)
+        rescue ArgumentError
+          false
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/auto_source/discovery.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class AutoSource
+    ##
+    # Card/list discovery helpers used by AutoSource scrapers before field extraction.
+    #
+    # Html2rss::Html::ArticleExtractor fills article fields from a container; discovery finds those containers.
+    #
+    # DOM list discovery for anchorless or classless pages is owned by {Discovery::DomClustering}.
+    module Discovery
+    end
+  end
+end