html2rss 0.24.0 → 0.26.0

data/lib/html2rss/auto_source/discovery/semantic_anchor_candidates.rb

@@ -1,219 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class AutoSource
-    module Discovery
-      ##
-      # Builds ranked anchor facts for one semantic content container.
-      class SemanticAnchorCandidates
-        # Anchor candidate plus scoring signals used by semantic anchor ranking.
-        AnchorFacts = Data.define(
-          :anchor,
-          :text,
-          :url,
-          :destination,
-          :segments,
-          :meaningful_text,
-          :content_like_destination,
-          :heading_anchor,
-          :heading_text_match,
-          :score
-        ) do
-          # @param candidate [Candidate] eligible semantic anchor candidate
-          # @return [AnchorFacts] serializable facts used for ranking and dedupe
-          def self.from_candidate(candidate)
-            signals = candidate.anchor_signals
-
-            new(
-              **candidate.anchor_identity_attributes,
-              meaningful_text: signals.meaningful_text,
-              content_like_destination: signals.content_like_destination,
-              heading_anchor: signals.heading_anchor,
-              heading_text_match: signals.heading_text_match,
-              score: signals.score
-            )
-          end
-        end
-
-        # Shared context for all anchors in one semantic container.
-        class Context
-          attr_reader :container, :link_heuristics
-
-          # @param container [Nokogiri::XML::Node] semantic container
-          # @param link_heuristics [Html2rss::AutoSource::LinkHeuristics] destination/text heuristics
-          def initialize(container, link_heuristics:)
-            @container = container
-            @link_heuristics = link_heuristics
-          end
-
-          # @return [Nokogiri::XML::Node, nil] heading used to identify title anchors
-          def heading
-            @heading ||= @container.at_css(Html2rss::Html::Navigator::HEADING_TAGS.join(','))
-          end
-
-          # @return [String] visible heading text
-          def heading_text
-            @heading_text ||= visible_text(heading)
-          end
-
-          # @param node [Nokogiri::XML::Node, nil] node to extract text from
-          # @return [String] visible text for the node
-          def visible_text(node)
-            return '' unless node
-
-            (@visible_texts ||= {}.compare_by_identity)[node] ||= Html2rss::Html::Navigator.extract_visible_text(node).to_s.strip
-          end
-
-          # @param anchor [Nokogiri::XML::Node] anchor candidate
-          # @return [Html2rss::AutoSource::LinkHeuristics::DestinationFacts, nil] destination facts
-          def destination_facts(anchor)
-            @link_heuristics.destination_facts(anchor)
-          end
-        end
-
-        # One anchor plus the facts needed to decide whether it represents content.
-        class Candidate
-          attr_reader :anchor
-
-          # @param anchor [Nokogiri::XML::Node] anchor candidate
-          # @param context [Context] semantic container context
-          def initialize(anchor, context)
-            @anchor = anchor
-            @context = context
-          end
-
-          # @return [AnchorFacts, nil] ranked anchor facts when the anchor is eligible
-          def facts
-            return unless destination_facts
-            return if noise_anchor?
-            return unless representative_content_anchor?
-
-            AnchorFacts.from_candidate(self)
-          end
-
-          # @return [Html2rss::AutoSource::LinkHeuristics::DestinationFacts, nil] destination facts
-          def destination_facts
-            @destination_facts ||= @context.destination_facts(@anchor)
-          end
-
-          # @return [String] visible anchor text
-          def text
-            @text ||= @context.visible_text(@anchor)
-          end
-
-          # @return [Hash] anchor identity attributes used to build AnchorFacts
-          def anchor_identity_attributes
-            {
-              anchor:,
-              text:,
-              url: destination_facts.url,
-              destination: destination_facts.destination,
-              segments: destination_facts.segments
-            }
-          end
-
-          # @return [Html2rss::AutoSource::LinkHeuristics::AnchorSignals] ranking signals
-          def anchor_signals
-            LinkHeuristics::AnchorSignals.new(
-              meaningful_text: meaningful_text?,
-              content_like_destination: content_like_destination?,
-              heading_anchor: heading_anchor?,
-              heading_text_match: heading_text_match?
-            )
-          end
-
-          # @return [Boolean] true when visible anchor text has words
-          def meaningful_text?
-            @meaningful_text ||= text.match?(/\p{Alnum}/)
-          end
-
-          # @return [Boolean] true when the destination route has content signals
-          def content_like_destination?
-            destination_facts.content_path
-          end
-
-          # @return [Boolean] true when the anchor is inside the selected heading
-          def heading_anchor?
-            heading = @context.heading
-            heading && (@anchor == heading || Html2rss::Html::Navigator.descendant_of?(@anchor, heading))
-          end
-
-          # @return [Boolean] true when anchor text exactly matches heading text
-          def heading_text_match?
-            heading_text = @context.heading_text
-
-            meaningful_text? &&
-              heading_text.match?(/\p{Alnum}/) &&
-              heading_text == text
-          end
-
-          private
-
-          def representative_content_anchor?
-            meaningful_text? || content_like_destination? || heading_anchor?
-          end
-
-          def noise_anchor?
-            @context.link_heuristics.noise_anchor?(
-              text:,
-              destination_facts:,
-              anchor: @anchor,
-              container: @context.container,
-              heading_anchor: heading_anchor?
-            )
-          end
-        end
-
-        # Keeps the strongest semantic anchor fact for each destination.
-        class DestinationWinners
-          def initialize
-            @winners = {}
-          end
-
-          # @return [Array<AnchorFacts>] strongest candidate per destination
-          def to_a
-            @winners.values
-          end
-
-          # @param facts [AnchorFacts] candidate anchor facts
-          # @return [void]
-          def add(facts)
-            destination = facts.destination
-            @winners[destination] = stronger_fact(@winners[destination], facts)
-          end
-
-          private
-
-          def stronger_fact(current, candidate)
-            return candidate unless current
-
-            current.score >= candidate.score ? current : candidate
-          end
-        end
-
-        # @param container [Nokogiri::XML::Node] semantic container
-        # @param link_heuristics [Html2rss::AutoSource::LinkHeuristics] destination/text heuristics
-        def initialize(container, link_heuristics:)
-          @container = container
-          @context = Context.new(container, link_heuristics:)
-        end
-
-        # @return [Array<AnchorFacts>] strongest candidate per destination
-        def to_a
-          @container.css(Html2rss::Html::Navigator::MAIN_ANCHOR_SELECTOR)
-                    .each_with_object(DestinationWinners.new) { |anchor, winners| add_anchor(anchor, winners) }
-                    .to_a
-        end
-
-        private
-
-        def add_anchor(anchor, winners)
-          return if Html2rss::Html::Navigator.ignored_container_path?(anchor)
-
-          facts = Candidate.new(anchor, @context).facts
-          winners.add(facts) if facts
-        end
-      end
-    end
-  end
-end

data/lib/html2rss/auto_source/discovery/semantic_containers.rb

@@ -1,71 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class AutoSource
-    module Discovery
-      ##
-      # Collects semantic content containers from a parsed HTML document.
-      class SemanticContainers
-        # Candidate selectors used to locate extractable semantic content blocks.
-        SELECTORS = [
-          'article:not(:has(article))',
-          'section:not(:has(section))',
-          'li:not(:has(li))',
-          'tr:not(:has(tr))',
-          'div:not(:has(div))'
-        ].freeze
-
-        # @param parsed_body [Nokogiri::HTML::Document] parsed document
-        # @return [Array<Nokogiri::XML::Node>] candidate semantic containers
-        def self.call(parsed_body)
-          new(parsed_body).call
-        end
-
-        # @param parsed_body [Nokogiri::HTML::Document] parsed document
-        def initialize(parsed_body)
-          @parsed_body = parsed_body
-        end
-
-        # @return [Array<Nokogiri::XML::Node>] candidate semantic containers
-        def call
-          cache = {}.compare_by_identity
-          candidates = @parsed_body.css(SELECTORS.join(',')).reject do |node|
-            Html2rss::Html::Navigator.ignored_container_path?(node, cache)
-          end
-
-          candidates = filter_nested_containers(candidates)
-          sort_by_depth(candidates)
-        end
-
-        private
-
-        def filter_nested_containers(candidates)
-          candidate_set = Set.new(candidates)
-          rejected = Set.new
-
-          candidates.each do |candidate_b|
-            next if candidate_b.name == 'div'
-
-            find_and_reject_ancestors(candidate_b, candidate_set, rejected)
-          end
-
-          candidates.reject { |c| rejected.include?(c) }
-        end
-
-        def find_and_reject_ancestors(node, candidate_set, rejected)
-          curr = node.parent
-          while curr && !curr.document? && curr.name != 'html'
-            rejected << curr if candidate_set.include?(curr)
-            curr = curr.parent
-          end
-        end
-
-        def sort_by_depth(candidates)
-          candidates.each_with_index
-                    .sort_by { |node, index| [-node.ancestors.size, index] }
-                    .map!(&:first)
-        end
-      end
-    end
-  end
-end

data/lib/html2rss/auto_source/discovery/sitemap.rb

@@ -1,159 +0,0 @@
-# frozen_string_literal: true
-
-require 'date'
-require 'nokogiri'
-
-module Html2rss
-  class AutoSource
-    module Discovery
-      ##
-      # Parses XML sitemap documents and extracts prioritized, recent article entries.
-      class Sitemap
-        # Default minimum priority threshold (0.0 to 1.0)
-        DEFAULT_MIN_PRIORITY = 0.3
-        # Default maximum age in days for sitemap entries
-        DEFAULT_MAX_AGE_DAYS = 30
-        # Google News sitemap XML namespace
-        NEWS_NS = 'http://www.google.com/schemas/sitemap-news/0.9'
-
-        # Result value object returned by .call
-        # @!attribute entries [Array<SitemapEntry>] parsed entries (empty for sitemapindex docs)
-        # @!attribute sub_sitemap_urls [Array<String>] child sitemap URLs (empty for urlset docs)
-        Result = Data.define(:entries, :sub_sitemap_urls)
-
-        # Immutable facts for one sitemap entry
-        SitemapEntry = Data.define(:url, :title, :published_at, :priority, :changefreq)
-
-        class << self
-          ##
-          # Parses sitemap XML body and returns a Result.
-          #
-          # @param xml_content [String] sitemap XML body string
-          # @param min_priority [Float] minimum priority score (0.0..1.0)
-          # @param max_age_days [Integer] maximum age in days for entries
-          # @return [Result] result with entries and/or sub_sitemap_urls
-          def call(xml_content, min_priority: DEFAULT_MIN_PRIORITY, max_age_days: DEFAULT_MAX_AGE_DAYS)
-            new(xml_content, min_priority:, max_age_days:).call
-          end
-        end
-
-        ##
-        # @param xml_content [String]
-        # @param min_priority [Float]
-        # @param max_age_days [Integer]
-        def initialize(xml_content, min_priority: DEFAULT_MIN_PRIORITY, max_age_days: DEFAULT_MAX_AGE_DAYS)
-          @xml_content = xml_content
-          @min_priority = min_priority
-          @max_age_days = max_age_days
-        end
-
-        ##
-        # @return [Result]
-        def call
-          document = Nokogiri::XML(xml_content)
-          return Result.new(entries: [], sub_sitemap_urls: []) if invalid_xml?(document)
-
-          if sitemapindex?(document)
-            Result.new(entries: [], sub_sitemap_urls: extract_sub_sitemap_urls(document))
-          else
-            Result.new(entries: extract_entries(document), sub_sitemap_urls: [])
-          end
-        end
-
-        private
-
-        attr_reader :xml_content, :min_priority, :max_age_days
-
-        def invalid_xml?(document)
-          document.errors.any? && document.at_css('urlset, sitemapindex').nil?
-        end
-
-        def sitemapindex?(document)
-          !document.at_xpath('//*[local-name()="sitemapindex"]').nil?
-        end
-
-        def extract_sub_sitemap_urls(document)
-          document.xpath('//*[local-name()="sitemap"]/*[local-name()="loc"]').filter_map do |node|
-            text = node.text.strip
-            text unless text.empty?
-          end
-        end
-
-        def extract_entries(document)
-          document.xpath('//*[local-name()="url"]').filter_map { build_entry(_1) }
-        end
-
-        def build_entry(url_node)
-          loc = parse_loc(url_node)
-          return unless loc
-          return if invalid_entry_signals?(url_node)
-
-          published_at = parse_date(url_node)
-          return if stale_date?(published_at)
-
-          build_entry_facts(url_node, loc, published_at)
-        end
-
-        def build_entry_facts(url_node, loc, published_at)
-          SitemapEntry.new(
-            url: loc,
-            title: parse_title(url_node),
-            published_at:,
-            priority: parse_priority(url_node),
-            changefreq: parse_changefreq(url_node)
-          )
-        end
-
-        def parse_loc(url_node)
-          node = url_node.at_xpath('./*[local-name()="loc"]')
-          text = node&.text&.strip
-          text unless text.to_s.empty?
-        end
-
-        def parse_changefreq(url_node)
-          node = url_node.at_xpath('./*[local-name()="changefreq"]')
-          node ? node.text.strip.downcase : nil
-        end
-
-        def invalid_entry_signals?(url_node)
-          parse_priority(url_node) < min_priority || parse_changefreq(url_node) == 'never'
-        end
-
-        def parse_priority(url_node)
-          node = url_node.at_xpath('./*[local-name()="priority"]')
-          text = node&.text&.strip
-          text ? text.to_f : 0.5
-        end
-
-        def parse_date(url_node)
-          raw_date = raw_date_from(url_node)
-          return unless raw_date
-
-          Time.parse(raw_date).utc.iso8601
-        rescue ArgumentError
-          nil
-        end
-
-        def raw_date_from(url_node)
-          news_node = url_node.at_xpath('.//news:publication_date', 'news' => NEWS_NS)
-          lastmod_node = url_node.at_xpath('./*[local-name()="lastmod"]')
-          (news_node || lastmod_node)&.text&.strip
-        end
-
-        def parse_title(url_node)
-          node = url_node.at_xpath('.//news:title', 'news' => NEWS_NS)
-          node&.text&.strip
-        end
-
-        def stale_date?(published_at)
-          return false unless published_at
-
-          time = Time.parse(published_at)
-          (Time.now.utc - time) > (max_age_days * 86_400)
-        rescue ArgumentError
-          false
-        end
-      end
-    end
-  end
-end

data/lib/html2rss/auto_source/discovery.rb

@@ -1,14 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class AutoSource
-    ##
-    # Card/list discovery helpers used by AutoSource scrapers before field extraction.
-    #
-    # Html2rss::Html::ArticleExtractor fills article fields from a container; discovery finds those containers.
-    #
-    # DOM list discovery for anchorless or classless pages is owned by {Discovery::DomClustering}.
-    module Discovery
-    end
-  end
-end

data/lib/html2rss/auto_source/link_heuristics/anchor_signals.rb

@@ -1,28 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class AutoSource
-    class LinkHeuristics
-      # Score weights keyed by AnchorSignals member name.
-      ANCHOR_SCORE_RULES = {
-        heading_anchor: 100,
-        heading_text_match: 20,
-        meaningful_text: 10,
-        content_like_destination: 10
-      }.freeze
-
-      # Anchor ranking signals used by semantic primary-link selection.
-      AnchorSignals = Data.define(
-        :heading_anchor,
-        :heading_text_match,
-        :meaningful_text,
-        :content_like_destination
-      ) do
-        # @return [Integer] ranking score for one eligible anchor
-        def score
-          ANCHOR_SCORE_RULES.sum { |signal, weight| public_send(signal) ? weight : 0 }
-        end
-      end
-    end
-  end
-end

data/lib/html2rss/auto_source/link_heuristics/container_assessor.rb

@@ -1,106 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class AutoSource
-    class LinkHeuristics
-      ##
-      # Turns container DOM observations into ContainerSignals inputs.
-      #
-      # Owns publish markers, headings, visible text, and class/id tokens so
-      # LinkHeuristics can keep eligibility policy and ContainerSignals can
-      # keep scoring without re-reading the DOM.
-      class ContainerAssessor
-        # Token pattern for content-like class/id markers on containers.
-        CONTENT_TOKEN_REGEXP = begin
-          words = PathClassifier::SEGMENT_SETS.fetch(:content)
-          /(?:^|\s|[-_])(#{Regexp.union(words.to_a).source})(?:\s|[-_]|$)/i
-        end.freeze
-
-        # Token pattern for junk/utility class/id markers on containers.
-        JUNK_TOKEN_REGEXP = begin
-          words = PathClassifier::SEGMENT_SETS.fetch(:utility)
-          /(?:^|\s|[-_])(#{Regexp.union(words.to_a).source})(?:\s|[-_]|$)/i
-        end.freeze
-
-        # @param text_classifier [TextClassifier] shared utility/recommended text policy
-        def initialize(text_classifier:)
-          @text_classifier = text_classifier
-        end
-
-        ##
-        # Observes a container and builds ranking signals, including hard-junk.
-        #
-        # @param container [Nokogiri::XML::Node] semantic container node
-        # @param selected_anchor [Nokogiri::XML::Node, nil] primary anchor for the container
-        # @param destination_facts [DestinationFacts, nil] route facts for the selected anchor
-        # @return [ContainerSignals] observation + scoring signals for the container
-        # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-        def call(container, selected_anchor, destination_facts:)
-          title = entry_title(container, selected_anchor)
-          tokens = container_tokens(container)
-
-          ContainerSignals.new(
-            title_word_count: word_count(title),
-            path_length: destination_facts&.url&.path.to_s.length,
-            content_path: destination_facts&.content_path,
-            publish_marker: publish_marker?(container),
-            descriptive_context: descriptive_context?(visible_text(container), title),
-            article_container: container.name == 'article',
-            content_tokens: content_tokens?(tokens),
-            junk_tokens: junk_tokens?(tokens),
-            utility_prefix_title: @text_classifier.utility_prefix?(title),
-            recommended_title: @text_classifier.recommended?(title),
-            utility_path: destination_facts&.utility_path,
-            strong_post_suffix: destination_facts&.strong_post_suffix,
-            shallow: destination_facts&.shallow,
-            high_confidence_junk_path: destination_facts&.high_confidence_junk_path,
-            high_confidence_utility_destination: destination_facts&.high_confidence_utility_destination,
-            selected_anchor_present: !selected_anchor.nil?
-          )
-        end
-        # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
-
-        private
-
-        def publish_marker?(container)
-          (@publish_markers ||= {}.compare_by_identity)[container] ||=
-            !!container.at_css('time, [datetime], [itemprop="datePublished"], [itemprop="dateModified"]')
-        end
-
-        def descriptive_context?(container_text, title)
-          snippet = container_text.to_s.sub(/\A#{Regexp.escape(title.to_s)}/i, '')
-          snippet.length > 30 && word_count(snippet) >= 8
-        end
-
-        def heading_for(container)
-          (@headings ||= {}.compare_by_identity)[container] ||=
-            container.at_css(Html2rss::Html::Navigator::HEADING_TAGS.join(','))
-        end
-
-        def visible_text(node)
-          return '' unless node
-
-          (@visible_texts ||= {}.compare_by_identity)[node] ||= Html2rss::Html::Navigator.extract_visible_text(node).to_s.strip
-        end
-
-        def entry_title(container, selected_anchor) = visible_text(heading_for(container) || selected_anchor)
-
-        def word_count(text)
-          (@word_counts ||= {})[text] ||= begin
-            count = 0
-            text.to_s.scan(/\p{Alnum}+/) { count += 1 }
-            count
-          end
-        end
-
-        def container_tokens(container)
-          (@container_tokens ||= {}.compare_by_identity)[container] ||= "#{container['class']} #{container['id']}"
-        end
-
-        def content_tokens?(tokens) = tokens.to_s.match?(CONTENT_TOKEN_REGEXP)
-
-        def junk_tokens?(tokens) = tokens.to_s.match?(JUNK_TOKEN_REGEXP)
-      end
-    end
-  end
-end

data/lib/html2rss/auto_source/link_heuristics/container_signals.rb

@@ -1,80 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class AutoSource
-    class LinkHeuristics
-      # Container observations used to compute quality/junk scores.
-      ContainerSignals = Data.define(
-        :title_word_count,
-        :path_length,
-        :content_path,
-        :publish_marker,
-        :descriptive_context,
-        :article_container,
-        :content_tokens,
-        :junk_tokens,
-        :utility_prefix_title,
-        :recommended_title,
-        :utility_path,
-        :strong_post_suffix,
-        :shallow,
-        :high_confidence_junk_path,
-        :high_confidence_utility_destination,
-        :selected_anchor_present
-      ) do
-        # @return [Integer] positive quality contribution for ranking
-        def quality_score # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-          score = 0
-          score += 40 if title_word_count >= 3
-          score += 15 if title_word_count >= 7
-          score += 20 if path_length > 6
-          score += 15 if content_path
-          score += 15 if publish_marker
-          score += 10 if descriptive_context
-          score += 10 if article_container
-          score += 10 if content_tokens
-          score
-        end
-
-        # @return [Integer] junk penalty subtracted from quality
-        def junk_score # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-          score = 0
-          score += 25 if non_content_utility_path?
-          score += 15 if utility_prefix_title && title_word_count <= 6
-          score += 10 if shallow
-          score += 10 if weak_container?
-          score += 10 if recommended_title && !content_path
-          score += 5 if high_confidence_junk_path
-          score += 15 if junk_tokens
-          score
-        end
-
-        # @return [Integer] quality minus junk for stable ranking
-        def final_score = quality_score - junk_score
-
-        # @return [Boolean] true when the entry should be dropped before ranking
-        def hard_junk? # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-          weak = weak_article_candidate?
-
-          high_confidence_junk_path ||
-            (selected_anchor_present && recommended_title && shallow && weak) ||
-            (selected_anchor_present && utility_prefix_title &&
-              high_confidence_utility_destination && weak)
-        end
-
-        private
-
-        # @return [Boolean] true when article evidence is too weak to keep
-        def weak_article_candidate?
-          [article_container, publish_marker, descriptive_context, content_path].count(&:itself) < 2
-        end
-
-        def non_content_utility_path?
-          utility_path && !content_path && !strong_post_suffix
-        end
-
-        def weak_container? = !publish_marker && !descriptive_context
-      end
-    end
-  end
-end