html2rss 0.22.2 → 0.24.0

data/lib/html2rss/html_extractor/semantic_anchor_candidates.rb

@@ -1,260 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class HtmlExtractor
-    ##
-    # Builds ranked anchor facts for one semantic content container.
-    class SemanticAnchorCandidates
-      # Anchor candidate plus scoring signals used by semantic anchor ranking.
-      AnchorFacts = Data.define(
-        :anchor,
-        :text,
-        :url,
-        :destination,
-        :segments,
-        :meaningful_text,
-        :content_like_destination,
-        :heading_anchor,
-        :heading_text_match,
-        :score
-      ) do
-        # @param candidate [Candidate] eligible semantic anchor candidate
-        # @return [AnchorFacts] serializable facts used for ranking and dedupe
-        def self.from_candidate(candidate)
-          new(
-            **candidate.anchor_identity_attributes,
-            **candidate.anchor_signal_attributes,
-            score: Score.new(candidate).value
-          )
-        end
-      end
-
-      # Shared context for all anchors in one semantic container.
-      class Context
-        attr_reader :container
-
-        # Ancestor tags that usually indicate navigation/utility regions.
-        UTILITY_LANDMARK_TAGS = %w[nav aside footer menu].freeze
-
-        # @param container [Nokogiri::XML::Node] semantic container
-        # @param link_heuristics [Html2rss::AutoSource::Scraper::LinkHeuristics] destination/text heuristics
-        def initialize(container, link_heuristics:)
-          @container = container
-          @link_heuristics = link_heuristics
-        end
-
-        # @return [Nokogiri::XML::Node, nil] heading used to identify title anchors
-        def heading
-          @heading ||= @container.at_css(HtmlExtractor::HEADING_TAGS.join(','))
-        end
-
-        # @return [String] visible heading text
-        def heading_text
-          @heading_text ||= visible_text(heading)
-        end
-
-        # @param node [Nokogiri::XML::Node, nil] node to extract text from
-        # @return [String] visible text for the node
-        def visible_text(node)
-          return '' unless node
-
-          (@visible_texts ||= {}.compare_by_identity)[node] ||= HtmlExtractor.extract_visible_text(node).to_s.strip
-        end
-
-        # @param anchor [Nokogiri::XML::Node] anchor candidate
-        # @return [Html2rss::AutoSource::Scraper::LinkHeuristics::DestinationFacts, nil] destination facts
-        def destination_facts(anchor)
-          @link_heuristics.destination_facts(anchor)
-        end
-
-        # @param text [String] visible anchor text
-        # @return [Boolean] true when text is utility chrome
-        def utility_text?(text)
-          @link_heuristics.utility_text?(text)
-        end
-      end
-
-      # One anchor plus the facts needed to decide whether it represents content.
-      class Candidate
-        attr_reader :anchor
-
-        # @param anchor [Nokogiri::XML::Node] anchor candidate
-        # @param context [Context] semantic container context
-        def initialize(anchor, context)
-          @anchor = anchor
-          @context = context
-        end
-
-        # @return [AnchorFacts, nil] ranked anchor facts when the anchor is eligible
-        def facts
-          return unless destination_facts
-          return if utility_text_suppressed? || ineligible_anchor?
-          return unless representative_content_anchor?
-
-          AnchorFacts.from_candidate(self)
-        end
-
-        # @return [Html2rss::AutoSource::Scraper::LinkHeuristics::DestinationFacts, nil] destination facts
-        def destination_facts
-          @destination_facts ||= @context.destination_facts(@anchor)
-        end
-
-        # @return [String] visible anchor text
-        def text
-          @text ||= @context.visible_text(@anchor)
-        end
-
-        # @return [Hash] anchor identity attributes used to build AnchorFacts
-        def anchor_identity_attributes
-          {
-            anchor:,
-            text:,
-            url: destination_facts.url,
-            destination: destination_facts.destination,
-            segments: destination_facts.segments
-          }
-        end
-
-        # @return [Hash] anchor signal attributes used to build AnchorFacts
-        def anchor_signal_attributes
-          {
-            meaningful_text: meaningful_text?,
-            content_like_destination: content_like_destination?,
-            heading_anchor: heading_anchor?,
-            heading_text_match: heading_text_match?
-          }
-        end
-
-        # @return [Boolean] true when visible anchor text has words
-        def meaningful_text?
-          @meaningful_text ||= text.match?(/\p{Alnum}/)
-        end
-
-        # @return [Boolean] true when the destination route has content signals
-        def content_like_destination?
-          destination_facts.content_path
-        end
-
-        # @return [Boolean] true when the anchor is inside the selected heading
-        def heading_anchor?
-          heading = @context.heading
-          heading && (@anchor == heading || HtmlNavigator.descendant_of?(@anchor, heading))
-        end
-
-        # @return [Boolean] true when anchor text exactly matches heading text
-        def heading_text_match?
-          heading_text = @context.heading_text
-
-          meaningful_text? &&
-            heading_text.match?(/\p{Alnum}/) &&
-            heading_text == text
-        end
-
-        private
-
-        def representative_content_anchor?
-          meaningful_text? || content_like_destination? || heading_anchor?
-        end
-
-        def utility_text_suppressed?
-          !content_like_destination? &&
-            @context.utility_text?(text) &&
-            (destination_facts.high_confidence_utility_destination || non_heading_weak_post?)
-        end
-
-        def non_heading_weak_post?
-          !heading_anchor? && !destination_facts.strong_post_suffix
-        end
-
-        def ineligible_anchor?
-          destination_facts.high_confidence_utility_destination ||
-            icon_only_anchor? ||
-            utility_landmark_ancestor?
-        end
-
-        def utility_landmark_ancestor?
-          container = @context.container
-          condition = proc { |node| node == container || Context::UTILITY_LANDMARK_TAGS.include?(node.name) }
-          landmark = HtmlNavigator.parent_until_condition(@anchor.parent, condition)
-
-          landmark && landmark != container
-        end
-
-        def icon_only_anchor?
-          !meaningful_text? && @anchor.at_css('img, svg')
-        end
-      end
-
-      # Scores an eligible semantic anchor candidate.
-      class Score
-        # Score weights keyed by candidate signal predicate.
-        RULES = {
-          heading_anchor?: 100,
-          heading_text_match?: 20,
-          meaningful_text?: 10,
-          content_like_destination?: 10
-        }.freeze
-
-        # @param candidate [Candidate] eligible semantic anchor candidate
-        def initialize(candidate)
-          @candidate = candidate
-        end
-
-        # @return [Integer] ranking score
-        def value
-          RULES.sum { |predicate, weight| @candidate.public_send(predicate) ? weight : 0 }
-        end
-      end
-
-      # Keeps the strongest semantic anchor fact for each destination.
-      class DestinationWinners
-        def initialize
-          @winners = {}
-        end
-
-        # @return [Array<AnchorFacts>] strongest candidate per destination
-        def to_a
-          @winners.values
-        end
-
-        # @param facts [AnchorFacts] candidate anchor facts
-        # @return [void]
-        def add(facts)
-          destination = facts.destination
-          @winners[destination] = stronger_fact(@winners[destination], facts)
-        end
-
-        private
-
-        def stronger_fact(current, candidate)
-          return candidate unless current
-
-          current.score >= candidate.score ? current : candidate
-        end
-      end
-
-      # @param container [Nokogiri::XML::Node] semantic container
-      # @param link_heuristics [Html2rss::AutoSource::Scraper::LinkHeuristics] destination/text heuristics
-      def initialize(container, link_heuristics:)
-        @container = container
-        @context = Context.new(container, link_heuristics:)
-      end
-
-      # @return [Array<AnchorFacts>] strongest candidate per destination
-      def to_a
-        @container.css(HtmlExtractor::MAIN_ANCHOR_SELECTOR)
-                  .each_with_object(DestinationWinners.new) { |anchor, winners| add_anchor(anchor, winners) }
-                  .to_a
-      end
-
-      private
-
-      def add_anchor(anchor, winners)
-        return if HtmlExtractor.ignored_container_path?(anchor)
-
-        facts = Candidate.new(anchor, @context).facts
-        winners.add(facts) if facts
-      end
-    end
-  end
-end

data/lib/html2rss/html_extractor/semantic_containers.rb

@@ -1,69 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class HtmlExtractor
-    ##
-    # Collects semantic content containers from a parsed HTML document.
-    class SemanticContainers
-      # Candidate selectors used to locate extractable semantic content blocks.
-      SELECTORS = [
-        'article:not(:has(article))',
-        'section:not(:has(section))',
-        'li:not(:has(li))',
-        'tr:not(:has(tr))',
-        'div:not(:has(div))'
-      ].freeze
-
-      # @param parsed_body [Nokogiri::HTML::Document] parsed document
-      # @return [Array<Nokogiri::XML::Node>] candidate semantic containers
-      def self.call(parsed_body)
-        new(parsed_body).call
-      end
-
-      # @param parsed_body [Nokogiri::HTML::Document] parsed document
-      def initialize(parsed_body)
-        @parsed_body = parsed_body
-      end
-
-      # @return [Array<Nokogiri::XML::Node>] candidate semantic containers
-      def call
-        cache = {}.compare_by_identity
-        candidates = @parsed_body.css(SELECTORS.join(',')).reject do |node|
-          HtmlExtractor.ignored_container_path?(node, cache)
-        end
-
-        candidates = filter_nested_containers(candidates)
-        sort_by_depth(candidates)
-      end
-
-      private
-
-      def filter_nested_containers(candidates)
-        candidate_set = Set.new(candidates)
-        rejected = Set.new
-
-        candidates.each do |candidate_b|
-          next if candidate_b.name == 'div'
-
-          find_and_reject_ancestors(candidate_b, candidate_set, rejected)
-        end
-
-        candidates.reject { |c| rejected.include?(c) }
-      end
-
-      def find_and_reject_ancestors(node, candidate_set, rejected)
-        curr = node.parent
-        while curr && !curr.document? && curr.name != 'html'
-          rejected << curr if candidate_set.include?(curr)
-          curr = curr.parent
-        end
-      end
-
-      def sort_by_depth(candidates)
-        candidates.each_with_index
-                  .sort_by { |node, index| [-node.ancestors.size, index] }
-                  .map!(&:first)
-      end
-    end
-  end
-end

data/lib/html2rss/html_extractor/text_extractor.rb

@@ -1,77 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  class HtmlExtractor
-    ##
-    # TextExtractor extracts visible text from DOM elements, preserving lists
-    # and block spacing while sanitizing white spaces.
-    class TextExtractor
-      # HTML block elements that trigger line breaks or special formatting.
-      BLOCK_TAGS = %w[p div li ul ol h1 h2 h3 h4 h5 h6 tr br].to_set.freeze
-      # Tags ignored when extracting visible text content.
-      INVISIBLE_CONTENT_TAGS = %w[svg script noscript style template].to_set.freeze
-
-      class << self
-        ##
-        # @param tag [Nokogiri::XML::Node] the node from which to extract visible text
-        # @param separator [String] separator used to join text fragments (default is a space)
-        # @param exclude_nodes [Array<Nokogiri::XML::Node>, nil] nodes to exclude from extraction
-        # @return [String, nil] the concatenated visible text, or nil if none is found
-        def call(tag, separator: ' ', exclude_nodes: nil)
-          return tag.text.gsub(/\s+/, ' ').strip if tag.respond_to?(:text?) && tag.text?
-
-          parts = iterate_children(tag, separator, exclude_nodes)
-          return if parts.empty?
-
-          parts.join.squeeze(' ').strip
-        end
-
-        private
-
-        def iterate_children(tag, separator, exclude_nodes)
-          last = false
-          tag.children.each_with_object([]) do |c, p|
-            next if exclude_nodes&.include?(c) || !visible_child?(c)
-
-            text, block = process_child_node(c, separator, exclude_nodes)
-            next if text.empty?
-
-            append_separator!(p, separator, block, last)
-            (p << text) && (last = block)
-          end
-        end
-
-        def process_child_node(child, separator, exclude_nodes)
-          child_text = get_child_text(child, separator, exclude_nodes)
-          return ['', false] if child_text.empty?
-
-          child_text = "- #{child_text}" if child.name == 'li'
-          [child_text, BLOCK_TAGS.include?(child.name)]
-        end
-
-        def get_child_text(child, separator, exclude_nodes)
-          if child.children.empty?
-            child.text.to_s.gsub(/\s+/, ' ').strip
-          else
-            call(child, separator:, exclude_nodes:).to_s.strip
-          end
-        end
-
-        def append_separator!(parts, separator, is_block, last_was_block)
-          return if parts.empty?
-
-          parts << if is_block || last_was_block
-                     (separator == ' ' ? "\n" : separator)
-                   else
-                     ' '
-                   end
-        end
-
-        def visible_child?(node)
-          !INVISIBLE_CONTENT_TAGS.include?(node.name) &&
-            !(node.name == 'a' && node['href']&.start_with?('#'))
-        end
-      end
-    end
-  end
-end

data/lib/html2rss/html_extractor.rb

@@ -1,190 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  ##
-  # HtmlExtractor is responsible for extracting details (headline, url, images, etc.)
-  # from an article_tag.
-  # rubocop:disable Metrics/ClassLength
-  class HtmlExtractor
-    # Heading tags used to prioritize title extraction.
-    HEADING_TAGS = %w[h1 h2 h3 h4 h5 h6].freeze
-
-    # Element tags that indicate ignored DOM chrome when found in a container path.
-    IGNORED_CONTAINER_TAGS = %w[nav footer header svg script style].to_set.freeze
-
-    # Anchor selector used to identify the canonical article link element.
-    MAIN_ANCHOR_SELECTOR = begin
-      buf = +'a[href]:not([href=""])'
-      %w[# javascript: mailto: tel: file:// sms: data:].each do |prefix|
-        buf << %[:not([href^="#{prefix}"])]
-      end
-      buf.freeze
-    end
-
-    class << self
-      ##
-      # Extracts visible text from a given node and its children.
-      # Delegates to TextExtractor.
-      #
-      # @param tag [Nokogiri::XML::Node] the node from which to extract visible text
-      # @param separator [String] separator used to join text fragments (default is a space)
-      # @param exclude_nodes [Array<Nokogiri::XML::Node>, nil] nodes to exclude from extraction
-      # @return [String, nil] the concatenated visible text, or nil if none is found
-      def extract_visible_text(tag, separator: ' ', exclude_nodes: nil)
-        TextExtractor.call(tag, separator:, exclude_nodes:)
-      end
-
-      ##
-      # @param article_tag [Nokogiri::XML::Node] article-like container to search within
-      # @return [Nokogiri::XML::Node, nil] first eligible descendant anchor
-      def main_anchor_for(article_tag)
-        return article_tag if article_tag.name == 'a' && article_tag.matches?(MAIN_ANCHOR_SELECTOR)
-
-        article_tag.at_css(MAIN_ANCHOR_SELECTOR)
-      end
-
-      ##
-      # @param node [Nokogiri::XML::Node]
-      # @param cache [Hash, nil] identity cache used to store results (must use compare_by_identity)
-      # @return [Boolean] true when the node belongs to ignored DOM chrome
-      # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
-      def ignored_container_path?(node, cache = nil)
-        return cache[node] if cache&.key?(node)
-
-        curr = node
-        visited = []
-        is_ignored = false
-
-        while curr.respond_to?(:parent) && curr
-          if cache&.key?(curr)
-            is_ignored = cache[curr]
-            break
-          end
-
-          if IGNORED_CONTAINER_TAGS.include?(curr.name)
-            is_ignored = true
-            break
-          end
-
-          visited << curr
-          curr = curr.parent
-        end
-        visited.each { |n| cache[n] = is_ignored } if cache
-
-        is_ignored
-      end
-      # rubocop:enable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
-    end
-
-    ##
-    # @param article_tag [Nokogiri::XML::Node] article-like container to extract from
-    # @param base_url [String, Html2rss::Url] base url used to resolve relative links
-    # @param selected_anchor [Nokogiri::XML::Node, nil] explicit primary anchor for the container
-    # @param fallback_anchorless [Boolean] whether to fall back to anchorless extraction
-    def initialize(article_tag, base_url:, selected_anchor:, fallback_anchorless: false)
-      raise ArgumentError, 'article_tag is required' unless article_tag
-
-      @article_tag = article_tag
-      @base_url = base_url
-      @selected_anchor = selected_anchor
-      @fallback_anchorless = fallback_anchorless
-    end
-
-    # @return [Hash{Symbol => Object}] extracted article attributes
-    def call
-      {
-        title: extract_title,
-        url: extract_url,
-        image: extract_image,
-        description: extract_description,
-        id: generate_id,
-        published_at: extract_published_at,
-        enclosures: extract_enclosures,
-        categories: extract_categories
-      }
-    end
-
-    private
-
-    attr_reader :article_tag, :base_url, :selected_anchor
-
-    def extract_url
-      @extract_url ||= begin
-        href = selected_anchor&.[]('href').to_s
-
-        if href.empty?
-          anchorless_url_fallback
-        else
-          Url.from_relative(href.split('#').first.strip, base_url)
-        end
-      end
-    end
-
-    def anchorless_url_fallback
-      return unless @fallback_anchorless
-
-      id = generate_id
-      Url.from_relative("##{id}", base_url) if id
-    end
-
-    # rubocop:disable Metrics/CyclomaticComplexity
-    def extract_title
-      source = heading || selected_anchor
-      title_text = source ? self.class.extract_visible_text(source) : fallback_anchorless_title
-      return unless title_text
-
-      kicker = kicker_node ? self.class.extract_visible_text(kicker_node).to_s.strip : nil
-      kicker && !kicker.empty? && !title_text.include?(kicker) ? "#{kicker}: #{title_text}" : title_text
-    end
-    # rubocop:enable Metrics/CyclomaticComplexity
-
-    def fallback_anchorless_title
-      return unless @fallback_anchorless && selected_anchor.nil?
-
-      text_node = article_tag.xpath('.//text()').find { |t| !t.text.strip.empty? }
-      text_node&.text&.strip
-    end
-
-    def heading
-      @heading ||= HeadingExtractor.call(
-        article_tag,
-        fallback_anchorless: @fallback_anchorless,
-        selected_anchor:
-      )
-    end
-
-    def kicker_node
-      @kicker_node ||= begin
-        selector = '[data-tb-kicker], [class*="kicker"], [class*="eyebrow"], ' \
-                   '[class*="pre-title"], [class*="pretitle"], [class*="overline"]'
-        node = article_tag.at_css(selector)
-        node && heading && (node == heading || HtmlNavigator.descendant_of?(node, heading)) ? nil : node
-      end
-    end
-
-    def extract_description
-      exclude = [heading, selected_anchor, kicker_node].compact.to_set
-      description = self.class.extract_visible_text(article_tag, exclude_nodes: exclude)
-      return if description.nil?
-
-      desc = description.strip
-      desc.empty? ? nil : desc
-    end
-
-    def generate_id
-      @generate_id ||= IdGenerator.call(
-        article_tag,
-        heading:,
-        url: (selected_anchor ? extract_url : nil),
-        selected_anchor:,
-        fallback_anchorless: @fallback_anchorless
-      )
-    end
-
-    def extract_image = ImageExtractor.call(article_tag, base_url:)
-    def extract_published_at = DateExtractor.call(article_tag)
-    def extract_enclosures = EnclosureExtractor.call(article_tag, base_url)
-    def extract_categories = CategoryExtractor.call(article_tag)
-  end
-  # rubocop:enable Metrics/ClassLength
-end

data/lib/html2rss/html_navigator.rb

@@ -1,71 +0,0 @@
-# frozen_string_literal: true
-
-module Html2rss
-  ##
-  # HtmlNavigator provides methods to navigate through HTML nodes.
-  class HtmlNavigator
-    class << self
-      ##
-      # Returns the first parent that satisfies the condition.
-      # If the condition is met, it returns the node itself.
-      #
-      # @param node [Nokogiri::XML::Node] The node to start the search from.
-      # @param condition [Proc] The condition to be met.
-      # @return [Nokogiri::XML::Node, nil] The first parent that satisfies the condition.
-      def parent_until_condition(node, condition)
-        while node && !node.document? && node.name != 'html'
-          return node if condition.call(node)
-
-          node = node.parent
-        end
-      end
-
-      ##
-      # Think of it as `css_upwards` method.
-      # It searches for the closest parent that matches the given selector.
-      #
-      # @param current_tag [Nokogiri::XML::Node, nil] starting node
-      # @param selector [String] CSS selector to search upwards for
-      # @return [Nokogiri::XML::Node, nil] first matching node in upward traversal
-      def find_closest_selector_upwards(current_tag, selector)
-        while current_tag
-          found = current_tag.at_css(selector)
-          return found if found
-
-          return nil unless current_tag.respond_to?(:parent)
-
-          current_tag = current_tag.parent
-        end
-      end
-
-      ##
-      # Searches for the closest parent that matches the given tag name.
-      #
-      # @param current_tag [Nokogiri::XML::Node] starting node
-      # @param tag_name [String] tag name to find in ancestors
-      # @return [Nokogiri::XML::Node, nil] matching ancestor node
-      def find_tag_in_ancestors(current_tag, tag_name)
-        return current_tag if current_tag.name == tag_name
-
-        current_tag.ancestors(tag_name).first
-      end
-
-      ##
-      # Returns true if child_node is a descendant of parent_node.
-      # Walks up using parent pointers to avoid NodeSet allocations.
-      #
-      # @param child_node [Nokogiri::XML::Node] potential descendant
-      # @param parent_node [Nokogiri::XML::Node] potential ancestor
-      # @return [Boolean] true when child_node is a descendant of parent_node
-      def descendant_of?(child_node, parent_node)
-        curr = child_node.respond_to?(:parent) ? child_node.parent : nil
-        while curr
-          return true if curr == parent_node
-
-          curr = curr.respond_to?(:parent) ? curr.parent : nil
-        end
-        false
-      end
-    end
-  end
-end