html2rss 0.20.1 → 0.22.0

data/lib/html2rss/html_extractor/enclosure_extractor.rb

@@ -5,116 +5,87 @@ module Html2rss
     ##
     # Extracts enclosures from HTML tags using various strategies.
     class EnclosureExtractor
+      # CSS union query covering images, media, PDFs, iframes, and archives.
+      SELECTOR = [
+        'img[src]:not([src^="data"])',
+        'video source[src]',
+        'audio source[src]',
+        'audio[src]',
+        'a[href$=".pdf"]',
+        'iframe[src]',
+        'a[href$=".zip"]',
+        'a[href$=".tar.gz"]',
+        'a[href$=".tgz"]'
+      ].join(',').freeze
+
       # @param article_tag [Nokogiri::XML::Element] article container node
       # @param base_url [String, Html2rss::Url] base URL for relative enclosure links
       # @return [Array<Hash{Symbol => Object}>] normalized enclosure hashes
       def self.call(article_tag, base_url)
-        [
-          Extractors::Image,
-          Extractors::Media,
-          Extractors::Pdf,
-          Extractors::Iframe,
-          Extractors::Archive
-        ].flat_map { |strategy| strategy.call(article_tag, base_url:) }
+        article_tag.css(SELECTOR).filter_map do |element|
+          extract_from_element(element, base_url)
+        end
       end
-    end
 
-    # Extraction strategies for enclosure-like media/link tags.
-    module Extractors
-      # Extracts image enclosures from HTML tags.
-      # Finds all image sources and returns them in a format suitable for RSS.
-      class Image
-        # @param article_tag [Nokogiri::XML::Element] article container node
-        # @param base_url [String, Html2rss::Url] base URL for relative image sources
-        # @return [Array<Hash{Symbol => Object}>] image enclosure hashes
-        def self.call(article_tag, base_url:)
-          article_tag.css('img[src]:not([src^="data"])').filter_map do |img|
-            src = img['src'].to_s
-            next if src.empty?
-
-            abs_url = Url.from_relative(src, base_url)
-            {
-              url: abs_url,
-              type: RssBuilder::Enclosure.guess_content_type_from_url(abs_url, default: 'image/jpeg')
-            }
-          end
+      def self.extract_from_element(element, base_url)
+        case element.name
+        when 'img'
+          extract_image(element, base_url)
+        when 'video', 'audio', 'source'
+          extract_media(element, base_url)
+        when 'iframe'
+          extract_iframe(element, base_url)
+        when 'a'
+          extract_a(element, base_url)
         end
       end
 
-      # Extracts media enclosures (video/audio) from HTML tags.
-      class Media
-        # @param article_tag [Nokogiri::XML::Element] article container node
-        # @param base_url [String, Html2rss::Url] base URL for relative media sources
-        # @return [Array<Hash{Symbol => Object}>] media enclosure hashes
-        def self.call(article_tag, base_url:)
-          article_tag.css('video source[src], audio source[src], audio[src]').filter_map do |element|
-            src = element['src'].to_s
-            next if src.empty?
+      def self.extract_image(img, base_url)
+        src = img['src'].to_s
+        return if src.empty?
 
-            {
-              url: Url.from_relative(src, base_url),
-              type: element['type']
-            }
-          end
-        end
+        abs_url = Url.from_relative(src, base_url)
+        {
+          url: abs_url,
+          type: RssBuilder::Enclosure.guess_content_type_from_url(abs_url, default: 'image/jpeg')
+        }
       end
 
-      # Extracts PDF enclosures from HTML tags.
-      class Pdf
-        # @param article_tag [Nokogiri::XML::Element] article container node
-        # @param base_url [String, Html2rss::Url] base URL for relative PDF links
-        # @return [Array<Hash{Symbol => Object}>] PDF enclosure hashes
-        def self.call(article_tag, base_url:)
-          article_tag.css('a[href$=".pdf"]').filter_map do |link|
-            href = link['href'].to_s
-            next if href.empty?
+      def self.extract_media(element, base_url)
+        src = element['src'].to_s
+        return if src.empty?
 
-            abs_url = Url.from_relative(href, base_url)
-            {
-              url: abs_url,
-              type: RssBuilder::Enclosure.guess_content_type_from_url(abs_url)
-            }
-          end
-        end
+        {
+          url: Url.from_relative(src, base_url),
+          type: element['type']
+        }
       end
 
-      # Extracts iframe enclosures from HTML tags.
-      class Iframe
-        # @param article_tag [Nokogiri::XML::Element] article container node
-        # @param base_url [String, Html2rss::Url] base URL for relative iframe links
-        # @return [Array<Hash{Symbol => Object}>] iframe enclosure hashes
-        def self.call(article_tag, base_url:)
-          article_tag.css('iframe[src]').filter_map do |iframe|
-            src = iframe['src']
-            next if src.nil? || src.empty?
+      def self.extract_iframe(iframe, base_url)
+        src = iframe['src'].to_s
+        return if src.empty?
 
-            abs_url = Url.from_relative(src, base_url)
-            {
-              url: abs_url,
-              type: RssBuilder::Enclosure.guess_content_type_from_url(abs_url, default: 'text/html')
-            }
-          end
-        end
+        abs_url = Url.from_relative(src, base_url)
+        {
+          url: abs_url,
+          type: RssBuilder::Enclosure.guess_content_type_from_url(abs_url, default: 'text/html')
+        }
       end
 
-      # Extracts archive enclosures (zip, tar.gz, tgz) from HTML tags.
-      class Archive
-        # @param article_tag [Nokogiri::XML::Element] article container node
-        # @param base_url [String, Html2rss::Url] base URL for relative archive links
-        # @return [Array<Hash{Symbol => Object}>] archive enclosure hashes
-        def self.call(article_tag, base_url:)
-          article_tag.css('a[href$=".zip"], a[href$=".tar.gz"], a[href$=".tgz"]').filter_map do |link|
-            href = link['href'].to_s
-            next if href.empty?
+      def self.extract_a(link, base_url)
+        href = link['href'].to_s
+        return if href.empty?
 
-            abs_url = Url.from_relative(href, base_url)
-            {
-              url: abs_url,
-              type: 'application/zip'
-            }
-          end
+        abs_url = Url.from_relative(href, base_url)
+
+        if href.end_with?('.pdf')
+          { url: abs_url, type: RssBuilder::Enclosure.guess_content_type_from_url(abs_url) }
+        else
+          { url: abs_url, type: 'application/zip' }
         end
       end
+
+      private_class_method :extract_from_element, :extract_image, :extract_media, :extract_iframe, :extract_a
     end
   end
 end

data/lib/html2rss/html_extractor/heading_extractor.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class HtmlExtractor
+    ##
+    # HeadingExtractor identifies and returns the best heading element within a container.
+    class HeadingExtractor
+      # Heading tags used to prioritize title extraction.
+      HEADING_TAGS = HtmlExtractor::HEADING_TAGS
+
+      class << self
+        ##
+        # @param article_tag [Nokogiri::XML::Element] container node
+        # @param fallback_anchorless [Boolean] whether to use fallback search
+        # @param selected_anchor [Nokogiri::XML::Node, nil] anchor element
+        # @return [Nokogiri::XML::Node, nil] the heading node, if found
+        def call(article_tag, fallback_anchorless:, selected_anchor:)
+          tags = article_tag.css(HEADING_TAGS.join(','))
+          if tags.any?
+            select_best_heading(tags)
+          elsif fallback_anchorless && selected_anchor.nil?
+            fallback_heading(article_tag)
+          end
+        end
+
+        private
+
+        def select_best_heading(tags)
+          min_tag_name = tags.map(&:name).min
+          best_tag = nil
+          max_size = -1
+
+          tags.each do |tag|
+            next if tag.name != min_tag_name
+
+            size = TextExtractor.call(tag)&.size.to_i
+            (best_tag = tag) && (max_size = size) if size > max_size
+          end
+
+          best_tag
+        end
+
+        def fallback_heading(article_tag)
+          fallback_tags = article_tag.css('strong, b, [class*="title"], [class*="font-bold"], [class*="font-semibold"]')
+          fallback_tags.find { |t| !TextExtractor.call(t).to_s.strip.empty? }
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html_extractor/id_generator.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require 'zlib'
+
+module Html2rss
+  class HtmlExtractor
+    ##
+    # IdGenerator determines the unique ID for an article container node.
+    class IdGenerator
+      class << self
+        ##
+        # @param article_tag [Nokogiri::XML::Element] container node
+        # @param heading [Nokogiri::XML::Node, nil] heading node
+        # @param url [Html2rss::Url, nil] absolute article URL
+        # @param selected_anchor [Nokogiri::XML::Node, nil] anchor element
+        # @param fallback_anchorless [Boolean] whether to use fallback hashing
+        # @return [String, nil] the generated ID, if any
+        def call(article_tag, heading:, url:, selected_anchor:, fallback_anchorless:)
+          id_from_dom = parse_id_from_dom(article_tag, url, selected_anchor)
+          return id_from_dom if id_from_dom
+
+          heading_text = resolve_heading_text(article_tag, heading, fallback_anchorless)
+          if heading_text && !heading_text.strip.empty?
+            generate_slug(heading_text)
+          elsif fallback_anchorless
+            generate_content_hash(article_tag)
+          end
+        end
+
+        private
+
+        def parse_id_from_dom(article_tag, url, selected_anchor)
+          candidates = [article_tag['id'], article_tag.at_css('[id]')&.attr('id')]
+          candidates += [url&.path, url&.query] if selected_anchor
+          candidates.compact.reject(&:empty?).first
+        end
+
+        def resolve_heading_text(article_tag, heading, fallback_anchorless)
+          text = heading ? TextExtractor.call(heading) : nil
+          if text.nil? || text.strip.empty?
+            fallback_text_node_content(article_tag, fallback_anchorless)
+          else
+            text
+          end
+        end
+
+        def fallback_text_node_content(article_tag, fallback_anchorless)
+          return unless fallback_anchorless
+
+          article_tag.xpath('.//text()').find { |t| !t.text.strip.empty? }&.text&.strip
+        end
+
+        def generate_slug(text)
+          slug = text.downcase.gsub(/[^a-z0-9]+/, '-')
+          slug = slug[1..] if slug.start_with?('-')
+          slug = slug[0..-2] if slug.end_with?('-')
+          slug unless slug.empty?
+        end
+
+        def generate_content_hash(article_tag)
+          text = TextExtractor.call(article_tag).to_s.strip
+          Zlib.crc32(text).to_s(36) unless text.empty?
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html_extractor/list_candidates.rb

@@ -75,17 +75,11 @@ module Html2rss
       def each_anchor(anchor_filter:)
         return enum_for(:each_anchor, anchor_filter:) unless block_given?
 
-        traversal_root&.traverse do |node|
-          yield node if relevant_anchor?(node, anchor_filter:)
+        traversal_root&.css(HtmlExtractor::MAIN_ANCHOR_SELECTOR)&.each do |node|
+          yield node if anchor_filter.call(node)
         end
       end
 
-      def relevant_anchor?(node, anchor_filter:)
-        node.element? &&
-          node.matches?(HtmlExtractor::MAIN_ANCHOR_SELECTOR) &&
-          anchor_filter.call(node)
-      end
-
       def traversal_root
         parsed_body.at_css('body, html') || parsed_body.root
       end

data/lib/html2rss/html_extractor/semantic_anchor_candidates.rb

@@ -31,6 +31,8 @@ module Html2rss
 
       # Shared context for all anchors in one semantic container.
       class Context
+        attr_reader :container
+
         # Ancestor tags that usually indicate navigation/utility regions.
         UTILITY_LANDMARK_TAGS = %w[nav aside footer menu].freeze
 
@@ -56,7 +58,7 @@ module Html2rss
         def visible_text(node)
           return '' unless node
 
-          HtmlExtractor.extract_visible_text(node).to_s.strip
+          (@visible_texts ||= {}.compare_by_identity)[node] ||= HtmlExtractor.extract_visible_text(node).to_s.strip
         end
 
         # @param anchor [Nokogiri::XML::Node] anchor candidate
@@ -70,12 +72,6 @@ module Html2rss
         def utility_text?(text)
           @link_heuristics.utility_text?(text)
         end
-
-        # @param ancestors [Array<Nokogiri::XML::Node>]
-        # @return [Boolean] true when the anchor lives inside navigation chrome
-        def utility_landmark?(ancestors)
-          ancestors.any? { |node| UTILITY_LANDMARK_TAGS.include?(node.name) }
-        end
       end
 
       # One anchor plus the facts needed to decide whether it represents content.
@@ -131,7 +127,7 @@ module Html2rss
 
         # @return [Boolean] true when visible anchor text has words
         def meaningful_text?
-          text.scan(/\p{Alnum}+/).any?
+          @meaningful_text ||= text.match?(/\p{Alnum}/)
         end
 
         # @return [Boolean] true when the destination route has content signals
@@ -142,8 +138,17 @@ module Html2rss
         # @return [Boolean] true when the anchor is inside the selected heading
         def heading_anchor?
           heading = @context.heading
+          return false unless heading
+
+          curr = @anchor
+          container = @context.container
+          while curr.respond_to?(:parent)
+            return true if curr == heading
+            break if curr == container
 
-          heading && @anchor.ancestors.include?(heading)
+            curr = curr.parent
+          end
+          false
         end
 
         # @return [Boolean] true when anchor text exactly matches heading text
@@ -151,14 +156,14 @@ module Html2rss
           heading_text = @context.heading_text
 
           meaningful_text? &&
-            heading_text.scan(/\p{Alnum}+/).any? &&
+            heading_text.match?(/\p{Alnum}/) &&
             heading_text == text
         end
 
         private
 
         def representative_content_anchor?
-          heading_anchor? || meaningful_text? || content_like_destination?
+          meaningful_text? || content_like_destination? || heading_anchor?
         end
 
         def utility_text_suppressed?
@@ -174,7 +179,19 @@ module Html2rss
         def ineligible_anchor?
           destination_facts.high_confidence_utility_destination ||
             icon_only_anchor? ||
-            @context.utility_landmark?(@anchor.ancestors.to_a)
+            utility_landmark_ancestor?
+        end
+
+        def utility_landmark_ancestor?
+          curr = @anchor.parent
+          container = @context.container
+          while curr.respond_to?(:parent)
+            return true if Context::UTILITY_LANDMARK_TAGS.include?(curr.name)
+            break if curr == container
+
+            curr = curr.parent
+          end
+          false
         end
 
         def icon_only_anchor?

data/lib/html2rss/html_extractor/semantic_containers.rb

@@ -27,43 +27,17 @@ module Html2rss
 
       # @return [Array<Nokogiri::XML::Node>] candidate semantic containers
       def call
-        containers = SELECTORS.each_with_object([]) do |selector, memo|
-          collect_selector_containers(selector, memo)
+        cache = {}.compare_by_identity
+        candidates = @parsed_body.css(SELECTORS.join(',')).reject do |node|
+          HtmlExtractor.ignored_container_path?(node, cache)
         end
 
-        containers.sort_by { document_order.fetch(_1) }
-      end
-
-      private
-
-      def document_order
-        @document_order ||= begin
-          order = {}
-          index = 0
-
-          @parsed_body.traverse do |node|
-            next unless node.element?
-
-            order[node] = index
-            index += 1
-          end
-
-          order.compare_by_identity
-        end
-      end
-
-      def collect_selector_containers(selector, containers)
-        @parsed_body.css(selector).each do |container|
-          next if HtmlExtractor.ignored_container_path?(container)
-          next if seen[container]
-
-          seen[container] = true
-          containers << container
-        end
-      end
-
-      def seen
-        @seen ||= {}.compare_by_identity
+        # Preserve the original post-order traversal intent (specific-first)
+        # by sorting candidates by depth (descending) while keeping original document
+        # order for nodes at the same depth.
+        candidates.each_with_index
+                  .sort_by { |node, index| [-node.ancestors.size, index] }
+                  .map!(&:first)
       end
     end
   end

data/lib/html2rss/html_extractor/text_extractor.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class HtmlExtractor
+    ##
+    # TextExtractor extracts visible text from DOM elements, preserving lists
+    # and block spacing while sanitizing white spaces.
+    class TextExtractor
+      # HTML block elements that trigger line breaks or special formatting.
+      BLOCK_TAGS = %w[p div li ul ol h1 h2 h3 h4 h5 h6 tr br].to_set.freeze
+      # Tags ignored when extracting visible text content.
+      INVISIBLE_CONTENT_TAGS = %w[svg script noscript style template].to_set.freeze
+
+      class << self
+        ##
+        # @param tag [Nokogiri::XML::Node] the node from which to extract visible text
+        # @param separator [String] separator used to join text fragments (default is a space)
+        # @param exclude_nodes [Array<Nokogiri::XML::Node>, nil] nodes to exclude from extraction
+        # @return [String, nil] the concatenated visible text, or nil if none is found
+        def call(tag, separator: ' ', exclude_nodes: nil)
+          return tag.text.gsub(/\s+/, ' ').strip if tag.respond_to?(:text?) && tag.text?
+
+          parts = iterate_children(tag, separator, exclude_nodes)
+          return if parts.empty?
+
+          parts.join.squeeze(' ').strip
+        end
+
+        private
+
+        def iterate_children(tag, separator, exclude_nodes)
+          last = false
+          tag.children.each_with_object([]) do |c, p|
+            next if exclude_nodes&.include?(c) || !visible_child?(c)
+
+            text, block = process_child_node(c, separator, exclude_nodes)
+            next if text.empty?
+
+            append_separator!(p, separator, block, last)
+            (p << text) && (last = block)
+          end
+        end
+
+        def process_child_node(child, separator, exclude_nodes)
+          child_text = get_child_text(child, separator, exclude_nodes)
+          return ['', false] if child_text.empty?
+
+          child_text = "- #{child_text}" if child.name == 'li'
+          [child_text, BLOCK_TAGS.include?(child.name)]
+        end
+
+        def get_child_text(child, separator, exclude_nodes)
+          if child.children.empty?
+            child.text.to_s.gsub(/\s+/, ' ').strip
+          else
+            call(child, separator:, exclude_nodes:).to_s.strip
+          end
+        end
+
+        def append_separator!(parts, separator, is_block, last_was_block)
+          return if parts.empty?
+
+          parts << if is_block || last_was_block
+                     (separator == ' ' ? "\n" : separator)
+                   else
+                     ' '
+                   end
+        end
+
+        def visible_child?(node)
+          !INVISIBLE_CONTENT_TAGS.include?(node.name) &&
+            !(node.name == 'a' && node['href']&.start_with?('#'))
+        end
+      end
+    end
+  end
+end