html2rss 0.20.1 → 0.21.0

data/lib/html2rss/html_extractor/enclosure_extractor.rb

@@ -5,116 +5,87 @@ module Html2rss
     ##
     # Extracts enclosures from HTML tags using various strategies.
     class EnclosureExtractor
+      # CSS union query covering images, media, PDFs, iframes, and archives.
+      SELECTOR = [
+        'img[src]:not([src^="data"])',
+        'video source[src]',
+        'audio source[src]',
+        'audio[src]',
+        'a[href$=".pdf"]',
+        'iframe[src]',
+        'a[href$=".zip"]',
+        'a[href$=".tar.gz"]',
+        'a[href$=".tgz"]'
+      ].join(',').freeze
+
       # @param article_tag [Nokogiri::XML::Element] article container node
       # @param base_url [String, Html2rss::Url] base URL for relative enclosure links
       # @return [Array<Hash{Symbol => Object}>] normalized enclosure hashes
       def self.call(article_tag, base_url)
-        [
-          Extractors::Image,
-          Extractors::Media,
-          Extractors::Pdf,
-          Extractors::Iframe,
-          Extractors::Archive
-        ].flat_map { |strategy| strategy.call(article_tag, base_url:) }
+        article_tag.css(SELECTOR).filter_map do |element|
+          extract_from_element(element, base_url)
+        end
       end
-    end
 
-    # Extraction strategies for enclosure-like media/link tags.
-    module Extractors
-      # Extracts image enclosures from HTML tags.
-      # Finds all image sources and returns them in a format suitable for RSS.
-      class Image
-        # @param article_tag [Nokogiri::XML::Element] article container node
-        # @param base_url [String, Html2rss::Url] base URL for relative image sources
-        # @return [Array<Hash{Symbol => Object}>] image enclosure hashes
-        def self.call(article_tag, base_url:)
-          article_tag.css('img[src]:not([src^="data"])').filter_map do |img|
-            src = img['src'].to_s
-            next if src.empty?
-
-            abs_url = Url.from_relative(src, base_url)
-            {
-              url: abs_url,
-              type: RssBuilder::Enclosure.guess_content_type_from_url(abs_url, default: 'image/jpeg')
-            }
-          end
+      def self.extract_from_element(element, base_url)
+        case element.name
+        when 'img'
+          extract_image(element, base_url)
+        when 'video', 'audio', 'source'
+          extract_media(element, base_url)
+        when 'iframe'
+          extract_iframe(element, base_url)
+        when 'a'
+          extract_a(element, base_url)
         end
       end
 
-      # Extracts media enclosures (video/audio) from HTML tags.
-      class Media
-        # @param article_tag [Nokogiri::XML::Element] article container node
-        # @param base_url [String, Html2rss::Url] base URL for relative media sources
-        # @return [Array<Hash{Symbol => Object}>] media enclosure hashes
-        def self.call(article_tag, base_url:)
-          article_tag.css('video source[src], audio source[src], audio[src]').filter_map do |element|
-            src = element['src'].to_s
-            next if src.empty?
+      def self.extract_image(img, base_url)
+        src = img['src'].to_s
+        return if src.empty?
 
-            {
-              url: Url.from_relative(src, base_url),
-              type: element['type']
-            }
-          end
-        end
+        abs_url = Url.from_relative(src, base_url)
+        {
+          url: abs_url,
+          type: RssBuilder::Enclosure.guess_content_type_from_url(abs_url, default: 'image/jpeg')
+        }
       end
 
-      # Extracts PDF enclosures from HTML tags.
-      class Pdf
-        # @param article_tag [Nokogiri::XML::Element] article container node
-        # @param base_url [String, Html2rss::Url] base URL for relative PDF links
-        # @return [Array<Hash{Symbol => Object}>] PDF enclosure hashes
-        def self.call(article_tag, base_url:)
-          article_tag.css('a[href$=".pdf"]').filter_map do |link|
-            href = link['href'].to_s
-            next if href.empty?
+      def self.extract_media(element, base_url)
+        src = element['src'].to_s
+        return if src.empty?
 
-            abs_url = Url.from_relative(href, base_url)
-            {
-              url: abs_url,
-              type: RssBuilder::Enclosure.guess_content_type_from_url(abs_url)
-            }
-          end
-        end
+        {
+          url: Url.from_relative(src, base_url),
+          type: element['type']
+        }
       end
 
-      # Extracts iframe enclosures from HTML tags.
-      class Iframe
-        # @param article_tag [Nokogiri::XML::Element] article container node
-        # @param base_url [String, Html2rss::Url] base URL for relative iframe links
-        # @return [Array<Hash{Symbol => Object}>] iframe enclosure hashes
-        def self.call(article_tag, base_url:)
-          article_tag.css('iframe[src]').filter_map do |iframe|
-            src = iframe['src']
-            next if src.nil? || src.empty?
+      def self.extract_iframe(iframe, base_url)
+        src = iframe['src'].to_s
+        return if src.empty?
 
-            abs_url = Url.from_relative(src, base_url)
-            {
-              url: abs_url,
-              type: RssBuilder::Enclosure.guess_content_type_from_url(abs_url, default: 'text/html')
-            }
-          end
-        end
+        abs_url = Url.from_relative(src, base_url)
+        {
+          url: abs_url,
+          type: RssBuilder::Enclosure.guess_content_type_from_url(abs_url, default: 'text/html')
+        }
       end
 
-      # Extracts archive enclosures (zip, tar.gz, tgz) from HTML tags.
-      class Archive
-        # @param article_tag [Nokogiri::XML::Element] article container node
-        # @param base_url [String, Html2rss::Url] base URL for relative archive links
-        # @return [Array<Hash{Symbol => Object}>] archive enclosure hashes
-        def self.call(article_tag, base_url:)
-          article_tag.css('a[href$=".zip"], a[href$=".tar.gz"], a[href$=".tgz"]').filter_map do |link|
-            href = link['href'].to_s
-            next if href.empty?
+      def self.extract_a(link, base_url)
+        href = link['href'].to_s
+        return if href.empty?
 
-            abs_url = Url.from_relative(href, base_url)
-            {
-              url: abs_url,
-              type: 'application/zip'
-            }
-          end
+        abs_url = Url.from_relative(href, base_url)
+
+        if href.end_with?('.pdf')
+          { url: abs_url, type: RssBuilder::Enclosure.guess_content_type_from_url(abs_url) }
+        else
+          { url: abs_url, type: 'application/zip' }
         end
       end
+
+      private_class_method :extract_from_element, :extract_image, :extract_media, :extract_iframe, :extract_a
     end
   end
 end

data/lib/html2rss/html_extractor/list_candidates.rb

@@ -75,17 +75,11 @@ module Html2rss
       def each_anchor(anchor_filter:)
         return enum_for(:each_anchor, anchor_filter:) unless block_given?
 
-        traversal_root&.traverse do |node|
-          yield node if relevant_anchor?(node, anchor_filter:)
+        traversal_root&.css(HtmlExtractor::MAIN_ANCHOR_SELECTOR)&.each do |node|
+          yield node if anchor_filter.call(node)
         end
       end
 
-      def relevant_anchor?(node, anchor_filter:)
-        node.element? &&
-          node.matches?(HtmlExtractor::MAIN_ANCHOR_SELECTOR) &&
-          anchor_filter.call(node)
-      end
-
       def traversal_root
         parsed_body.at_css('body, html') || parsed_body.root
       end

data/lib/html2rss/html_extractor/semantic_anchor_candidates.rb

@@ -31,6 +31,8 @@ module Html2rss
 
       # Shared context for all anchors in one semantic container.
       class Context
+        attr_reader :container
+
         # Ancestor tags that usually indicate navigation/utility regions.
         UTILITY_LANDMARK_TAGS = %w[nav aside footer menu].freeze
 
@@ -56,7 +58,7 @@ module Html2rss
         def visible_text(node)
           return '' unless node
 
-          HtmlExtractor.extract_visible_text(node).to_s.strip
+          (@visible_texts ||= {}.compare_by_identity)[node] ||= HtmlExtractor.extract_visible_text(node).to_s.strip
         end
 
         # @param anchor [Nokogiri::XML::Node] anchor candidate
@@ -70,12 +72,6 @@ module Html2rss
         def utility_text?(text)
           @link_heuristics.utility_text?(text)
         end
-
-        # @param ancestors [Array<Nokogiri::XML::Node>]
-        # @return [Boolean] true when the anchor lives inside navigation chrome
-        def utility_landmark?(ancestors)
-          ancestors.any? { |node| UTILITY_LANDMARK_TAGS.include?(node.name) }
-        end
       end
 
       # One anchor plus the facts needed to decide whether it represents content.
@@ -131,7 +127,7 @@ module Html2rss
 
         # @return [Boolean] true when visible anchor text has words
         def meaningful_text?
-          text.scan(/\p{Alnum}+/).any?
+          @meaningful_text ||= text.match?(/\p{Alnum}/)
         end
 
         # @return [Boolean] true when the destination route has content signals
@@ -142,8 +138,17 @@ module Html2rss
         # @return [Boolean] true when the anchor is inside the selected heading
         def heading_anchor?
           heading = @context.heading
+          return false unless heading
+
+          curr = @anchor
+          container = @context.container
+          while curr.respond_to?(:parent)
+            return true if curr == heading
+            break if curr == container
 
-          heading && @anchor.ancestors.include?(heading)
+            curr = curr.parent
+          end
+          false
         end
 
         # @return [Boolean] true when anchor text exactly matches heading text
@@ -151,14 +156,14 @@ module Html2rss
           heading_text = @context.heading_text
 
           meaningful_text? &&
-            heading_text.scan(/\p{Alnum}+/).any? &&
+            heading_text.match?(/\p{Alnum}/) &&
             heading_text == text
         end
 
         private
 
         def representative_content_anchor?
-          heading_anchor? || meaningful_text? || content_like_destination?
+          meaningful_text? || content_like_destination? || heading_anchor?
         end
 
         def utility_text_suppressed?
@@ -174,7 +179,19 @@ module Html2rss
         def ineligible_anchor?
           destination_facts.high_confidence_utility_destination ||
             icon_only_anchor? ||
-            @context.utility_landmark?(@anchor.ancestors.to_a)
+            utility_landmark_ancestor?
+        end
+
+        def utility_landmark_ancestor?
+          curr = @anchor.parent
+          container = @context.container
+          while curr.respond_to?(:parent)
+            return true if Context::UTILITY_LANDMARK_TAGS.include?(curr.name)
+            break if curr == container
+
+            curr = curr.parent
+          end
+          false
         end
 
         def icon_only_anchor?

data/lib/html2rss/html_extractor/semantic_containers.rb

@@ -27,43 +27,17 @@ module Html2rss
 
       # @return [Array<Nokogiri::XML::Node>] candidate semantic containers
       def call
-        containers = SELECTORS.each_with_object([]) do |selector, memo|
-          collect_selector_containers(selector, memo)
+        cache = {}.compare_by_identity
+        candidates = @parsed_body.css(SELECTORS.join(',')).reject do |node|
+          HtmlExtractor.ignored_container_path?(node, cache)
         end
 
-        containers.sort_by { document_order.fetch(_1) }
-      end
-
-      private
-
-      def document_order
-        @document_order ||= begin
-          order = {}
-          index = 0
-
-          @parsed_body.traverse do |node|
-            next unless node.element?
-
-            order[node] = index
-            index += 1
-          end
-
-          order.compare_by_identity
-        end
-      end
-
-      def collect_selector_containers(selector, containers)
-        @parsed_body.css(selector).each do |container|
-          next if HtmlExtractor.ignored_container_path?(container)
-          next if seen[container]
-
-          seen[container] = true
-          containers << container
-        end
-      end
-
-      def seen
-        @seen ||= {}.compare_by_identity
+        # Preserve the original post-order traversal intent (specific-first)
+        # by sorting candidates by depth (descending) while keeping original document
+        # order for nodes at the same depth.
+        candidates.each_with_index
+                  .sort_by { |node, index| [-node.ancestors.size, index] }
+                  .map!(&:first)
       end
     end
   end

data/lib/html2rss/html_extractor.rb

@@ -4,15 +4,15 @@ module Html2rss
   ##
   # HtmlExtractor is responsible for extracting details (headline, url, images, etc.)
   # from an article_tag.
-  class HtmlExtractor
+  class HtmlExtractor # rubocop:disable Metrics/ClassLength
     # Tags ignored when extracting visible text content from article containers.
     INVISIBLE_CONTENT_TAGS = %w[svg script noscript style template].to_set.freeze
-    # Element path pattern ignored when traversing candidate article containers.
-    IGNORED_CONTAINER_PATH = /(nav|footer|header|svg|script|style)/i
     # Heading tags used to prioritize title extraction.
     HEADING_TAGS = %w[h1 h2 h3 h4 h5 h6].freeze
     # Selector used to derive non-headline description nodes.
     NON_HEADLINE_SELECTOR = (HEADING_TAGS.map { |tag| ":not(#{tag})" } + INVISIBLE_CONTENT_TAGS.to_a).freeze
+    # Element tags that indicate ignored DOM chrome when found in a container path.
+    IGNORED_CONTAINER_TAGS = %w[nav footer header svg script style].to_set.freeze
 
     # Anchor selector used to identify the canonical article link element.
     MAIN_ANCHOR_SELECTOR = begin
@@ -42,8 +42,39 @@ module Html2rss
         parts.join(separator).squeeze(' ').strip unless parts.empty?
       end
 
+      ##
+      # @param article_tag [Nokogiri::XML::Node] article-like container to search within
+      # @return [Nokogiri::XML::Node, nil] first eligible descendant anchor
+      def main_anchor_for(article_tag)
+        return article_tag if article_tag.name == 'a' && article_tag.matches?(MAIN_ANCHOR_SELECTOR)
+
+        article_tag.at_css(MAIN_ANCHOR_SELECTOR)
+      end
+
+      ##
+      # @param node [Nokogiri::XML::Node]
+      # @param cache [Hash, nil] identity cache used to store results (must use compare_by_identity)
+      # @return [Boolean] true when the node belongs to ignored DOM chrome
+      def ignored_container_path?(node, cache = nil)
+        return cache[node] if cache&.key?(node)
+
+        res = walk_ignored_container_path?(node)
+        cache[node] = res if cache
+        res
+      end
+
       private
 
+      def walk_ignored_container_path?(node)
+        curr = node
+        while curr.respond_to?(:parent)
+          return true if IGNORED_CONTAINER_TAGS.include?(curr.name)
+
+          curr = curr.parent
+        end
+        false
+      end
+
       def visible_child?(node)
         !INVISIBLE_CONTENT_TAGS.include?(node.name) &&
           !(node.name == 'a' && node['href']&.start_with?('#'))
@@ -80,26 +111,6 @@ module Html2rss
 
     attr_reader :article_tag, :base_url, :selected_anchor
 
-    class << self
-      ##
-      # @param article_tag [Nokogiri::XML::Node] article-like container to search within
-      # @return [Nokogiri::XML::Node, nil] first eligible descendant anchor
-      def main_anchor_for(article_tag)
-        return article_tag if article_tag.name == 'a' && article_tag.matches?(MAIN_ANCHOR_SELECTOR)
-
-        article_tag.at_css(MAIN_ANCHOR_SELECTOR)
-      end
-
-      ##
-      # @param node [Nokogiri::XML::Node, String] node or path to test
-      # @return [Boolean] true when the node belongs to ignored DOM chrome
-      def ignored_container_path?(node)
-        path = node.respond_to?(:path) ? node.path : node.to_s
-
-        path.match?(IGNORED_CONTAINER_PATH)
-      end
-    end
-
     def extract_url
       @extract_url ||= begin
         href = selected_anchor&.[]('href').to_s
@@ -115,14 +126,24 @@ module Html2rss
 
     def heading
       @heading ||= begin
-        heading_tags = article_tag.css(HEADING_TAGS.join(',')).group_by(&:name)
-        smallest_heading = heading_tags.keys.min
-        if smallest_heading
-          heading_tags[smallest_heading]&.max_by do |tag|
-            self.class.extract_visible_text(tag)&.size.to_i
-          end
-        end
+        tags = article_tag.css(HEADING_TAGS.join(','))
+        tags.any? ? select_best_heading(tags) : nil
+      end
+    end
+
+    def select_best_heading(tags)
+      min_tag_name = tags.map(&:name).min
+      best_tag = nil
+      max_size = -1
+
+      tags.each do |tag|
+        next if tag.name != min_tag_name
+
+        size = self.class.extract_visible_text(tag)&.size.to_i
+        (best_tag = tag) && (max_size = size) if size > max_size
       end
+
+      best_tag
     end
 
     def extract_description

data/lib/html2rss/rendering/description_builder.rb

@@ -25,12 +25,12 @@ module Html2rss
       # @param end_of_range [Integer] Optional, defaults to half the text length
       # @return [String]
       def self.remove_pattern_from_start(text, pattern, end_of_range: (text.size * 0.5).to_i)
-        return text unless text.is_a?(String) && pattern.is_a?(String)
+        return text unless text.is_a?(String) && pattern.is_a?(String) && !pattern.empty?
 
         index = text.index(pattern)
-        return text if index.nil? || index >= end_of_range
+        return text if index.nil? || index > end_of_range
 
-        text.gsub(/^(.{0,#{end_of_range}})#{Regexp.escape(pattern)}/, '\1')
+        "#{text[0, index]}#{text[(index + pattern.size)..]}"
       end
 
       # @param base [String] The base text content for the description

data/lib/html2rss/rss_builder/article.rb

@@ -9,6 +9,7 @@ module Html2rss
     ##
     # Article is a simple data object representing an article extracted from a page.
     # It is enumerable and responds to all keys specified in PROVIDED_KEYS.
+    # rubocop:disable Metrics/ClassLength
     class Article
       include Enumerable
       include Comparable
@@ -17,6 +18,11 @@ module Html2rss
       PROVIDED_KEYS = %i[id title description url image author guid published_at enclosures categories scraper].freeze
       # Separator used to build deterministic deduplication fingerprints.
       DEDUP_FINGERPRINT_SEPARATOR = '#!/'
+      # Sentinel object used to pre-initialize instance variables in the constructor.
+      # This ensures all Article instances share the exact same object shape (Ruby 3.3+ optimization),
+      # preventing performance warnings and slower instance variable access due to shape transitions
+      # when attributes are lazily/conditionally accessed in different sequences.
+      NOT_SET = Object.new.freeze
 
       # @param options [Hash{Symbol => String}]
       # @option options [String] :id stable article identifier
@@ -31,9 +37,9 @@ module Html2rss
       # @option options [Array<String>] :categories category labels
       # @option options [Class] :scraper scraper class that produced the article
       def initialize(**options)
-        @to_h = {}
-        options.each_pair { |key, value| @to_h[key] = value.freeze if value }
-        @to_h.freeze
+        @to_h = options.each_with_object({}) { |(k, v), h| h[k] = v.freeze if v }.freeze
+
+        @description = @url = @image = @guid = @enclosures = @enclosure = @categories = @published_at = NOT_SET
 
         return unless (unknown_keys = options.keys - PROVIDED_KEYS).any?
 
@@ -62,7 +68,9 @@ module Html2rss
 
       # @return [String] rendered article description
       def description
-        @description ||= Rendering::DescriptionBuilder.new(
+        return @description unless @description == NOT_SET
+
+        @description = Rendering::DescriptionBuilder.new(
           base: @to_h[:description],
           title:,
           url:,
@@ -73,12 +81,16 @@ module Html2rss
 
       # @return [Url, nil]
       def url
-        @url ||= Url.sanitize(@to_h[:url])
+        return @url unless @url == NOT_SET
+
+        @url = Url.sanitize(@to_h[:url])
       end
 
       # @return [Url, nil]
       def image
-        @image ||= Url.sanitize(@to_h[:image])
+        return @image unless @image == NOT_SET
+
+        @image = Url.sanitize(@to_h[:image])
       end
 
       # @return [String, nil]
@@ -87,7 +99,9 @@ module Html2rss
       # Generates a unique identifier based on the URL and ID using CRC32.
       # @return [String]
       def guid
-        @guid ||= Zlib.crc32(fetch_guid).to_s(36).encode('utf-8')
+        return @guid unless @guid == NOT_SET
+
+        @guid = Zlib.crc32(fetch_guid).to_s(36).encode('utf-8')
       end
 
       ##
@@ -100,27 +114,32 @@ module Html2rss
 
       # @return [Array<Html2rss::RssBuilder::Enclosure>] normalized enclosure objects
       def enclosures
-        @enclosures ||= Array(@to_h[:enclosures])
-                        .map { |enclosure| Html2rss::RssBuilder::Enclosure.new(**enclosure) }
+        return @enclosures unless @enclosures == NOT_SET
+
+        @enclosures = Array(@to_h[:enclosures])
+                      .map { |enclosure| Html2rss::RssBuilder::Enclosure.new(**enclosure) }
       end
 
       # @return [Html2rss::RssBuilder::Enclosure, nil]
       def enclosure
-        return @enclosure if defined?(@enclosure)
-
-        case (object = @to_h[:enclosures]&.first)
-        when Hash
-          @enclosure = Html2rss::RssBuilder::Enclosure.new(**object)
-        when nil
-          @enclosure = Html2rss::RssBuilder::Enclosure.new(url: image) if image
-        else
-          Log.warn "Article: unknown enclosure type: #{object.class}"
-        end
+        return @enclosure unless @enclosure == NOT_SET
+
+        @enclosure = case (object = @to_h[:enclosures]&.first)
+                     when Hash
+                       Html2rss::RssBuilder::Enclosure.new(**object)
+                     when nil
+                       Html2rss::RssBuilder::Enclosure.new(url: image) if image
+                     else
+                       Log.warn "Article: unknown enclosure type: #{object.class}"
+                       nil
+                     end
       end
 
       # @return [Array<String>] normalized, unique category names
       def categories
-        @categories ||= @to_h[:categories].dup.to_a.tap do |categories|
+        return @categories unless @categories == NOT_SET
+
+        @categories = @to_h[:categories].dup.to_a.tap do |categories|
           categories.map! { |category| category.to_s.strip }
           categories.reject!(&:empty?)
           categories.uniq!
@@ -130,11 +149,12 @@ module Html2rss
       # Parses and returns the published_at time.
       # @return [DateTime, nil]
       def published_at
-        return if (string = @to_h[:published_at].to_s.strip).empty?
+        return @published_at unless @published_at == NOT_SET
 
-        @published_at ||= DateTime.parse(string)
+        string = @to_h[:published_at].to_s.strip
+        @published_at = string.empty? ? nil : DateTime.parse(string)
       rescue ArgumentError
-        nil
+        @published_at = nil
       end
 
       # @return [Class, nil] scraper class that produced this article
@@ -183,5 +203,6 @@ module Html2rss
         value
       end
     end
+    # rubocop:enable Metrics/ClassLength
   end
 end

data/lib/html2rss/rss_builder/enclosure.rb

@@ -16,9 +16,11 @@ module Html2rss
       def self.guess_content_type_from_url(url, default: 'application/octet-stream')
         return default unless url
 
-        url = url.path.split('?').first
+        path = url.path
+        ext = File.extname(path)
+        ext = ext[1..] if ext.start_with?('.')
 
-        content_type = MIME::Types.type_for(File.extname(url).delete('.'))
+        content_type = MIME::Types.type_for(ext)
         content_type.first&.to_s || 'application/octet-stream'
       end