html2rss 0.22.2 → 0.23.0

data/lib/html2rss/html/article_extractor.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    ##
+    # ArticleExtractor is responsible for extracting details (headline, url, images, etc.)
+    # from an article_tag DOM node. DOM chrome helpers live on {Navigator}.
+    class ArticleExtractor
+      class << self
+        ##
+        # Extracts article attributes from a DOM element.
+        #
+        # @param article_tag [Nokogiri::XML::Node] article-like container to extract from
+        # @param base_url [String, Html2rss::Url] base url used to resolve relative links
+        # @param selected_anchor [Nokogiri::XML::Node, nil] explicit primary anchor for the container
+        # @param fallback_anchorless [Boolean] whether to fall back to anchorless extraction
+        # @return [Hash{Symbol => Object}] extracted article attributes
+        def call(article_tag, base_url:, selected_anchor: nil, fallback_anchorless: false)
+          new(article_tag, base_url:, selected_anchor:, fallback_anchorless:).call
+        end
+      end
+
+      ##
+      # @param article_tag [Nokogiri::XML::Node] article-like container to extract from
+      # @param base_url [String, Html2rss::Url] base url used to resolve relative links
+      # @param selected_anchor [Nokogiri::XML::Node, nil] explicit primary anchor for the container
+      # @param fallback_anchorless [Boolean] whether to fall back to anchorless extraction
+      def initialize(article_tag, base_url:, selected_anchor: nil, fallback_anchorless: false)
+        raise ArgumentError, 'article_tag is required' unless article_tag
+
+        @article_tag = article_tag
+        @base_url = base_url
+        @selected_anchor = selected_anchor
+        @fallback_anchorless = fallback_anchorless
+      end
+
+      # @return [Hash{Symbol => Object}] extracted article attributes
+      def call
+        {
+          title: extract_title,
+          url: extract_url,
+          image: extract_image,
+          description: extract_description,
+          id: generate_id,
+          published_at: extract_published_at,
+          enclosures: extract_enclosures,
+          categories: extract_categories
+        }
+      end
+
+      private
+
+      attr_reader :article_tag, :base_url, :selected_anchor
+
+      def extract_url
+        @extract_url ||= begin
+          href = selected_anchor&.[]('href').to_s
+
+          if href.empty?
+            anchorless_url_fallback
+          else
+            Url.from_relative(href.split('#').first.strip, base_url)
+          end
+        end
+      end
+
+      def anchorless_url_fallback
+        return unless @fallback_anchorless
+
+        id = generate_id
+        Url.from_relative("##{id}", base_url) if id
+      end
+
+      # rubocop:disable Metrics/CyclomaticComplexity
+      def extract_title
+        source = heading || selected_anchor
+        title_text = source ? Navigator.extract_visible_text(source) : fallback_anchorless_title
+        return unless title_text
+
+        kicker = kicker_node ? Navigator.extract_visible_text(kicker_node).to_s.strip : nil
+        kicker && !kicker.empty? && !title_text.include?(kicker) ? "#{kicker}: #{title_text}" : title_text
+      end
+      # rubocop:enable Metrics/CyclomaticComplexity
+
+      def fallback_anchorless_title
+        return unless @fallback_anchorless && selected_anchor.nil?
+
+        text_node = article_tag.xpath('.//text()').find { |t| !t.text.strip.empty? }
+        text_node&.text&.strip
+      end
+
+      def heading
+        @heading ||= HeadingExtractor.call(
+          article_tag,
+          fallback_anchorless: @fallback_anchorless,
+          selected_anchor:
+        )
+      end
+
+      def kicker_node
+        @kicker_node ||= begin
+          selector = '[data-tb-kicker], [class*="kicker"], [class*="eyebrow"], ' \
+                     '[class*="pre-title"], [class*="pretitle"], [class*="overline"]'
+          node = article_tag.at_css(selector)
+          node && heading && (node == heading || Navigator.descendant_of?(node, heading)) ? nil : node
+        end
+      end
+
+      def extract_description
+        exclude = [heading, selected_anchor, kicker_node].compact
+        description = Navigator.extract_visible_text(article_tag, exclude_nodes: exclude)
+        return if description.nil?
+
+        desc = description.strip
+        desc.empty? ? nil : desc
+      end
+
+      def generate_id
+        @generate_id ||= IdGenerator.call(
+          article_tag,
+          heading:,
+          url: (selected_anchor ? extract_url : nil),
+          selected_anchor:,
+          fallback_anchorless: @fallback_anchorless
+        )
+      end
+
+      def extract_image = ImageExtractor.call(article_tag, base_url:)
+      def extract_published_at = DateExtractor.call(article_tag)
+      def extract_enclosures = EnclosureExtractor.call(article_tag, base_url)
+      def extract_categories = CategoryExtractor.call(article_tag)
+    end
+  end
+end

data/lib/html2rss/html/navigator/text_extractor.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    class Navigator
+      ##
+      # TextExtractor extracts visible text from DOM elements, preserving lists
+      # and block spacing while sanitizing white spaces.
+      class TextExtractor
+        # HTML block elements that trigger line breaks or special formatting.
+        BLOCK_TAGS = %w[p div li ul ol h1 h2 h3 h4 h5 h6 tr br].to_set.freeze
+        # Tags ignored when extracting visible text content.
+        INVISIBLE_CONTENT_TAGS = %w[svg script noscript style template].to_set.freeze
+
+        class << self
+          ##
+          # @param tag [Nokogiri::XML::Node] the node from which to extract visible text
+          # @param separator [String] separator used to join text fragments (default is a space)
+          # @param exclude_nodes [Array<Nokogiri::XML::Node>, nil] nodes to exclude from extraction
+          # @return [String, nil] the concatenated visible text, or nil if none is found
+          def call(tag, separator: ' ', exclude_nodes: nil)
+            return tag.text.gsub(/\s+/, ' ').strip if tag.respond_to?(:text?) && tag.text?
+
+            parts = iterate_children(tag, separator, exclude_nodes)
+            return if parts.empty?
+
+            parts.join.squeeze(' ').strip
+          end
+
+          private
+
+          def iterate_children(tag, separator, exclude_nodes)
+            last = false
+            tag.children.each_with_object([]) do |c, p|
+              next if exclude_nodes&.include?(c) || !visible_child?(c)
+
+              text, block = process_child_node(c, separator, exclude_nodes)
+              next if text.empty?
+
+              append_separator!(p, separator, block, last)
+              (p << text) && (last = block)
+            end
+          end
+
+          def process_child_node(child, separator, exclude_nodes)
+            child_text = get_child_text(child, separator, exclude_nodes)
+            return ['', false] if child_text.empty?
+
+            child_text = "- #{child_text}" if child.name == 'li'
+            [child_text, BLOCK_TAGS.include?(child.name)]
+          end
+
+          def get_child_text(child, separator, exclude_nodes)
+            if child.children.empty?
+              child.text.to_s.gsub(/\s+/, ' ').strip
+            else
+              call(child, separator:, exclude_nodes:).to_s.strip
+            end
+          end
+
+          def append_separator!(parts, separator, is_block, last_was_block)
+            return if parts.empty?
+
+            parts << if is_block || last_was_block
+                       (separator == ' ' ? "\n" : separator)
+                     else
+                       ' '
+                     end
+          end
+
+          def visible_child?(node)
+            !INVISIBLE_CONTENT_TAGS.include?(node.name) &&
+              !(node.name == 'a' && node['href']&.start_with?('#'))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html/navigator.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    ##
+    # Navigator owns DOM chrome recognition and node traversal helpers.
+    class Navigator
+      # Heading tags used to prioritize title extraction and container assessment.
+      HEADING_TAGS = %w[h1 h2 h3 h4 h5 h6].freeze
+
+      # Element tags that indicate ignored DOM chrome when found in a container path.
+      IGNORED_CONTAINER_TAGS = %w[nav footer header svg script style].to_set.freeze
+
+      # Layout roots and chrome tags excluded from class-clustering candidate nodes.
+      CLUSTER_EXCLUDED_TAGS = Set['html', 'body', 'nav', 'footer', 'header', 'svg', 'script', 'style'].freeze
+
+      # Ancestor tags that usually indicate navigation/utility regions inside a content container.
+      UTILITY_LANDMARK_TAGS = %w[nav aside footer menu].to_set.freeze
+
+      # Anchor selector used to identify the canonical article link element.
+      MAIN_ANCHOR_SELECTOR = begin
+        buf = +'a[href]:not([href=""])'
+        %w[# javascript: mailto: tel: file:// sms: data:].each do |prefix|
+          buf << %[:not([href^="#{prefix}"])]
+        end
+        buf.freeze
+      end
+
+      class << self
+        ##
+        # Extracts visible text from a given node and its children.
+        #
+        # @param tag [Nokogiri::XML::Node] the node from which to extract visible text
+        # @param separator [String] separator used to join text fragments (default is a space)
+        # @param exclude_nodes [Array<Nokogiri::XML::Node>, nil] nodes to exclude from extraction
+        # @return [String, nil] the concatenated visible text, or nil if none is found
+        def extract_visible_text(tag, separator: ' ', exclude_nodes: nil)
+          TextExtractor.call(tag, separator:, exclude_nodes:)
+        end
+
+        ##
+        # @param article_tag [Nokogiri::XML::Node] article-like container to search within
+        # @return [Nokogiri::XML::Node, nil] first eligible descendant anchor
+        def main_anchor_for(article_tag)
+          return article_tag if article_tag.name == 'a' && article_tag.matches?(MAIN_ANCHOR_SELECTOR)
+
+          article_tag.at_css(MAIN_ANCHOR_SELECTOR)
+        end
+
+        ##
+        # @param node [Nokogiri::XML::Node]
+        # @param cache [Hash, nil] identity cache used to store results (must use compare_by_identity)
+        # @return [Boolean] true when the node belongs to ignored DOM chrome
+        # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+        def ignored_container_path?(node, cache = nil)
+          return cache[node] if cache&.key?(node)
+
+          curr = node
+          visited = []
+          is_ignored = false
+
+          while curr.respond_to?(:parent) && curr
+            if cache&.key?(curr)
+              is_ignored = cache[curr]
+              break
+            end
+
+            if IGNORED_CONTAINER_TAGS.include?(curr.name)
+              is_ignored = true
+              break
+            end
+
+            visited << curr
+            curr = curr.parent
+          end
+          visited.each { |n| cache[n] = is_ignored } if cache
+
+          is_ignored
+        end
+        # rubocop:enable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+
+        ##
+        # Returns the first parent that satisfies the condition.
+        # If the condition is met, it returns the node itself.
+        #
+        # @param node [Nokogiri::XML::Node] The node to start the search from.
+        # @param condition [Proc] The condition to be met.
+        # @return [Nokogiri::XML::Node, nil] The first parent that satisfies the condition.
+        def parent_until_condition(node, condition)
+          while node && !node.document? && node.name != 'html'
+            return node if condition.call(node)
+
+            node = node.parent
+          end
+        end
+
+        ##
+        # Think of it as `css_upwards` method.
+        # It searches for the closest parent that matches the given selector.
+        #
+        # @param current_tag [Nokogiri::XML::Node, nil] starting node
+        # @param selector [String] CSS selector to search upwards for
+        # @return [Nokogiri::XML::Node, nil] first matching node in upward traversal
+        def find_closest_selector_upwards(current_tag, selector)
+          while current_tag
+            found = current_tag.at_css(selector)
+            return found if found
+
+            return nil unless current_tag.respond_to?(:parent)
+
+            current_tag = current_tag.parent
+          end
+        end
+
+        ##
+        # Searches for the closest parent that matches the given tag name.
+        #
+        # @param current_tag [Nokogiri::XML::Node] starting node
+        # @param tag_name [String] tag name to find in ancestors
+        # @return [Nokogiri::XML::Node, nil] matching ancestor node
+        def find_tag_in_ancestors(current_tag, tag_name)
+          return current_tag if current_tag.name == tag_name
+
+          current_tag.ancestors(tag_name).first
+        end
+
+        ##
+        # Returns true if child_node is a descendant of parent_node.
+        # Walks up using parent pointers to avoid NodeSet allocations.
+        #
+        # @param child_node [Nokogiri::XML::Node] potential descendant
+        # @param parent_node [Nokogiri::XML::Node] potential ancestor
+        # @return [Boolean] true when child_node is a descendant of parent_node
+        def descendant_of?(child_node, parent_node)
+          curr = child_node.respond_to?(:parent) ? child_node.parent : nil
+          while curr
+            return true if curr == parent_node
+
+            curr = curr.respond_to?(:parent) ? curr.parent : nil
+          end
+          false
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html/rendering/audio_renderer.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require 'cgi'
+
+module Html2rss
+  module Html
+    module Rendering
+      # Renders an HTML <audio> tag from a URL and type.
+      class AudioRenderer
+        # @param url [String, Html2rss::Url] media URL for the audio source
+        # @param type [String] MIME type for the audio source
+        def initialize(url:, type:)
+          @url = url
+          @type = type
+        end
+
+        # @return [String] HTML audio snippet for article rendering
+        def to_html
+          [
+            '<audio controls preload="none" referrerpolicy="no-referrer" crossorigin="anonymous">',
+            %(<source src="#{escaped_url}" type="#{escaped_type}">),
+            '</audio>'
+          ].join
+        end
+
+        private
+
+        def escaped_url
+          CGI.escapeHTML(@url.to_s)
+        end
+
+        def escaped_type
+          CGI.escapeHTML(@type.to_s)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html/rendering/description_builder.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    module Rendering
+      # Builds a sanitized article description from the base text, title, and optional media.
+      #
+      # Combines media elements (images, audio, video, PDFs) with sanitized text content
+      # to create rich RSS descriptions that reveal more scraped information.
+      #
+      # @example Basic usage
+      #   builder = DescriptionBuilder.new(
+      #     base: "Article content",
+      #     title: "Article Title",
+      #     url: "https://example.com",
+      #     enclosures: [enclosure_object],
+      #     image: "https://example.com/image.jpg"
+      #   )
+      #   description = builder.call
+      class DescriptionBuilder
+        # Removes the specified pattern from the beginning of the text
+        # within a given range if the pattern occurs before the range's end.
+        #
+        # @param text [String]
+        # @param pattern [String]
+        # @param end_of_range [Integer] Optional, defaults to half the text length
+        # @return [String]
+        def self.remove_pattern_from_start(text, pattern, end_of_range: (text.size * 0.5).to_i)
+          return text unless text.is_a?(String) && pattern.is_a?(String) && !pattern.empty?
+
+          index = text.index(pattern)
+          return text if index.nil? || index > end_of_range
+
+          "#{text[0, index]}#{text[(index + pattern.size)..]}"
+        end
+
+        # @param base [String] The base text content for the description
+        # @param title [String] The article title (used for alt text and title removal)
+        # @param url [String, Html2rss::Url] The article URL (used for sanitization)
+        # @param enclosures [Array<Html2rss::Article::Enclosure>, nil] Media enclosures
+        # @param image [String, Html2rss::Url, nil] Fallback image URL
+        def initialize(base:, title:, url:, enclosures:, image:)
+          @base = base.to_s
+          @title = title
+          @url = url
+          @enclosures = Array(enclosures)
+          @image = image
+        end
+
+        # Generates the complete description with media and sanitized text.
+        #
+        # @return [String, nil] The complete description or nil if empty
+        def call
+          fragments = []
+          fragments.concat(Array(rendered_media))
+          fragments << processed_base_description
+
+          result = fragments.compact.join("\n\n").strip
+          result.empty? ? nil : result
+        end
+
+        private
+
+        def rendered_media
+          rendered = render_enclosures
+          return rendered if rendered.any?
+          return render_fallback_image if @image
+
+          []
+        end
+
+        def render_enclosures
+          @enclosures.filter_map do |enclosure|
+            MediaRenderer.for(enclosure:, image: @image, title: @title)&.to_html
+          end
+        end
+
+        def render_fallback_image
+          [MediaRenderer.for(enclosure: nil, image: @image, title: @title)&.to_html]
+        end
+
+        def processed_base_description
+          text = self.class.remove_pattern_from_start(@base, @title)
+          Html2rss::Selectors::PostProcessors::SanitizeHtml.get(text, @url)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html/rendering/image_renderer.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'cgi'
+
+module Html2rss
+  module Html
+    module Rendering
+      # Renders an HTML <img> tag from a URL and title.
+      class ImageRenderer
+        # @param url [String, Html2rss::Url] image URL for the src attribute
+        # @param title [String, nil] title/alt text for the image
+        def initialize(url:, title:)
+          @url = url
+          @title = title
+        end
+
+        # @return [String] HTML image snippet for article rendering
+        def to_html
+          attributes = [
+            %(src="#{escaped_url}"),
+            %(alt="#{escaped_title}"),
+            %(title="#{escaped_title}"),
+            'loading="lazy"',
+            'referrerpolicy="no-referrer"',
+            'decoding="async"',
+            'crossorigin="anonymous"'
+          ]
+          "<img #{attributes.join(' ')}>"
+        end
+
+        private
+
+        def escaped_url
+          CGI.escapeHTML(@url.to_s)
+        end
+
+        def escaped_title
+          CGI.escapeHTML(@title.to_s)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html/rendering/media_renderer.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    module Rendering
+      # Factory: picks the appropriate renderer for a given enclosure or fallback image.
+      class MediaRenderer
+        # @param enclosure [Html2rss::Article::Enclosure, nil]
+        # @param image [String, Html2rss::Url, nil] Fallback image URL
+        # @param title [String]
+        # @return [ImageRenderer, VideoRenderer, AudioRenderer, PdfRenderer, nil]
+        def self.for(enclosure:, image:, title:)
+          return ImageRenderer.new(url: image, title:) if enclosure.nil? && image
+          return nil unless enclosure
+
+          create_renderer_for_type(enclosure.type, url: enclosure.url, title:)
+        end
+
+        # @private
+        # @param type [String, nil] enclosure MIME type
+        # @param url [String, Html2rss::Url] enclosure URL
+        # @param title [String, nil] title used by image renderer
+        # @return [ImageRenderer, VideoRenderer, AudioRenderer, PdfRenderer, nil]
+        def self.create_renderer_for_type(type, url:, title:)
+          case type
+          when %r{^image/}
+            ImageRenderer.new(url:, title:)
+          when %r{^video/}
+            VideoRenderer.new(url:, type:)
+          when %r{^audio/}
+            AudioRenderer.new(url:, type:)
+          when 'application/pdf'
+            PdfRenderer.new(url:)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html/rendering/pdf_renderer.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'cgi'
+
+module Html2rss
+  module Html
+    module Rendering
+      # Renders an HTML <iframe> for PDF documents.
+      class PdfRenderer
+        # @param url [String, Html2rss::Url] PDF URL rendered in the iframe
+        def initialize(url:)
+          @url = url
+        end
+
+        # @return [String] HTML iframe snippet for PDF rendering
+        def to_html
+          attributes = [
+            %(src="#{escaped_url}"),
+            'width="100%"',
+            'height="75vh"',
+            'sandbox=""',
+            'referrerpolicy="no-referrer"',
+            'loading="lazy"'
+          ]
+          "<iframe #{attributes.join(' ')}></iframe>"
+        end
+
+        private
+
+        def escaped_url
+          CGI.escapeHTML(@url.to_s)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html/rendering/video_renderer.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require 'cgi'
+
+module Html2rss
+  module Html
+    module Rendering
+      # Renders an HTML <video> tag from a URL and type.
+      class VideoRenderer
+        # @param url [String, Html2rss::Url] media URL for the video source
+        # @param type [String] MIME type for the video source
+        def initialize(url:, type:)
+          @url = url
+          @type = type
+        end
+
+        # @return [String] HTML video snippet for article rendering
+        def to_html
+          [
+            '<video controls preload="none" referrerpolicy="no-referrer" crossorigin="anonymous" playsinline>',
+            %(<source src="#{escaped_url}" type="#{escaped_type}">),
+            '</video>'
+          ].join
+        end
+
+        private
+
+        def escaped_url
+          CGI.escapeHTML(@url.to_s)
+        end
+
+        def escaped_type
+          CGI.escapeHTML(@type.to_s)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html/rendering.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    # Namespace for HTML rendering logic, used to generate rich content such as
+    # images, audio, video, or embedded documents for feed descriptions.
+    #
+    # @see Html2rss::Html::Rendering::DescriptionBuilder
+    #
+    # @example
+    #   Html2rss::Html::Rendering::ImageRenderer.new(
+    #     url: "https://example.com/image.jpg",
+    #     title: "Example"
+    #   ).to_html
+    #
+    # @example
+    #   Html2rss::Html::Rendering::MediaRenderer.for(
+    #     enclosure: nil,
+    #     image: "https://example.com/image.jpg",
+    #     title: "Example"
+    #   )
+    module Rendering
+    end
+  end
+end

data/lib/html2rss/html.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # DOM helpers for navigating and extracting from HTML trees.
+  module Html
+  end
+end