html2rss 0.22.2 → 0.24.0

data/lib/html2rss/html/article_extractor/enclosure_extractor.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    class ArticleExtractor
+      ##
+      # Extracts enclosures from HTML tags using various strategies.
+      class EnclosureExtractor
+        # CSS union query covering images, media, PDFs, iframes, and archives.
+        SELECTOR = [
+          'img[src]:not([src^="data"])',
+          'video source[src]',
+          'audio source[src]',
+          'audio[src]',
+          'a[href$=".pdf"]',
+          'iframe[src]',
+          'a[href$=".zip"]',
+          'a[href$=".tar.gz"]',
+          'a[href$=".tgz"]'
+        ].join(',').freeze
+
+        # @param article_tag [Nokogiri::XML::Element] article container node
+        # @param base_url [String, Html2rss::Url] base URL for relative enclosure links
+        # @return [Array<Hash{Symbol => Object}>] normalized enclosure hashes
+        def self.call(article_tag, base_url)
+          return [] unless article_tag
+
+          article_tag.css(SELECTOR).filter_map do |element|
+            extract_from_element(element, base_url)
+          end
+        end
+
+        def self.extract_from_element(element, base_url)
+          case element.name
+          when 'img'
+            extract_image(element, base_url)
+          when 'video', 'audio', 'source'
+            extract_media(element, base_url)
+          when 'iframe'
+            extract_iframe(element, base_url)
+          when 'a'
+            extract_a(element, base_url)
+          end
+        end
+
+        def self.extract_image(img, base_url)
+          src = img['src'].to_s
+          return if src.empty?
+
+          abs_url = Url.from_relative(src, base_url)
+          {
+            url: abs_url,
+            type: Article::Enclosure.guess_content_type_from_url(abs_url, default: 'image/jpeg')
+          }
+        end
+
+        def self.extract_media(element, base_url)
+          src = element['src'].to_s
+          return if src.empty?
+
+          {
+            url: Url.from_relative(src, base_url),
+            type: element['type']
+          }
+        end
+
+        def self.extract_iframe(iframe, base_url)
+          src = iframe['src'].to_s
+          return if src.empty?
+
+          abs_url = Url.from_relative(src, base_url)
+          {
+            url: abs_url,
+            type: Article::Enclosure.guess_content_type_from_url(abs_url, default: 'text/html')
+          }
+        end
+
+        def self.extract_a(link, base_url)
+          href = link['href'].to_s
+          return if href.empty?
+
+          abs_url = Url.from_relative(href, base_url)
+
+          if href.end_with?('.pdf')
+            { url: abs_url, type: Article::Enclosure.guess_content_type_from_url(abs_url) }
+          else
+            { url: abs_url, type: 'application/zip' }
+          end
+        end
+
+        private_class_method :extract_from_element, :extract_image, :extract_media, :extract_iframe, :extract_a
+      end
+    end
+  end
+end

data/lib/html2rss/html/article_extractor/heading_extractor.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    class ArticleExtractor
+      ##
+      # HeadingExtractor identifies and returns the best heading element within a container.
+      class HeadingExtractor
+        # Heading tags used to prioritize title extraction.
+        HEADING_TAGS = Navigator::HEADING_TAGS
+
+        class << self
+          ##
+          # @param article_tag [Nokogiri::XML::Element] container node
+          # @param fallback_anchorless [Boolean] whether to use fallback search
+          # @param selected_anchor [Nokogiri::XML::Node, nil] anchor element
+          # @return [Nokogiri::XML::Node, nil] the heading node, if found
+          def call(article_tag, fallback_anchorless:, selected_anchor:)
+            tags = article_tag.css(HEADING_TAGS.join(','))
+            if tags.any?
+              select_best_heading(tags)
+            elsif fallback_anchorless && selected_anchor.nil?
+              fallback_heading(article_tag)
+            end
+          end
+
+          private
+
+          def select_best_heading(tags)
+            min_tag_name = tags.map(&:name).min
+            best_tag = nil
+            max_size = -1
+
+            tags.each do |tag|
+              next if tag.name != min_tag_name
+
+              size = Navigator::TextExtractor.call(tag)&.size.to_i
+              (best_tag = tag) && (max_size = size) if size > max_size
+            end
+
+            best_tag
+          end
+
+          def fallback_heading(article_tag)
+            fallback_tags = article_tag.css(
+              'strong, b, [class*="title"], [class*="font-bold"], [class*="font-semibold"]'
+            )
+            fallback_tags.find { |t| !Navigator::TextExtractor.call(t).to_s.strip.empty? }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html/article_extractor/id_generator.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require 'zlib'
+
+module Html2rss
+  module Html
+    class ArticleExtractor
+      ##
+      # IdGenerator determines the unique ID for an article container node.
+      class IdGenerator
+        class << self
+          ##
+          # @param article_tag [Nokogiri::XML::Element] container node
+          # @param heading [Nokogiri::XML::Node, nil] heading node
+          # @param url [Html2rss::Url, nil] absolute article URL
+          # @param selected_anchor [Nokogiri::XML::Node, nil] anchor element
+          # @param fallback_anchorless [Boolean] whether to use fallback hashing
+          # @return [String, nil] the generated ID, if any
+          def call(article_tag, heading:, url:, selected_anchor:, fallback_anchorless:)
+            id_from_dom = parse_id_from_dom(article_tag, url, selected_anchor)
+            return id_from_dom if id_from_dom
+
+            heading_text = resolve_heading_text(article_tag, heading, fallback_anchorless)
+            if heading_text && !heading_text.strip.empty?
+              generate_slug(heading_text)
+            elsif fallback_anchorless
+              generate_content_hash(article_tag)
+            end
+          end
+
+          private
+
+          def parse_id_from_dom(article_tag, url, selected_anchor)
+            candidates = [article_tag['id'], article_tag.at_css('[id]')&.attr('id')]
+            candidates += [url&.path, url&.query] if selected_anchor
+            candidates.compact.reject(&:empty?).first
+          end
+
+          def resolve_heading_text(article_tag, heading, fallback_anchorless)
+            text = heading ? Navigator::TextExtractor.call(heading) : nil
+            if text.nil? || text.strip.empty?
+              fallback_text_node_content(article_tag, fallback_anchorless)
+            else
+              text
+            end
+          end
+
+          def fallback_text_node_content(article_tag, fallback_anchorless)
+            return unless fallback_anchorless
+
+            article_tag.xpath('.//text()').find { |t| !t.text.strip.empty? }&.text&.strip
+          end
+
+          def generate_slug(text)
+            slug = text.downcase.gsub(/[^a-z0-9]+/, '-')
+            slug = slug[1..] if slug.start_with?('-')
+            slug = slug[0..-2] if slug.end_with?('-')
+            slug unless slug.empty?
+          end
+
+          def generate_content_hash(article_tag)
+            text = Navigator::TextExtractor.call(article_tag).to_s.strip
+            Zlib.crc32(text).to_s(36) unless text.empty?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html/article_extractor/image_extractor.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    class ArticleExtractor
+      ##
+      # Image is responsible for extracting image URLs the article_tag.
+      class ImageExtractor
+        # @param article_tag [Nokogiri::XML::Element] article container node
+        # @param base_url [String, Html2rss::Url] base URL for relative image URLs
+        # @return [Html2rss::Url, nil] best candidate image URL
+        def self.call(article_tag, base_url:)
+          img_src = from_source(article_tag) ||
+                    from_img(article_tag) ||
+                    from_style(article_tag)
+
+          Url.from_relative(img_src, base_url) if img_src
+        end
+
+        # @param article_tag [Nokogiri::XML::Element] article container node
+        # @return [String, nil] src attribute from first matching image tag
+        def self.from_img(article_tag)
+          article_tag.at_css('img[src]:not([src^="data"])')&.[]('src')
+        end
+
+        ##
+        # Extracts the largest image source from the srcset attribute
+        # of an img tag or a source tag inside a picture tag.
+        #
+        # @param article_tag [Nokogiri::XML::Element] article container node
+        # @return [String, nil] largest srcset URL candidate
+        # @see <https://developer.mozilla.org/en-US/docs/Learn/HTML/Multimedia_and_embedding/Responsive_images>
+        # @see <https://developer.mozilla.org/en-US/docs/Web/HTML/Element/img#srcset>
+        # @see <https://developer.mozilla.org/en-US/docs/Web/HTML/Element/picture>
+        def self.from_source(article_tag) # rubocop:disable Metrics/AbcSize
+          hash = article_tag.css('img[srcset], picture > source[srcset]').flat_map do |source|
+            source['srcset'].to_s.scan(/(\S+)\s+(\d+w|\d+h)[\s,]?/).map do |url, width|
+              next if url.nil? || url.start_with?('data:')
+
+              width_value = width.to_i.zero? ? 0 : width.scan(/\d+/).first.to_i
+
+              [width_value, url.strip]
+            end
+          end.compact.to_h
+
+          hash[hash.keys.max]
+        end
+
+        # @param article_tag [Nokogiri::XML::Element] article container node
+        # @return [String, nil] best style-based background image URL
+        def self.from_style(article_tag)
+          article_tag.css('[style*="url"]')
+                     .filter_map { |tag| tag['style'][/url\(['"]?(.*?)['"]?\)/, 1] }
+                     .reject { |src| src.start_with?('data:') }
+                     .max_by(&:size)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html/article_extractor.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    ##
+    # ArticleExtractor is responsible for extracting details (headline, url, images, etc.)
+    # from an article_tag DOM node. DOM chrome helpers live on {Navigator}.
+    class ArticleExtractor
+      class << self
+        ##
+        # Extracts article attributes from a DOM element.
+        #
+        # @param article_tag [Nokogiri::XML::Node] article-like container to extract from
+        # @param base_url [String, Html2rss::Url] base url used to resolve relative links
+        # @param selected_anchor [Nokogiri::XML::Node, nil] explicit primary anchor for the container
+        # @param fallback_anchorless [Boolean] whether to fall back to anchorless extraction
+        # @return [Hash{Symbol => Object}] extracted article attributes
+        def call(article_tag, base_url:, selected_anchor: nil, fallback_anchorless: false)
+          new(article_tag, base_url:, selected_anchor:, fallback_anchorless:).call
+        end
+      end
+
+      ##
+      # @param article_tag [Nokogiri::XML::Node] article-like container to extract from
+      # @param base_url [String, Html2rss::Url] base url used to resolve relative links
+      # @param selected_anchor [Nokogiri::XML::Node, nil] explicit primary anchor for the container
+      # @param fallback_anchorless [Boolean] whether to fall back to anchorless extraction
+      def initialize(article_tag, base_url:, selected_anchor: nil, fallback_anchorless: false)
+        raise ArgumentError, 'article_tag is required' unless article_tag
+
+        @article_tag = article_tag
+        @base_url = base_url
+        @selected_anchor = selected_anchor
+        @fallback_anchorless = fallback_anchorless
+      end
+
+      # @return [Hash{Symbol => Object}] extracted article attributes
+      def call
+        {
+          title: extract_title,
+          url: extract_url,
+          image: extract_image,
+          description: extract_description,
+          id: generate_id,
+          published_at: extract_published_at,
+          enclosures: extract_enclosures,
+          categories: extract_categories
+        }
+      end
+
+      private
+
+      attr_reader :article_tag, :base_url, :selected_anchor
+
+      def extract_url
+        @extract_url ||= begin
+          href = selected_anchor&.[]('href').to_s
+
+          if href.empty?
+            anchorless_url_fallback
+          else
+            Url.from_relative(href.split('#').first.strip, base_url)
+          end
+        end
+      end
+
+      def anchorless_url_fallback
+        return unless @fallback_anchorless
+
+        id = generate_id
+        Url.from_relative("##{id}", base_url) if id
+      end
+
+      # rubocop:disable Metrics/CyclomaticComplexity
+      def extract_title
+        source = heading || selected_anchor
+        title_text = source ? Navigator.extract_visible_text(source) : fallback_anchorless_title
+        return unless title_text
+
+        kicker = kicker_node ? Navigator.extract_visible_text(kicker_node).to_s.strip : nil
+        kicker && !kicker.empty? && !title_text.include?(kicker) ? "#{kicker}: #{title_text}" : title_text
+      end
+      # rubocop:enable Metrics/CyclomaticComplexity
+
+      def fallback_anchorless_title
+        return unless @fallback_anchorless && selected_anchor.nil?
+
+        text_node = article_tag.xpath('.//text()').find { |t| !t.text.strip.empty? }
+        text_node&.text&.strip
+      end
+
+      def heading
+        @heading ||= HeadingExtractor.call(
+          article_tag,
+          fallback_anchorless: @fallback_anchorless,
+          selected_anchor:
+        )
+      end
+
+      def kicker_node
+        @kicker_node ||= begin
+          selector = '[data-tb-kicker], [class*="kicker"], [class*="eyebrow"], ' \
+                     '[class*="pre-title"], [class*="pretitle"], [class*="overline"]'
+          node = article_tag.at_css(selector)
+          node && heading && (node == heading || Navigator.descendant_of?(node, heading)) ? nil : node
+        end
+      end
+
+      def extract_description
+        exclude = [heading, selected_anchor, kicker_node].compact
+        description = Navigator.extract_visible_text(article_tag, exclude_nodes: exclude)
+        return if description.nil?
+
+        desc = description.strip
+        desc.empty? ? nil : desc
+      end
+
+      def generate_id
+        @generate_id ||= IdGenerator.call(
+          article_tag,
+          heading:,
+          url: (selected_anchor ? extract_url : nil),
+          selected_anchor:,
+          fallback_anchorless: @fallback_anchorless
+        )
+      end
+
+      def extract_image = ImageExtractor.call(article_tag, base_url:)
+      def extract_published_at = DateExtractor.call(article_tag)
+      def extract_enclosures = EnclosureExtractor.call(article_tag, base_url)
+      def extract_categories = CategoryExtractor.call(article_tag)
+    end
+  end
+end

data/lib/html2rss/html/navigator/text_extractor.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    class Navigator
+      ##
+      # TextExtractor extracts visible text from DOM elements, preserving lists
+      # and block spacing while sanitizing white spaces.
+      class TextExtractor
+        # HTML block elements that trigger line breaks or special formatting.
+        BLOCK_TAGS = %w[p div li ul ol h1 h2 h3 h4 h5 h6 tr br].to_set.freeze
+        # Tags ignored when extracting visible text content.
+        INVISIBLE_CONTENT_TAGS = %w[svg script noscript style template].to_set.freeze
+
+        class << self
+          ##
+          # @param tag [Nokogiri::XML::Node] the node from which to extract visible text
+          # @param separator [String] separator used to join text fragments (default is a space)
+          # @param exclude_nodes [Array<Nokogiri::XML::Node>, nil] nodes to exclude from extraction
+          # @return [String, nil] the concatenated visible text, or nil if none is found
+          def call(tag, separator: ' ', exclude_nodes: nil)
+            return tag.text.gsub(/\s+/, ' ').strip if tag.respond_to?(:text?) && tag.text?
+
+            parts = iterate_children(tag, separator, exclude_nodes)
+            return if parts.empty?
+
+            parts.join.squeeze(' ').strip
+          end
+
+          private
+
+          def iterate_children(tag, separator, exclude_nodes)
+            last = false
+            tag.children.each_with_object([]) do |c, p|
+              next if exclude_nodes&.include?(c) || !visible_child?(c)
+
+              text, block = process_child_node(c, separator, exclude_nodes)
+              next if text.empty?
+
+              append_separator!(p, separator, block, last)
+              (p << text) && (last = block)
+            end
+          end
+
+          def process_child_node(child, separator, exclude_nodes)
+            child_text = get_child_text(child, separator, exclude_nodes)
+            return ['', false] if child_text.empty?
+
+            child_text = "- #{child_text}" if child.name == 'li'
+            [child_text, BLOCK_TAGS.include?(child.name)]
+          end
+
+          def get_child_text(child, separator, exclude_nodes)
+            if child.children.empty?
+              child.text.to_s.gsub(/\s+/, ' ').strip
+            else
+              call(child, separator:, exclude_nodes:).to_s.strip
+            end
+          end
+
+          def append_separator!(parts, separator, is_block, last_was_block)
+            return if parts.empty?
+
+            parts << if is_block || last_was_block
+                       (separator == ' ' ? "\n" : separator)
+                     else
+                       ' '
+                     end
+          end
+
+          def visible_child?(node)
+            !INVISIBLE_CONTENT_TAGS.include?(node.name) &&
+              !(node.name == 'a' && node['href']&.start_with?('#'))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html/navigator.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    ##
+    # Navigator owns DOM chrome recognition and node traversal helpers.
+    class Navigator
+      # Heading tags used to prioritize title extraction and container assessment.
+      HEADING_TAGS = %w[h1 h2 h3 h4 h5 h6].freeze
+
+      # Element tags that indicate ignored DOM chrome when found in a container path.
+      IGNORED_CONTAINER_TAGS = %w[nav footer header svg script style].to_set.freeze
+
+      # Layout roots and chrome tags excluded from class-clustering candidate nodes.
+      CLUSTER_EXCLUDED_TAGS = Set['html', 'body', 'nav', 'footer', 'header', 'svg', 'script', 'style'].freeze
+
+      # Ancestor tags that usually indicate navigation/utility regions inside a content container.
+      UTILITY_LANDMARK_TAGS = %w[nav aside footer menu].to_set.freeze
+
+      # Anchor selector used to identify the canonical article link element.
+      MAIN_ANCHOR_SELECTOR = begin
+        buf = +'a[href]:not([href=""])'
+        %w[# javascript: mailto: tel: file:// sms: data:].each do |prefix|
+          buf << %[:not([href^="#{prefix}"])]
+        end
+        buf.freeze
+      end
+
+      class << self
+        ##
+        # Extracts visible text from a given node and its children.
+        #
+        # @param tag [Nokogiri::XML::Node] the node from which to extract visible text
+        # @param separator [String] separator used to join text fragments (default is a space)
+        # @param exclude_nodes [Array<Nokogiri::XML::Node>, nil] nodes to exclude from extraction
+        # @return [String, nil] the concatenated visible text, or nil if none is found
+        def extract_visible_text(tag, separator: ' ', exclude_nodes: nil)
+          TextExtractor.call(tag, separator:, exclude_nodes:)
+        end
+
+        ##
+        # @param article_tag [Nokogiri::XML::Node] article-like container to search within
+        # @return [Nokogiri::XML::Node, nil] first eligible descendant anchor
+        def main_anchor_for(article_tag)
+          return article_tag if article_tag.name == 'a' && article_tag.matches?(MAIN_ANCHOR_SELECTOR)
+
+          article_tag.at_css(MAIN_ANCHOR_SELECTOR)
+        end
+
+        ##
+        # @param node [Nokogiri::XML::Node]
+        # @param cache [Hash, nil] identity cache used to store results (must use compare_by_identity)
+        # @return [Boolean] true when the node belongs to ignored DOM chrome
+        # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+        def ignored_container_path?(node, cache = nil)
+          return cache[node] if cache&.key?(node)
+
+          curr = node
+          visited = []
+          is_ignored = false
+
+          while curr.respond_to?(:parent) && curr
+            if cache&.key?(curr)
+              is_ignored = cache[curr]
+              break
+            end
+
+            if IGNORED_CONTAINER_TAGS.include?(curr.name)
+              is_ignored = true
+              break
+            end
+
+            visited << curr
+            curr = curr.parent
+          end
+          visited.each { |n| cache[n] = is_ignored } if cache
+
+          is_ignored
+        end
+        # rubocop:enable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+
+        ##
+        # Returns the first parent that satisfies the condition.
+        # If the condition is met, it returns the node itself.
+        #
+        # @param node [Nokogiri::XML::Node] The node to start the search from.
+        # @param condition [Proc] The condition to be met.
+        # @return [Nokogiri::XML::Node, nil] The first parent that satisfies the condition.
+        def parent_until_condition(node, condition)
+          while node && !node.document? && node.name != 'html'
+            return node if condition.call(node)
+
+            node = node.parent
+          end
+        end
+
+        ##
+        # Think of it as `css_upwards` method.
+        # It searches for the closest parent that matches the given selector.
+        #
+        # @param current_tag [Nokogiri::XML::Node, nil] starting node
+        # @param selector [String] CSS selector to search upwards for
+        # @return [Nokogiri::XML::Node, nil] first matching node in upward traversal
+        def find_closest_selector_upwards(current_tag, selector)
+          while current_tag
+            found = current_tag.at_css(selector)
+            return found if found
+
+            return nil unless current_tag.respond_to?(:parent)
+
+            current_tag = current_tag.parent
+          end
+        end
+
+        ##
+        # Searches for the closest parent that matches the given tag name.
+        #
+        # @param current_tag [Nokogiri::XML::Node] starting node
+        # @param tag_name [String] tag name to find in ancestors
+        # @return [Nokogiri::XML::Node, nil] matching ancestor node
+        def find_tag_in_ancestors(current_tag, tag_name)
+          return current_tag if current_tag.name == tag_name
+
+          current_tag.ancestors(tag_name).first
+        end
+
+        ##
+        # Returns true if child_node is a descendant of parent_node.
+        # Walks up using parent pointers to avoid NodeSet allocations.
+        #
+        # @param child_node [Nokogiri::XML::Node] potential descendant
+        # @param parent_node [Nokogiri::XML::Node] potential ancestor
+        # @return [Boolean] true when child_node is a descendant of parent_node
+        def descendant_of?(child_node, parent_node)
+          curr = child_node.respond_to?(:parent) ? child_node.parent : nil
+          while curr
+            return true if curr == parent_node
+
+            curr = curr.respond_to?(:parent) ? curr.parent : nil
+          end
+          false
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html/rendering/audio_renderer.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require 'cgi'
+
+module Html2rss
+  module Html
+    module Rendering
+      # Renders an HTML <audio> tag from a URL and type.
+      class AudioRenderer
+        # @param url [String, Html2rss::Url] media URL for the audio source
+        # @param type [String] MIME type for the audio source
+        def initialize(url:, type:)
+          @url = url
+          @type = type
+        end
+
+        # @return [String] HTML audio snippet for article rendering
+        def to_html
+          [
+            '<audio controls preload="none" referrerpolicy="no-referrer" crossorigin="anonymous">',
+            %(<source src="#{escaped_url}" type="#{escaped_type}">),
+            '</audio>'
+          ].join
+        end
+
+        private
+
+        def escaped_url
+          CGI.escapeHTML(@url.to_s)
+        end
+
+        def escaped_type
+          CGI.escapeHTML(@type.to_s)
+        end
+      end
+    end
+  end
+end