html2rss 0.16.0 → 0.18.0

data/lib/html2rss/html_extractor/enclosure_extractor.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class HtmlExtractor
+    ##
+    # Extracts enclosures from HTML tags using various strategies.
+    class EnclosureExtractor
+      def self.call(article_tag, base_url)
+        [
+          Extractors::Image,
+          Extractors::Media,
+          Extractors::Pdf,
+          Extractors::Iframe,
+          Extractors::Archive
+        ].flat_map { |strategy| strategy.call(article_tag, base_url:) }
+      end
+    end
+
+    module Extractors
+      # Extracts image enclosures from HTML tags.
+      # Finds all image sources and returns them in a format suitable for RSS.
+      class Image
+        def self.call(article_tag, base_url:)
+          article_tag.css('img[src]:not([src^="data"])').filter_map do |img|
+            src = img['src'].to_s
+            next if src.empty?
+
+            abs_url = Url.from_relative(src, base_url)
+            {
+              url: abs_url,
+              type: RssBuilder::Enclosure.guess_content_type_from_url(abs_url, default: 'image/jpeg')
+            }
+          end
+        end
+      end
+
+      # Extracts media enclosures (video/audio) from HTML tags.
+      class Media
+        def self.call(article_tag, base_url:)
+          article_tag.css('video source[src], audio source[src], audio[src]').filter_map do |element|
+            src = element['src'].to_s
+            next if src.empty?
+
+            {
+              url: Url.from_relative(src, base_url),
+              type: element['type']
+            }
+          end
+        end
+      end
+
+      # Extracts PDF enclosures from HTML tags.
+      class Pdf
+        def self.call(article_tag, base_url:)
+          article_tag.css('a[href$=".pdf"]').filter_map do |link|
+            href = link['href'].to_s
+            next if href.empty?
+
+            abs_url = Url.from_relative(href, base_url)
+            {
+              url: abs_url,
+              type: RssBuilder::Enclosure.guess_content_type_from_url(abs_url)
+            }
+          end
+        end
+      end
+
+      # Extracts iframe enclosures from HTML tags.
+      class Iframe
+        def self.call(article_tag, base_url:)
+          article_tag.css('iframe[src]').filter_map do |iframe|
+            src = iframe['src']
+            next if src.nil? || src.empty?
+
+            abs_url = Url.from_relative(src, base_url)
+            {
+              url: abs_url,
+              type: RssBuilder::Enclosure.guess_content_type_from_url(abs_url, default: 'text/html')
+            }
+          end
+        end
+      end
+
+      # Extracts archive enclosures (zip, tar.gz, tgz) from HTML tags.
+      class Archive
+        def self.call(article_tag, base_url:)
+          article_tag.css('a[href$=".zip"], a[href$=".tar.gz"], a[href$=".tgz"]').filter_map do |link|
+            href = link['href'].to_s
+            next if href.empty?
+
+            abs_url = Url.from_relative(href, base_url)
+            {
+              url: abs_url,
+              type: 'application/zip'
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html_extractor/image_extractor.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class HtmlExtractor
+    ##
+    # Image is responsible for extracting image URLs the article_tag.
+    class ImageExtractor
+      def self.call(article_tag, base_url:)
+        img_src = from_source(article_tag) ||
+                  from_img(article_tag) ||
+                  from_style(article_tag)
+
+        Url.from_relative(img_src, base_url) if img_src
+      end
+
+      def self.from_img(article_tag)
+        article_tag.at_css('img[src]:not([src^="data"])')&.[]('src')
+      end
+
+      ##
+      # Extracts the largest image source from the srcset attribute
+      # of an img tag or a source tag inside a picture tag.
+      #
+      # @see <https://developer.mozilla.org/en-US/docs/Learn/HTML/Multimedia_and_embedding/Responsive_images>
+      # @see <https://developer.mozilla.org/en-US/docs/Web/HTML/Element/img#srcset>
+      # @see <https://developer.mozilla.org/en-US/docs/Web/HTML/Element/picture>
+      def self.from_source(article_tag) # rubocop:disable Metrics/AbcSize
+        hash = article_tag.css('img[srcset], picture > source[srcset]').flat_map do |source|
+          source['srcset'].to_s.scan(/(\S+)\s+(\d+w|\d+h)[\s,]?/).map do |url, width|
+            next if url.nil? || url.start_with?('data:')
+
+            width_value = width.to_i.zero? ? 0 : width.scan(/\d+/).first.to_i
+
+            [width_value, url.strip]
+          end
+        end.compact.to_h
+
+        hash[hash.keys.max]
+      end
+
+      def self.from_style(article_tag)
+        article_tag.css('[style*="url"]')
+                   .filter_map { |tag| tag['style'][/url\(['"]?(.*?)['"]?\)/, 1] }
+                   .reject { |src| src.start_with?('data:') }
+                   .max_by(&:size)
+      end
+    end
+  end
+end

data/lib/html2rss/html_extractor.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # HtmlExtractor is responsible for extracting details (headline, url, images, etc.)
+  # from an article_tag.
+  class HtmlExtractor
+    INVISIBLE_CONTENT_TAGS = %w[svg script noscript style template].to_set.freeze
+    HEADING_TAGS = %w[h1 h2 h3 h4 h5 h6].freeze
+    NON_HEADLINE_SELECTOR = (HEADING_TAGS.map { |tag| ":not(#{tag})" } + INVISIBLE_CONTENT_TAGS.to_a).freeze
+
+    MAIN_ANCHOR_SELECTOR = begin
+      buf = +'a[href]:not([href=""])'
+      %w[# javascript: mailto: tel: file:// sms: data:].each do |prefix|
+        buf << %[:not([href^="#{prefix}"])]
+      end
+      buf.freeze
+    end
+
+    class << self
+      ##
+      # Extracts visible text from a given node and its children.
+      #
+      # @param tag [Nokogiri::XML::Node] the node from which to extract visible text
+      # @param separator [String] separator used to join text fragments (default is a space)
+      # @return [String, nil] the concatenated visible text, or nil if none is found
+      def extract_visible_text(tag, separator: ' ')
+        parts = tag.children.filter_map do |child|
+          next unless visible_child?(child)
+
+          raw_text = child.children.empty? ? child.text : extract_visible_text(child)
+          text = raw_text&.strip
+          text unless text.to_s.empty?
+        end
+
+        parts.join(separator).squeeze(' ').strip unless parts.empty?
+      end
+
+      private
+
+      def visible_child?(node)
+        !INVISIBLE_CONTENT_TAGS.include?(node.name) &&
+          !(node.name == 'a' && node['href']&.start_with?('#'))
+      end
+    end
+
+    ##
+    # @param article_tag [Nokogiri::XML::Node] article-like container to extract from
+    # @param base_url [String, Html2rss::Url] base url used to resolve relative links
+    # @param selected_anchor [Nokogiri::XML::Node, nil] explicit primary anchor for the container
+    def initialize(article_tag, base_url:, selected_anchor:)
+      raise ArgumentError, 'article_tag is required' unless article_tag
+
+      @article_tag = article_tag
+      @base_url = base_url
+      @selected_anchor = selected_anchor
+    end
+
+    def call
+      {
+        title: extract_title,
+        url: extract_url,
+        image: extract_image,
+        description: extract_description,
+        id: generate_id,
+        published_at: extract_published_at,
+        enclosures: extract_enclosures,
+        categories: extract_categories
+      }
+    end
+
+    private
+
+    attr_reader :article_tag, :base_url, :selected_anchor
+
+    class << self
+      ##
+      # @param article_tag [Nokogiri::XML::Node] article-like container to search within
+      # @return [Nokogiri::XML::Node, nil] first eligible descendant anchor
+      def main_anchor_for(article_tag)
+        return article_tag if article_tag.name == 'a' && article_tag.matches?(MAIN_ANCHOR_SELECTOR)
+
+        article_tag.at_css(MAIN_ANCHOR_SELECTOR)
+      end
+    end
+
+    def extract_url
+      @extract_url ||= begin
+        href = selected_anchor&.[]('href').to_s
+
+        Url.from_relative(href.split('#').first.strip, base_url) unless href.empty?
+      end
+    end
+
+    def extract_title
+      title_source = heading || selected_anchor
+      self.class.extract_visible_text(title_source) if title_source
+    end
+
+    def heading
+      @heading ||= begin
+        heading_tags = article_tag.css(HEADING_TAGS.join(',')).group_by(&:name)
+        smallest_heading = heading_tags.keys.min
+        if smallest_heading
+          heading_tags[smallest_heading]&.max_by do |tag|
+            self.class.extract_visible_text(tag)&.size.to_i
+          end
+        end
+      end
+    end
+
+    def extract_description
+      text = self.class.extract_visible_text(article_tag.css(NON_HEADLINE_SELECTOR), separator: '<br>')
+      return text if text && !text.empty?
+
+      description = self.class.extract_visible_text(article_tag)
+      return nil if description.nil? || description.strip.empty?
+
+      description.strip
+    end
+
+    def generate_id
+      [
+        article_tag['id'],
+        article_tag.at_css('[id]')&.attr('id'),
+        extract_url&.path,
+        extract_url&.query
+      ].compact.reject(&:empty?).first
+    end
+
+    def extract_image = ImageExtractor.call(article_tag, base_url:)
+    def extract_published_at = DateExtractor.call(article_tag)
+    def extract_enclosures = EnclosureExtractor.call(article_tag, base_url)
+    def extract_categories = CategoryExtractor.call(article_tag)
+  end
+end

data/lib/html2rss/html_navigator.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # HtmlNavigator provides methods to navigate through HTML nodes.
+  class HtmlNavigator
+    class << self
+      ##
+      # Returns the first parent that satisfies the condition.
+      # If the condition is met, it returns the node itself.
+      #
+      # @param node [Nokogiri::XML::Node] The node to start the search from.
+      # @param condition [Proc] The condition to be met.
+      # @return [Nokogiri::XML::Node, nil] The first parent that satisfies the condition.
+      def parent_until_condition(node, condition)
+        while node && !node.document? && node.name != 'html'
+          return node if condition.call(node)
+
+          node = node.parent
+        end
+      end
+
+      ##
+      # Think of it as `css_upwards` method.
+      # It searches for the closest parent that matches the given selector.
+      def find_closest_selector_upwards(current_tag, selector)
+        while current_tag
+          found = current_tag.at_css(selector)
+          return found if found
+
+          return nil unless current_tag.respond_to?(:parent)
+
+          current_tag = current_tag.parent
+        end
+      end
+
+      ##
+      # Searches for the closest parent that matches the given tag name.
+      def find_tag_in_ancestors(current_tag, tag_name)
+        return current_tag if current_tag.name == tag_name
+
+        current_tag.ancestors(tag_name).first
+      end
+    end
+  end
+end

data/lib/html2rss/json_feed_builder/item.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class JsonFeedBuilder
+    ##
+    # Maps an {Html2rss::RssBuilder::Article} to a JSONFeed 1.1 item hash.
+    class Item
+      ##
+      # @param article [Html2rss::RssBuilder::Article]
+      def initialize(article)
+        @article = article
+      end
+
+      ##
+      # @return [Hash, nil] the JSONFeed-compliant item hash
+      def to_h
+        content = content_fields
+        return if content.empty?
+
+        item_payload.merge(content).compact
+      end
+
+      private
+
+      attr_reader :article
+
+      ##
+      # @return [Hash]
+      def item_payload
+        {
+          id: article.guid,
+          url: article.url&.to_s,
+          title: article.title,
+          image: article.image&.to_s,
+          date_published: article.published_at&.iso8601,
+          authors: author_array,
+          tags:,
+          attachments:
+        }
+      end
+
+      ##
+      # @return [Array<Hash>, nil]
+      def author_array
+        return unless (name = article.author)
+
+        [{ name: }]
+      end
+
+      ##
+      # JSON Feed items must include content_html or content_text.
+      # @return [Hash]
+      def content_fields
+        description = article.description
+        return { content_html: description } if description
+
+        title = article.title
+        return { content_text: title } if title
+
+        {}
+      end
+
+      ##
+      # @return [Array<String>, nil]
+      def tags
+        cats = article.categories
+        cats.empty? ? nil : cats
+      end
+
+      ##
+      # Maps enclosures to JSONFeed attachment objects.
+      # @return [Array<Hash>, nil]
+      def attachments
+        enclosures = article.enclosures
+        return nil if enclosures.empty?
+
+        enclosures.map { |enc| attachment_hash(enc) }
+      end
+
+      ##
+      # @param enclosure [Html2rss::RssBuilder::Article::Enclosure]
+      # @return [Hash]
+      def attachment_hash(enclosure)
+        size = enclosure.bits_length
+
+        {
+          url: enclosure.url.to_s,
+          mime_type: enclosure.type,
+          size_in_bytes: size&.positive? ? size : nil
+        }.compact
+      end
+    end
+  end
+end

data/lib/html2rss/json_feed_builder.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Builds a JSONFeed 1.1 hash from channel metadata and articles.
+  #
+  # @see https://www.jsonfeed.org/version/1.1/
+  class JsonFeedBuilder
+    VERSION_URL = 'https://jsonfeed.org/version/1.1'
+
+    ##
+    # @param channel [Html2rss::RssBuilder::Channel]
+    # @param articles [Array<Html2rss::RssBuilder::Article>]
+    def initialize(channel:, articles:)
+      @channel = channel
+      @articles = articles
+    end
+
+    ##
+    # Builds and returns the JSONFeed hash.
+    #
+    # @return [Hash] the JSONFeed-compliant hash
+    def call
+      base_payload.merge(authors: author_array, items: item_hashes).compact
+    end
+
+    private
+
+    attr_reader :channel, :articles
+
+    ##
+    # @return [Hash]
+    def base_payload
+      {
+        version: VERSION_URL,
+        title: channel.title,
+        home_page_url: channel.url.to_s,
+        description: channel.description,
+        language: channel.language,
+        icon: channel.image&.to_s
+      }
+    end
+
+    ##
+    # @return [Array<Hash>]
+    def item_hashes
+      articles.filter_map { |article| Item.new(article).to_h }
+    end
+
+    ##
+    # @return [Array<Hash>, nil]
+    def author_array
+      return unless (name = channel.author)
+
+      [{ name: }]
+    end
+  end
+end

data/lib/html2rss/rendering/audio_renderer.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'cgi'
+
+module Html2rss
+  module Rendering
+    # Renders an HTML <audio> tag from a URL and type.
+    class AudioRenderer
+      def initialize(url:, type:)
+        @url = url
+        @type = type
+      end
+
+      def to_html
+        %(<audio controls preload="none" referrerpolicy="no-referrer" crossorigin="anonymous">
+            <source src="#{escaped_url}" type="#{escaped_type}">
+          </audio>)
+      end
+
+      private
+
+      def escaped_url
+        CGI.escapeHTML(@url.to_s)
+      end
+
+      def escaped_type
+        CGI.escapeHTML(@type.to_s)
+      end
+    end
+  end
+end

data/lib/html2rss/rendering/description_builder.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Rendering
+    # Builds a sanitized article description from the base text, title, and optional media.
+    #
+    # Combines media elements (images, audio, video, PDFs) with sanitized text content
+    # to create rich RSS descriptions that reveal more scraped information.
+    #
+    # @example Basic usage
+    #   builder = DescriptionBuilder.new(
+    #     base: "Article content",
+    #     title: "Article Title",
+    #     url: "https://example.com",
+    #     enclosures: [enclosure_object],
+    #     image: "https://example.com/image.jpg"
+    #   )
+    #   description = builder.call
+    #
+    class DescriptionBuilder
+      # Removes the specified pattern from the beginning of the text
+      # within a given range if the pattern occurs before the range's end.
+      #
+      # @param text [String]
+      # @param pattern [String]
+      # @param end_of_range [Integer] Optional, defaults to half the text length
+      # @return [String]
+      def self.remove_pattern_from_start(text, pattern, end_of_range: (text.size * 0.5).to_i)
+        return text unless text.is_a?(String) && pattern.is_a?(String)
+
+        index = text.index(pattern)
+        return text if index.nil? || index >= end_of_range
+
+        text.gsub(/^(.{0,#{end_of_range}})#{Regexp.escape(pattern)}/, '\1')
+      end
+
+      # @param base [String] The base text content for the description
+      # @param title [String] The article title (used for alt text and title removal)
+      # @param url [String, Html2rss::Url] The article URL (used for sanitization)
+      # @param enclosures [Array<Html2rss::RssBuilder::Enclosure>, nil] Media enclosures
+      # @param image [String, Html2rss::Url, nil] Fallback image URL
+      def initialize(base:, title:, url:, enclosures:, image:)
+        @base = base.to_s
+        @title = title
+        @url = url
+        @enclosures = Array(enclosures)
+        @image = image
+      end
+
+      # Generates the complete description with media and sanitized text.
+      #
+      # @return [String, nil] The complete description or nil if empty
+      def call
+        fragments = []
+        fragments.concat(Array(rendered_media))
+        fragments << processed_base_description
+
+        result = fragments.compact.join("\n\n").strip
+        result.empty? ? nil : result
+      end
+
+      private
+
+      def rendered_media
+        rendered = render_enclosures
+        return rendered if rendered.any?
+        return render_fallback_image if @image
+
+        []
+      end
+
+      def render_enclosures
+        @enclosures.filter_map do |enclosure|
+          MediaRenderer.for(enclosure:, image: @image, title: @title)&.to_html
+        end
+      end
+
+      def render_fallback_image
+        [MediaRenderer.for(enclosure: nil, image: @image, title: @title)&.to_html]
+      end
+
+      def processed_base_description
+        text = self.class.remove_pattern_from_start(@base, @title)
+        Html2rss::Selectors::PostProcessors::SanitizeHtml.get(text, @url)
+      end
+    end
+  end
+end

data/lib/html2rss/rendering/image_renderer.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'cgi'
+
+module Html2rss
+  module Rendering
+    # Renders an HTML <img> tag from a URL and title.
+    class ImageRenderer
+      def initialize(url:, title:)
+        @url = url
+        @title = title
+      end
+
+      def to_html
+        %(<img src="#{@url}"
+              alt="#{escaped_title}"
+              title="#{escaped_title}"
+              loading="lazy"
+              referrerpolicy="no-referrer"
+              decoding="async"
+              crossorigin="anonymous">).delete("\n").gsub(/\s+/, ' ')
+      end
+
+      private
+
+      def escaped_title
+        CGI.escapeHTML(@title.to_s)
+      end
+    end
+  end
+end

data/lib/html2rss/rendering/media_renderer.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Rendering
+    # Factory: picks the appropriate renderer for a given enclosure or fallback image.
+    class MediaRenderer
+      # @param enclosure [Html2rss::RssBuilder::Enclosure, nil]
+      # @param image [String, Html2rss::Url, nil] Fallback image URL
+      # @param title [String]
+      # @return [ImageRenderer, VideoRenderer, AudioRenderer, PdfRenderer, nil]
+      def self.for(enclosure:, image:, title:)
+        return ImageRenderer.new(url: image, title:) if enclosure.nil? && image
+        return nil unless enclosure
+
+        create_renderer_for_type(enclosure.type, url: enclosure.url, title:)
+      end
+
+      # @private
+      def self.create_renderer_for_type(type, url:, title:)
+        case type
+        when %r{^image/}
+          ImageRenderer.new(url:, title:)
+        when %r{^video/}
+          VideoRenderer.new(url:, type:)
+        when %r{^audio/}
+          AudioRenderer.new(url:, type:)
+        when 'application/pdf'
+          PdfRenderer.new(url:)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/rendering/pdf_renderer.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'cgi'
+
+module Html2rss
+  module Rendering
+    # Renders an HTML <iframe> for PDF documents.
+    class PdfRenderer
+      def initialize(url:)
+        @url = url
+      end
+
+      def to_html
+        %(<iframe src="#{escaped_url}" width="100%" height="75vh"
+                  sandbox=""
+                  referrerpolicy="no-referrer"
+                  loading="lazy">
+           </iframe>)
+      end
+
+      private
+
+      def escaped_url
+        CGI.escapeHTML(@url.to_s)
+      end
+    end
+  end
+end