html2rss 0.17.0 → 0.19.0

data/lib/html2rss/rss_builder/article.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+require 'zlib'
+require 'sanitize'
+require 'nokogiri'
+
+module Html2rss
+  class RssBuilder
+    ##
+    # Article is a simple data object representing an article extracted from a page.
+    # It is enumerable and responds to all keys specified in PROVIDED_KEYS.
+    class Article
+      include Enumerable
+      include Comparable
+
+      # Allowed article attributes accepted by the value object constructor.
+      PROVIDED_KEYS = %i[id title description url image author guid published_at enclosures categories scraper].freeze
+      # Separator used to build deterministic deduplication fingerprints.
+      DEDUP_FINGERPRINT_SEPARATOR = '#!/'
+
+      # @param options [Hash{Symbol => String}]
+      # @option options [String] :id stable article identifier
+      # @option options [String] :title article title
+      # @option options [String] :description article description/content
+      # @option options [String, Html2rss::Url] :url canonical article URL
+      # @option options [String, Html2rss::Url] :image image URL for fallback enclosure rendering
+      # @option options [String] :author author name
+      # @option options [String] :guid explicit GUID override
+      # @option options [String, Time, DateTime] :published_at publication timestamp
+      # @option options [Array<Hash{Symbol => Object}>] :enclosures enclosure attribute hashes
+      # @option options [Array<String>] :categories category labels
+      # @option options [Class] :scraper scraper class that produced the article
+      def initialize(**options)
+        @to_h = {}
+        options.each_pair { |key, value| @to_h[key] = value.freeze if value }
+        @to_h.freeze
+
+        return unless (unknown_keys = options.keys - PROVIDED_KEYS).any?
+
+        Log.warn "Article: unknown keys found: #{unknown_keys.join(', ')}"
+      end
+
+      # Checks if the article is valid based on the presence of URL, ID, and either title or description.
+      # @return [Boolean] True if the article is valid, otherwise false.
+      def valid?
+        !url.to_s.empty? && (!title.to_s.empty? || !description.to_s.empty?) && !id.to_s.empty?
+      end
+
+      # @yield [key, value]
+      # @return [Enumerator] if no block is given
+      def each
+        return enum_for(:each) unless block_given?
+
+        PROVIDED_KEYS.each { |key| yield(key, public_send(key)) }
+      end
+
+      # @return [String, nil] stable article identifier
+      def id = blank_string_to_nil(@to_h[:id])
+
+      # @return [String, nil] article title
+      def title = blank_string_to_nil(@to_h[:title])
+
+      # @return [String] rendered article description
+      def description
+        @description ||= Rendering::DescriptionBuilder.new(
+          base: @to_h[:description],
+          title:,
+          url:,
+          enclosures:,
+          image:
+        ).call
+      end
+
+      # @return [Url, nil]
+      def url
+        @url ||= Url.sanitize(@to_h[:url])
+      end
+
+      # @return [Url, nil]
+      def image
+        @image ||= Url.sanitize(@to_h[:image])
+      end
+
+      # @return [String, nil]
+      def author = blank_string_to_nil(@to_h[:author])
+
+      # Generates a unique identifier based on the URL and ID using CRC32.
+      # @return [String]
+      def guid
+        @guid ||= Zlib.crc32(fetch_guid).to_s(36).encode('utf-8')
+      end
+
+      ##
+      # Returns a deterministic fingerprint used to detect duplicate articles.
+      #
+      # @return [String, Integer]
+      def deduplication_fingerprint
+        dedup_from_url || dedup_from_id || dedup_from_guid || hash
+      end
+
+      # @return [Array<Html2rss::RssBuilder::Enclosure>] normalized enclosure objects
+      def enclosures
+        @enclosures ||= Array(@to_h[:enclosures])
+                        .map { |enclosure| Html2rss::RssBuilder::Enclosure.new(**enclosure) }
+      end
+
+      # @return [Html2rss::RssBuilder::Enclosure, nil]
+      def enclosure
+        return @enclosure if defined?(@enclosure)
+
+        case (object = @to_h[:enclosures]&.first)
+        when Hash
+          @enclosure = Html2rss::RssBuilder::Enclosure.new(**object)
+        when nil
+          @enclosure = Html2rss::RssBuilder::Enclosure.new(url: image) if image
+        else
+          Log.warn "Article: unknown enclosure type: #{object.class}"
+        end
+      end
+
+      # @return [Array<String>] normalized, unique category names
+      def categories
+        @categories ||= @to_h[:categories].dup.to_a.tap do |categories|
+          categories.map! { |category| category.to_s.strip }
+          categories.reject!(&:empty?)
+          categories.uniq!
+        end
+      end
+
+      # Parses and returns the published_at time.
+      # @return [DateTime, nil]
+      def published_at
+        return if (string = @to_h[:published_at].to_s.strip).empty?
+
+        @published_at ||= DateTime.parse(string)
+      rescue ArgumentError
+        nil
+      end
+
+      # @return [Class, nil] scraper class that produced this article
+      def scraper
+        @to_h[:scraper]
+      end
+
+      # @param other [Object] value compared against this article
+      # @return [Integer, nil] comparison result for compatible Article values
+      def <=>(other)
+        return nil unless other.is_a?(Article)
+
+        0 if other.all? { |key, value| value == public_send(key) ? public_send(key) <=> value : false }
+      end
+
+      private
+
+      def dedup_from_url
+        return unless (value = url)
+
+        [value.to_s, id].compact.join(DEDUP_FINGERPRINT_SEPARATOR)
+      end
+
+      def dedup_from_id
+        return if id.to_s.empty?
+
+        id
+      end
+
+      def dedup_from_guid
+        value = guid
+        return if value.to_s.empty?
+
+        [value, title, description].compact.join(DEDUP_FINGERPRINT_SEPARATOR)
+      end
+
+      def fetch_guid
+        guid = @to_h[:guid].map { |s| s.to_s.strip }.reject(&:empty?).join if @to_h[:guid].is_a?(Array)
+
+        guid || [url, id].join('#!/')
+      end
+
+      def blank_string_to_nil(value)
+        return if value.is_a?(String) && value.strip.empty?
+
+        value
+      end
+    end
+  end
+end

data/lib/html2rss/rss_builder/channel.rb

@@ -1,20 +1,114 @@
 # frozen_string_literal: true
 
 module Html2rss
-  module RssBuilder
+  class RssBuilder
     ##
-    # Builds the <channel> tag (with the provided maker).
+    # Extracts channel information from
+    # 1. the HTML document's <head>.
+    # 2. the HTTP response
     class Channel
+      # Fallback RSS ttl (in minutes) when no cache directives are present.
+      DEFAULT_TTL_IN_MINUTES = 360
+      # Description template used when no explicit or discovered description exists.
+      DEFAULT_DESCRIPTION_TEMPLATE = 'Latest items from %<url>s'
+
       ##
-      # @param maker [RSS::Maker::RSS20::Channel]
-      # @param config [Html2rss::Config]
-      # @param tags [Set<Symbol>]
-      # @return nil
-      def self.add(maker, config, tags)
-        tags.each { |tag| maker.public_send(:"#{tag}=", config.public_send(tag)) }
-
-        maker.generator = "html2rss V. #{::Html2rss::VERSION}"
-        maker.lastBuildDate = Time.now
+      # @param response [Html2rss::RequestService::Response]
+      # @param overrides [Hash{Symbol => String}] optional overrides for channel attributes
+      def initialize(response, overrides: {})
+        @response = response
+        @overrides = overrides
+      end
+
+      # @return [String] channel title derived from overrides, document title, or URL
+      def title
+        @title ||= fetch_title
+      end
+
+      # @return [Html2rss::Url] canonical channel URL
+      def url = @url ||= Html2rss::Url.from_absolute(@response.url)
+
+      # @return [String] channel description text
+      def description
+        return overrides[:description] unless overrides[:description].to_s.empty?
+
+        description = parsed_body.at_css('meta[name="description"]')&.[]('content') if html_response?
+
+        return format(DEFAULT_DESCRIPTION_TEMPLATE, url:) if description.to_s.empty?
+
+        description
+      end
+
+      # @return [Integer] cache time-to-live in minutes
+      def ttl
+        return overrides[:ttl] if overrides[:ttl]
+
+        if (ttl = headers['cache-control']&.match(/max-age=(\d+)/)&.[](1))
+          return ttl.to_i.fdiv(60).ceil
+        end
+
+        DEFAULT_TTL_IN_MINUTES
+      end
+
+      # @return [String, nil] ISO-like language code when available
+      def language
+        return overrides[:language] if overrides[:language]
+
+        if (language_code = headers['content-language']&.match(/^([a-z]{2})/))
+          return language_code[0]
+        end
+
+        return unless html_response?
+
+        parsed_body['lang'] || parsed_body.at_css('[lang]')&.[]('lang')
+      end
+
+      # @return [String, nil] channel author metadata
+      def author
+        return overrides[:author] if overrides[:author]
+
+        return unless html_response?
+
+        parsed_body.at_css('meta[name="author"]')&.[]('content')
+      end
+
+      # @return [String, Time] source last-modified timestamp or current time fallback
+      def last_build_date = headers['last-modified'] || Time.now
+
+      # @return [Html2rss::Url, nil] channel image URL
+      def image
+        return overrides[:image] if overrides[:image]
+
+        return unless html_response?
+
+        if (image_url = parsed_body.at_css('meta[property="og:image"]')&.[]('content'))
+          Url.sanitize(image_url)
+        end
+      end
+
+      private
+
+      attr_reader :overrides
+
+      def parsed_body = @parsed_body ||= @response.parsed_body
+      def headers = @headers ||= @response.headers
+      def html_response? = @html_response ||= @response.html_response?
+
+      def fetch_title
+        override_title = overrides[:title]
+        return override_title if override_title
+        return parsed_title if parsed_title
+
+        url.channel_titleized
+      end
+
+      def parsed_title
+        return unless html_response?
+
+        title = parsed_body.at_css('head > title')&.text.to_s
+        return if title.empty?
+
+        title.gsub(/\s+/, ' ').strip
       end
     end
   end

data/lib/html2rss/rss_builder/enclosure.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'mime/types'
+
+module Html2rss
+  class RssBuilder
+    ##
+    # Represents an enclosure for an RSS item.
+    class Enclosure
+      ##
+      # Guesses the content type based on the file extension of the URL.
+      #
+      # @param url [Html2rss::Url]
+      # @param default [String] default content type
+      # @return [String] guessed content type, or default
+      def self.guess_content_type_from_url(url, default: 'application/octet-stream')
+        return default unless url
+
+        url = url.path.split('?').first
+
+        content_type = MIME::Types.type_for(File.extname(url).delete('.'))
+        content_type.first&.to_s || 'application/octet-stream'
+      end
+
+      # @param enclosure [Html2rss::RssBuilder::Enclosure, nil] built enclosure object for the current RSS item
+      # @param maker [RSS::Maker::RSS20::ItemsBase::ItemBase] RSS item builder
+      # @return [void]
+      def self.add(enclosure, maker)
+        return unless enclosure
+
+        maker.enclosure.tap do |enclosure_maker|
+          enclosure_maker.url = enclosure.url.to_s
+          enclosure_maker.type = enclosure.type
+          enclosure_maker.length = enclosure.bits_length
+        end
+      end
+
+      # @param url [Html2rss::Url] absolute enclosure URL
+      # @param type [String, nil] optional enclosure MIME type
+      # @param bits_length [Integer] enclosure byte length (historical name)
+      def initialize(url:, type: nil, bits_length: 0)
+        raise ArgumentError, 'An Enclosure requires an absolute URL' if !url || !url.absolute?
+
+        @url = url
+        @type = type
+        @bits_length = bits_length
+      end
+
+      # @return [String] explicit MIME type or one inferred from URL extension
+      def type = @type || self.class.guess_content_type_from_url(url)
+
+      # @return [Integer] enclosure length in bytes
+      def bytes_length = @bits_length
+
+      # @return [Integer] enclosure length in bytes (legacy reader name)
+      def bits_length = bytes_length
+
+      # @return [Html2rss::Url] absolute enclosure URL
+      attr_reader :url
+    end
+  end
+end

data/lib/html2rss/rss_builder/stylesheet.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Html2rss
-  module RssBuilder
+  class RssBuilder
     ##
     # Represents a stylesheet.
     class Stylesheet
@@ -10,7 +10,7 @@ module Html2rss
         # Adds the stylesheet XML tags to the RSS.
         #
         # @param maker [RSS::Maker::RSS20] RSS maker object.
-        # @param stylesheets [Array<Html2rss::Config::Stylesheet>] Array of stylesheet configurations.
+        # @param stylesheets [Array<Html2rss::RssBuilder::Stylesheet>] Array of stylesheet configurations.
         # @return [nil]
         def add(maker, stylesheets)
           stylesheets.each do |stylesheet|
@@ -24,7 +24,7 @@ module Html2rss
         # Adds a single Stylesheet to the RSS.
         #
         # @param maker [RSS::Maker::RSS20] RSS maker object.
-        # @param stylesheet [Html2rss::Config::Stylesheet] Stylesheet configuration.
+        # @param stylesheet [Html2rss::RssBuilder::Stylesheet] Stylesheet configuration.
         # @return [nil]
         def add_stylesheet(maker, stylesheet)
           maker.xml_stylesheets.new_xml_stylesheet do |xss|
@@ -35,8 +35,12 @@ module Html2rss
         end
       end
 
-      TYPES = ['text/css', 'text/xsl'].freeze
+      # Allowed stylesheet MIME types for RSS processing instructions.
+      TYPES = ['text/css', 'text/xsl'].to_set.freeze
 
+      # @param href [String] stylesheet URL
+      # @param type [String] MIME type (`text/css` or `text/xsl`)
+      # @param media [String] media query hint for the stylesheet
       def initialize(href:, type:, media: 'all')
         raise ArgumentError, 'stylesheet.href must be a String' unless href.is_a?(String)
         raise ArgumentError, 'stylesheet.type invalid' unless TYPES.include?(type)

data/lib/html2rss/rss_builder.rb

@@ -4,93 +4,98 @@ require 'rss'
 
 module Html2rss
   ##
-  # Builds the RSS 2.0 feed, which consists of the '<channel>' and the '<item>'s
-  # tags in the RSS.
-  module RssBuilder
-    # Possible tags inside a RSS 2.0 <channel> tag.
-    CHANNEL_TAGS = %i[language author title description link ttl].freeze
-    # Possible tags inside a RSS 2.0 <item> tag.
-    ITEM_TAGS = %i[title link description author comments updated].freeze
+  # Builds an RSS Feed by providing channel, articles and stylesheets.
+  class RssBuilder
+    class << self
+      # @param article [Html2rss::RssBuilder::Article] source article
+      # @param item_maker [RSS::Maker::RSS20::ItemsBase::ItemBase] RSS item builder
+      # @return [void]
+      def add_item(article, item_maker)
+        add_item_string_values(article, item_maker)
+        add_item_categories(article, item_maker)
+        Enclosure.add(article.enclosure, item_maker)
+        add_item_guid(article, item_maker)
+      end
 
-    ##
-    # Builds an RSS 2.0 feed based on the provided configuration.
-    #
-    # @param config [Html2rss::Config] Configuration object containing feed details.
-    # @return [RSS::Rss] RSS feed object.
-    def self.build(config)
-      RSS::Maker.make('2.0') do |maker|
-        add_stylesheets(maker, config.stylesheets)
-        add_channel(maker, config)
-        add_items(maker, config)
+      private
+
+      def add_item_string_values(article, item_maker)
+        %i[title description author].each do |attr|
+          next unless (value = article.send(attr))
+          next if value.empty?
+
+          item_maker.send(:"#{attr}=", value)
+        end
+
+        item_maker.link = article.url.to_s if article.url
+        item_maker.pubDate = article.published_at&.rfc2822
       end
-    end
 
-    ##
-    # Adds stylesheets to the RSS maker.
-    #
-    # @param maker [RSS::Maker] RSS maker instance.
-    # @param stylesheets [Array<String>] Array of stylesheets to add.
-    def self.add_stylesheets(maker, stylesheets)
-      Stylesheet.add(maker, stylesheets)
-    end
+      def add_item_categories(article, item_maker)
+        article.categories.each { |category| item_maker.categories.new_category.content = category }
+      end
 
-    ##
-    # Adds channel information to the RSS maker.
-    #
-    # @param maker [RSS::Maker] RSS maker instance.
-    # @param config [Html2rss::Config] Configuration object containing feed details.
-    def self.add_channel(maker, config)
-      channel = maker.channel
-      CHANNEL_TAGS.each do |tag|
-        Channel.add(channel, config, [tag])
+      def add_item_guid(article, item_maker)
+        item_maker.guid.tap do |guid|
+          guid.content = article.guid
+          guid.isPermaLink = false
+        end
       end
     end
 
     ##
-    # Adds items to the RSS maker based on configuration.
-    #
-    # @param maker [RSS::Maker] RSS maker instance.
-    # @param config [Html2rss::Config] Configuration object containing feed details.
-    def self.add_items(maker, config)
-      item_attributes = extract_item_attributes(config)
-      items = fetch_items(config)
-      items.reverse! if config.items_order == :reverse
-
-      items.each do |item|
-        add_item(maker, item, item_attributes)
+    # @param channel [Html2rss::RssBuilder::Channel] The channel information for the RSS feed.
+    # @param articles [Array<Html2rss::RssBuilder::Article>] The list of articles to include in the RSS feed.
+    # @param stylesheets [Array<Hash>] An optional array of stylesheet configurations.
+    def initialize(channel:, articles:, stylesheets: [])
+      @channel = channel
+      @articles = articles
+      @stylesheets = stylesheets
+    end
+
+    # @return [RSS::Rss] RSS 2.0 document instance
+    def call
+      RSS::Maker.make('2.0') do |maker|
+        Stylesheet.add(maker, stylesheets)
+
+        make_channel(maker.channel)
+        make_items(maker)
       end
     end
 
-    ##
-    # Adds a single item to the RSS maker.
-    #
-    # @param maker [RSS::Maker] RSS maker instance.
-    # @param item [Html2rss::Item] Item to add.
-    # @param item_attributes [Array<Symbol>] Array of item attributes.
-    # @return [nil]
-    def self.add_item(maker, item, item_attributes)
-      new_item = maker.items.new_item
-      Item.add(new_item, item, item_attributes)
+    private
+
+    attr_reader :channel, :articles
+
+    def stylesheets
+      @stylesheets.map { |style| Stylesheet.new(**style) }
     end
 
-    ##
-    # Extracts item attributes from configuration.
-    #
-    # @param config [Html2rss::Config] Configuration object containing feed details.
-    # @return [Array<Symbol>] Array of item attributes.
-    def self.extract_item_attributes(config)
-      config.item_selector_names & ITEM_TAGS
+    def make_channel(maker)
+      %i[language title description ttl].each do |key|
+        maker.public_send(:"#{key}=", channel.public_send(key))
+      end
+
+      maker.link = channel.url.to_s
+      maker.generator = generator
+      maker.updated = channel.last_build_date
     end
 
-    ##
-    # Fetches items from the URL specified in configuration.
-    #
-    # @param config [Html2rss::Config] Configuration object containing feed details.
-    # @return [Array<Html2rss::Item>] Array of items.
-    def self.fetch_items(config)
-      Html2rss::Item.from_url(config.url, config)
+    def make_items(maker)
+      articles.each do |article|
+        maker.items.new_item { |item_maker| self.class.add_item(article, item_maker) }
+      end
     end
 
-    private_class_method :extract_item_attributes, :fetch_items, :add_item
+    def generator
+      scraper_namespace_regex = /(?<namespace>Html2rss|Scraper)::/
+
+      scraper_counts = articles.flat_map(&:scraper).tally.map do |klass, count|
+        scraper_name = klass.to_s.gsub(scraper_namespace_regex, '')
+        "#{scraper_name} (#{count})"
+      end
+
+      "html2rss V. #{Html2rss::VERSION} (scrapers: #{scraper_counts.join(', ')})"
+    end
   end
 end