html2rss 0.23.0 → 0.25.0

data/lib/html2rss/channel.rb

@@ -1,118 +1,198 @@
 # frozen_string_literal: true
 
+require 'time'
+
+# rubocop:disable Metrics/ModuleLength -- Channel value object owns extract + freeze + Marshal
 module Html2rss
   ##
-  # Channel metadata for feed builders, extracted from the HTML document head
-  # and HTTP response (with optional config overrides).
+  # Materialized channel metadata for feed builders and {FeedResult}.
+  #
+  # Extract once via {Channel.from_response}; builders consume this value object and
+  # do not own channel extraction. Safe to Marshal (no Response / Nokogiri retained).
+  # Strings and image URL are normalized and frozen at construction (including Marshal load).
   #
-  # Format adapters ({FeedBuilder::Rss}, {FeedBuilder::JsonFeed}) consume this type; they do
-  # not own channel extraction.
-  class Channel
-    # Fallback RSS ttl (in minutes) when no cache directives are present.
-    DEFAULT_TTL_IN_MINUTES = 360
-    # Description template used when no explicit or discovered description exists.
-    DEFAULT_DESCRIPTION_TEMPLATE = 'Latest items from %<url>s'
+  # @!attribute [r] title
+  #   @return [String]
+  # @!attribute [r] url
+  #   @return [Html2rss::Url]
+  # @!attribute [r] description
+  #   @return [String]
+  # @!attribute [r] language
+  #   @return [String, nil]
+  # @!attribute [r] ttl
+  #   @return [Integer]
+  # @!attribute [r] last_build_date
+  #   @return [Time]
+  # @!attribute [r] image
+  #   @return [Html2rss::Url, nil]
+  # @!attribute [r] author
+  #   @return [String, nil]
+  Channel = Data.define(:title, :url, :description, :language, :ttl, :last_build_date, :image, :author) do
+    class << self
+      ##
+      # Materializes channel attributes from an HTTP response and optional overrides.
+      #
+      # @param response [Html2rss::RequestService::Response]
+      # @param overrides [Hash{Symbol => Object}] optional overrides for channel attributes
+      # @return [Html2rss::Channel]
+      def from_response(response, overrides: {})
+        new(
+          title: title_for(response, overrides),
+          url: url_for(response),
+          description: description_for(response, overrides),
+          language: language_for(response, overrides),
+          ttl: ttl_for(response, overrides),
+          last_build_date: last_build_date_for(response),
+          image: image_for(response, overrides),
+          author: author_for(response, overrides)
+        )
+      end
 
-    ##
-    # @param response [Html2rss::RequestService::Response]
-    # @param overrides [Hash{Symbol => String}] optional overrides for channel attributes
-    def initialize(response, overrides: {})
-      @response = response
-      @overrides = overrides
-    end
+      private
 
-    # @return [String] channel title derived from overrides, document title, or URL
-    def title
-      @title ||= fetch_title
-    end
+      # Fallback RSS ttl (in minutes) when no cache directives are present.
+      def default_ttl_in_minutes = 360
 
-    # @return [Html2rss::Url] canonical channel URL
-    def url = @url ||= Html2rss::Url.from_absolute(@response.url)
+      # Description template used when no explicit or discovered description exists.
+      def default_description_template = 'Latest items from %<url>s'
 
-    # @return [String] channel description text
-    def description
-      return overrides[:description] unless overrides[:description].to_s.empty?
+      def url_for(response) = Html2rss::Url.from_absolute(response.url)
 
-      description = parsed_body.at_css('meta[name="description"]')&.[]('content') if html_response?
+      def title_for(response, overrides)
+        return overrides[:title] if overrides[:title]
 
-      return format(DEFAULT_DESCRIPTION_TEMPLATE, url:) if description.to_s.empty?
+        title = parsed_title(response)
+        return title if title
 
-      description
-    end
+        url_for(response).channel_titleized
+      end
 
-    # @return [Integer] cache time-to-live in minutes
-    def ttl
-      calculated = if overrides[:ttl]
-                     overrides[:ttl].to_i
-                   elsif (max_age = headers['cache-control']&.match(/max-age=(\d+)/)&.[](1))
-                     max_age.to_i.fdiv(60).ceil
-                   else
-                     DEFAULT_TTL_IN_MINUTES
-                   end
-
-      min_ttl = Html2rss.defaults.min_ttl
-      min_ttl ? [calculated, min_ttl].max : calculated
-    end
+      def parsed_title(response)
+        return unless html_response?(response)
 
-    # @return [String, nil] ISO-like language code when available
-    def language
-      return overrides[:language] if overrides[:language]
+        title = parsed_body(response).at_css('head > title')&.text.to_s
+        return if title.empty?
 
-      if (language_code = headers['content-language']&.match(/^([a-z]{2})/))
-        return language_code[0]
+        title.gsub(/\s+/, ' ').strip
       end
 
-      return unless html_response?
+      def description_for(response, overrides)
+        return overrides[:description] unless overrides[:description].to_s.empty?
 
-      parsed_body['lang'] || parsed_body.at_css('[lang]')&.[]('lang')
-    end
+        description = meta_description(response)
+        return format(default_description_template, url: url_for(response)) if description.to_s.empty?
 
-    # @return [String, nil] channel author metadata
-    def author
-      return overrides[:author] if overrides[:author]
+        description
+      end
 
-      return unless html_response?
+      def meta_description(response)
+        return unless html_response?(response)
 
-      parsed_body.at_css('meta[name="author"]')&.[]('content')
-    end
+        parsed_body(response).at_css('meta[name="description"]')&.[]('content')
+      end
+
+      def ttl_for(response, overrides)
+        calculated = if overrides[:ttl]
+                       overrides[:ttl].to_i
+                     elsif (max_age = headers(response)['cache-control']&.match(/max-age=(\d+)/)&.[](1))
+                       max_age.to_i.fdiv(60).ceil
+                     else
+                       default_ttl_in_minutes
+                     end
+
+        min_ttl = Html2rss.defaults.min_ttl
+        min_ttl ? [calculated, min_ttl].max : calculated
+      end
 
-    # @return [String, Time] source last-modified timestamp or current time fallback
-    def last_build_date = headers['last-modified'] || Time.now
+      def language_for(response, overrides)
+        return overrides[:language] if overrides[:language]
 
-    # @return [Html2rss::Url, nil] channel image URL
-    def image
-      return overrides[:image] if overrides[:image]
+        if (language_code = headers(response)['content-language']&.match(/^([a-z]{2})/))
+          return language_code[0]
+        end
 
-      return unless html_response?
+        return unless html_response?(response)
 
-      if (image_url = parsed_body.at_css('meta[property="og:image"]')&.[]('content'))
-        Url.sanitize(image_url)
+        parsed_body(response)['lang'] || parsed_body(response).at_css('[lang]')&.[]('lang')
       end
+
+      def author_for(response, overrides)
+        return overrides[:author] if overrides[:author]
+        return unless html_response?(response)
+
+        parsed_body(response).at_css('meta[name="author"]')&.[]('content')
+      end
+
+      def last_build_date_for(response)
+        header = headers(response)['last-modified']
+        return Time.now unless header
+
+        Time.httpdate(header)
+      rescue ArgumentError
+        Time.now
+      end
+
+      def image_for(response, overrides)
+        return overrides[:image] if overrides[:image]
+        return unless html_response?(response)
+
+        if (image_url = parsed_body(response).at_css('meta[property="og:image"]')&.[]('content'))
+          Url.sanitize(image_url)
+        end
+      end
+
+      def parsed_body(response) = response.parsed_body
+      def headers(response) = response.headers
+      def html_response?(response) = response.html_response?
+    end
+
+    ##
+    # @param title [String]
+    # @param url [Html2rss::Url, String]
+    # @param description [String]
+    # @param language [String, nil]
+    # @param ttl [Integer]
+    # @param last_build_date [Time, String]
+    # @param image [Html2rss::Url, String, nil]
+    # @param author [String, nil]
+    # rubocop:disable Metrics/ParameterLists -- Data.define members are the channel contract
+    def initialize(title:, url:, description:, language:, ttl:, last_build_date:, image:, author:)
+      super(
+        title: freeze_string(title),
+        url: url.is_a?(Url) ? url : Url.from_absolute(url),
+        description: freeze_string(description),
+        language: language && freeze_string(language),
+        ttl: Integer(ttl),
+        last_build_date: normalize_last_build_date(last_build_date),
+        image: normalize_image(image),
+        author: author && freeze_string(author)
+      )
     end
+    # rubocop:enable Metrics/ParameterLists
 
     private
 
-    attr_reader :overrides
+    def marshal_dump = [title, url, description, language, ttl, last_build_date, image, author]
 
-    def parsed_body = @parsed_body ||= @response.parsed_body
-    def headers = @headers ||= @response.headers
-    def html_response? = @html_response ||= @response.html_response?
+    def marshal_load((title, url, description, language, ttl, last_build_date, image, author))
+      initialize(title:, url:, description:, language:, ttl:, last_build_date:, image:, author:)
+    end
 
-    def fetch_title
-      override_title = overrides[:title]
-      return override_title if override_title
-      return parsed_title if parsed_title
+    def freeze_string(value) = value.to_s.dup.freeze
 
-      url.channel_titleized
-    end
+    def normalize_image(value)
+      return value if value.is_a?(Url)
+      return if value.nil?
 
-    def parsed_title
-      return unless html_response?
+      Url.sanitize(value.to_s)
+    end
 
-      title = parsed_body.at_css('head > title')&.text.to_s
-      return if title.empty?
+    def normalize_last_build_date(value)
+      return value if value.is_a?(Time)
+      raise ArgumentError, 'last_build_date must be a Time or HTTP-date String' unless value.is_a?(String)
 
-      title.gsub(/\s+/, ' ').strip
+      Time.httpdate(value)
     end
   end
+  # rubocop:enable Metrics/ModuleLength
 end

data/lib/html2rss/cli.rb

@@ -79,6 +79,9 @@ module Html2rss
     method_option :max_requests,
                   type: :numeric,
                   desc: 'Maximum requests to allow for this feed build'
+    method_option :limit,
+                  type: :numeric,
+                  desc: 'Maximum number of articles to keep (default: 25)'
     method_option :input,
                   type: :string,
                   desc: 'Local HTML file path to read input from'
@@ -187,22 +190,13 @@ module Html2rss
     end
 
     def request_controls
-      Html2rss::Config::RequestControls.new(
-        strategy: options[:strategy]&.to_sym,
+      Html2rss::Config::RequestControls.from_cli_options(
+        strategy: options[:strategy],
         max_redirects: options[:max_redirects],
-        max_requests: options[:max_requests],
-        explicit_keys: explicit_request_control_keys
+        max_requests: options[:max_requests]
       )
     end
 
-    def explicit_request_control_keys
-      keys = []
-      keys << :strategy if options[:strategy]
-      keys << :max_redirects unless options[:max_redirects].nil?
-      keys << :max_requests unless options[:max_requests].nil?
-      keys
-    end
-
     def current_strategy
       options[:strategy]&.to_sym || :auto
     end
@@ -249,7 +243,8 @@ module Html2rss
         items_selector: options[:items_selector],
         max_redirects: options[:max_redirects],
         max_requests: options[:max_requests],
-        local_file_path:
+        local_file_path:,
+        limit: options[:limit]&.to_i
       )
     end
 

data/lib/html2rss/config/auto_source_contract.rb

@@ -6,6 +6,8 @@ module Html2rss
   class Config
     # Runtime source of truth for validating auto-source config values.
     AutoSourceContract = Dry::Schema.Params do # rubocop:disable Metrics/BlockLength
+      optional(:limit).filled(:integer, gt?: 0)
+
       optional(:scraper).hash do # rubocop:disable Metrics/BlockLength
         optional(:wordpress_api).hash do
           optional(:enabled).filled(:bool)

data/lib/html2rss/config/request_controls.rb

@@ -43,6 +43,39 @@ module Html2rss
         request_config.is_a?(Hash) && request_config.key?(key)
       end
 
+      ##
+      # Builds controls for public API shortcuts ({Html2rss.auto_source}, etc.).
+      # Strategy is explicit only when non-nil and not the configured default.
+      #
+      # @param strategy [Symbol, nil]
+      # @param max_redirects [Integer, nil]
+      # @param max_requests [Integer, nil]
+      # @return [RequestControls]
+      def self.from_shortcut(strategy:, max_redirects: nil, max_requests: nil)
+        keys = []
+        keys << :strategy unless strategy.nil? || strategy == Config.default_strategy_name
+        keys << :max_redirects unless max_redirects.nil?
+        keys << :max_requests unless max_requests.nil?
+        new(strategy:, max_redirects:, max_requests:, explicit_keys: keys)
+      end
+
+      ##
+      # Builds controls from CLI option values. Strategy is explicit when the
+      # option was provided (truthy string/symbol), matching Thor option presence.
+      #
+      # @param strategy [Symbol, String, nil]
+      # @param max_redirects [Integer, nil]
+      # @param max_requests [Integer, nil]
+      # @return [RequestControls]
+      def self.from_cli_options(strategy:, max_redirects: nil, max_requests: nil)
+        strategy_sym = strategy&.to_sym
+        keys = []
+        keys << :strategy if strategy
+        keys << :max_redirects unless max_redirects.nil?
+        keys << :max_requests unless max_requests.nil?
+        new(strategy: strategy_sym, max_redirects:, max_requests:, explicit_keys: keys)
+      end
+
       private_class_method :explicit_keys_for, :request_value_for, :request_key?
 
       ##

data/lib/html2rss/config/validator.rb

@@ -24,6 +24,8 @@ module Html2rss
         optional(:language).maybe(:string, format?: LANGUAGE_FORMAT_REGEX)
         optional(:ttl).maybe(:integer, gt?: 0)
         optional(:time_zone).maybe(:string)
+        optional(:author).maybe(:string)
+        optional(:image).maybe(:string, format?: URI_REGEXP)
       end
 
       # Contract for a stylesheet entry in `stylesheets`.
@@ -132,15 +134,20 @@ module Html2rss
 
       # URL validation delegated to Url class
       rule(:channel) do
-        next unless values[:channel]&.key?(:url)
-
-        url_string = values[:channel][:url]
-        next if url_string.nil? || url_string.empty?
+        if (url_string = values.dig(:channel, :url)) && !url_string.empty?
+          begin
+            Html2rss::Url.for_channel(url_string)
+          rescue ArgumentError => error
+            key(%i[channel url]).failure(error.message)
+          end
+        end
 
-        begin
-          Html2rss::Url.for_channel(url_string)
-        rescue ArgumentError => error
-          key(%i[channel url]).failure(error.message)
+        if (image_string = values.dig(:channel, :image)) && !image_string.empty?
+          begin
+            Html2rss::Url.from_absolute(image_string)
+          rescue ArgumentError => error
+            key(%i[channel image]).failure(error.message)
+          end
         end
       end
     end

data/lib/html2rss/config.rb

@@ -120,11 +120,15 @@ module Html2rss
       # @param url [String] source page URL
       # @param items_selector [String, nil] optional selector hint for item extraction
       # @param request_controls [Html2rss::Config::RequestControls, nil] explicit request controls to write
+      # @param limit [Integer, nil] max articles to keep in the auto-sourced feed
       # @return [Hash{Symbol => Object}] feed config hash ready for {from_hash}
-      def auto_source_config(url:, items_selector: nil, request_controls: nil)
+      def auto_source_config(url:, items_selector: nil, request_controls: nil, limit: nil)
+        auto_source = AutoSource::DEFAULT_CONFIG
+        auto_source = auto_source.merge(limit:) unless limit.nil?
+
         config = {
           channel: default_config[:channel].merge(url:),
-          auto_source: AutoSource::DEFAULT_CONFIG
+          auto_source:
         }
 
         request_controls ||= RequestControls.new

data/lib/html2rss/feed_builder/item_presentation.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module FeedBuilder
+    ##
+    # Presentation rules for rendering an {Html2rss::Article} into feed wire formats.
+    #
+    # Owns description enrichment ({Html::Rendering::DescriptionBuilder}) and RSS
+    # non-image enclosure selection. {Article} keeps raw extracted fields only.
+    module ItemPresentation
+      class << self
+        ##
+        # @param article [Html2rss::Article]
+        # @return [String, nil] sanitized description HTML/text for feed item bodies
+        def description_for(article)
+          Html::Rendering::DescriptionBuilder.new(
+            base: article.description,
+            title: article.title,
+            url: article.url,
+            enclosures: article.enclosures,
+            image: article.image
+          ).call
+        end
+
+        ##
+        # First non-image enclosure for RSS +enclosure+ (images stay on description / JSON Feed).
+        #
+        # @param article [Html2rss::Article]
+        # @return [Html2rss::Article::Enclosure, nil]
+        def rss_enclosure_for(article)
+          article.enclosures.find { |enc| !image_mime_type?(enc.type) }
+        end
+
+        private
+
+        def image_mime_type?(type)
+          type.to_s.downcase.start_with?('image/')
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/feed_builder/json_feed/item.rb

@@ -50,17 +50,33 @@ module Html2rss
 
         ##
         # JSON Feed items must include content_html or content_text.
+        # Title is a fallback only when description is absent.
+        #
         # @return [Hash]
         def content_fields
-          description = article.description
-          return { content_html: description } if description
+          if (description = ItemPresentation.description_for(article))
+            return content_for(description)
+          end
 
-          title = article.title
-          return { content_text: title } if title
+          return { content_text: article.title } if article.title
 
           {}
         end
 
+        ##
+        # @param description [String]
+        # @return [Hash]
+        def content_for(description)
+          html?(description) ? { content_html: description } : { content_text: description }
+        end
+
+        ##
+        # @param value [String]
+        # @return [Boolean]
+        def html?(value)
+          Nokogiri::HTML.fragment(value).children.any?(&:element?)
+        end
+
         ##
         # @return [Array<String>, nil]
         def tags

data/lib/html2rss/feed_builder/json_feed.rb

@@ -13,9 +13,15 @@ module Html2rss
       ##
       # @param channel [Html2rss::Channel]
       # @param articles [Array<Html2rss::Article>]
-      def initialize(channel:, articles:)
+      # @param user_comment [String] required generator comment (from {Status})
+      # @param feed_url [String, nil] optional absolute self URL for the feed (JSON Feed feed_url)
+      def initialize(channel:, articles:, user_comment:, feed_url: nil)
+        raise ArgumentError, 'user_comment must be a non-blank String' if blank_string?(user_comment)
+
         @channel = channel
         @articles = articles
+        @feed_url = normalize_feed_url(feed_url)
+        @user_comment = user_comment
       end
 
       ##
@@ -28,7 +34,19 @@ module Html2rss
 
       private
 
-      attr_reader :channel, :articles
+      attr_reader :channel, :articles, :feed_url, :user_comment
+
+      def blank_string?(value)
+        !value.is_a?(String) || value.strip.empty?
+      end
+
+      def normalize_feed_url(value)
+        return if value.nil?
+        raise ArgumentError, 'feed_url must be a String or nil' unless value.is_a?(String)
+        return if value.strip.empty?
+
+        Url.from_absolute(value).to_s
+      end
 
       ##
       # @return [Hash]
@@ -37,7 +55,9 @@ module Html2rss
           version: VERSION_URL,
           title: channel.title,
           home_page_url: channel.url.to_s,
+          feed_url:,
           description: channel.description,
+          user_comment:,
           language: channel.language,
           icon: channel.image&.to_s
         }

data/lib/html2rss/feed_builder/rss.rb

@@ -14,7 +14,7 @@ module Html2rss
         def add_item(article, item_maker)
           add_item_string_values(article, item_maker)
           add_item_categories(article, item_maker)
-          add_enclosure(article.enclosure, item_maker)
+          add_enclosure(ItemPresentation.rss_enclosure_for(article), item_maker)
           add_item_guid(article, item_maker)
         end
 
@@ -34,11 +34,14 @@ module Html2rss
         private
 
         def add_item_string_values(article, item_maker)
-          %i[title description author].each do |attr|
-            next unless (value = article.send(attr))
-            next if value.empty?
-
-            item_maker.send(:"#{attr}=", value)
+          {
+            title: article.title,
+            description: ItemPresentation.description_for(article),
+            author: article.author
+          }.each do |attr, value|
+            next if value.nil? || value.empty?
+
+            item_maker.public_send(:"#{attr}=", value)
           end
 
           item_maker.link = article.url.to_s if article.url
@@ -61,10 +64,14 @@ module Html2rss
       # @param channel [Html2rss::Channel] The channel information for the RSS feed.
       # @param articles [Array<Html2rss::Article>] The list of articles to include in the RSS feed.
       # @param stylesheets [Array<Hash>] An optional array of stylesheet configurations.
-      def initialize(channel:, articles:, stylesheets: [])
+      # @param generator [String] required preformatted generator comment (from {Status})
+      def initialize(channel:, articles:, generator:, stylesheets: [])
+        raise ArgumentError, 'generator must be a non-blank String' if blank_string?(generator)
+
         @channel = channel
         @articles = articles
         @stylesheets = stylesheets
+        @generator = generator
       end
 
       # @return [RSS::Rss] RSS 2.0 document instance
@@ -72,27 +79,43 @@ module Html2rss
         RSS::Maker.make('2.0') do |maker|
           Stylesheet.add(maker, stylesheets)
 
-          make_channel(maker.channel)
+          make_channel(maker)
           make_items(maker)
         end
       end
 
       private
 
-      attr_reader :channel, :articles
+      attr_reader :channel, :articles, :generator
+
+      def blank_string?(value)
+        !value.is_a?(String) || value.strip.empty?
+      end
 
       def stylesheets
         @stylesheets.map { |style| Stylesheet.new(**style) }
       end
 
+      # rubocop:disable Metrics/AbcSize
       def make_channel(maker)
+        channel_maker = maker.channel
         %i[language title description ttl].each do |key|
-          maker.public_send(:"#{key}=", channel.public_send(key))
+          channel_maker.public_send(:"#{key}=", channel.public_send(key))
         end
 
-        maker.link = channel.url.to_s
-        maker.generator = generator
-        maker.updated = channel.last_build_date
+        channel_maker.managingEditor = channel.author if channel.author
+        channel_maker.author = channel.author if channel.author
+        channel_maker.link = channel.url.to_s
+        channel_maker.generator = generator
+        channel_maker.updated = channel.last_build_date
+
+        make_image(maker.image) if channel.image
+      end
+      # rubocop:enable Metrics/AbcSize
+
+      def make_image(image_maker)
+        image_maker.url = channel.image.to_s
+        image_maker.title = channel.title.to_s
       end
 
       def make_items(maker)
@@ -100,17 +123,6 @@ module Html2rss
           maker.items.new_item { |item_maker| self.class.add_item(article, item_maker) }
         end
       end
-
-      def generator
-        scraper_namespace_regex = /(?<namespace>Html2rss|Scraper)::/
-
-        scraper_counts = articles.flat_map(&:scraper).tally.map do |klass, count|
-          scraper_name = klass.to_s.gsub(scraper_namespace_regex, '')
-          "#{scraper_name} (#{count})"
-        end
-
-        "html2rss V. #{Html2rss::VERSION} (scrapers: #{scraper_counts.join(', ')})"
-      end
     end
   end
 end