html2rss 0.24.0 → 0.25.0

data/lib/html2rss/cli.rb

@@ -79,6 +79,9 @@ module Html2rss
     method_option :max_requests,
                   type: :numeric,
                   desc: 'Maximum requests to allow for this feed build'
+    method_option :limit,
+                  type: :numeric,
+                  desc: 'Maximum number of articles to keep (default: 25)'
     method_option :input,
                   type: :string,
                   desc: 'Local HTML file path to read input from'
@@ -187,22 +190,13 @@ module Html2rss
     end
 
     def request_controls
-      Html2rss::Config::RequestControls.new(
-        strategy: options[:strategy]&.to_sym,
+      Html2rss::Config::RequestControls.from_cli_options(
+        strategy: options[:strategy],
         max_redirects: options[:max_redirects],
-        max_requests: options[:max_requests],
-        explicit_keys: explicit_request_control_keys
+        max_requests: options[:max_requests]
       )
     end
 
-    def explicit_request_control_keys
-      keys = []
-      keys << :strategy if options[:strategy]
-      keys << :max_redirects unless options[:max_redirects].nil?
-      keys << :max_requests unless options[:max_requests].nil?
-      keys
-    end
-
     def current_strategy
       options[:strategy]&.to_sym || :auto
     end
@@ -249,7 +243,8 @@ module Html2rss
         items_selector: options[:items_selector],
         max_redirects: options[:max_redirects],
         max_requests: options[:max_requests],
-        local_file_path:
+        local_file_path:,
+        limit: options[:limit]&.to_i
       )
     end
 

data/lib/html2rss/config/auto_source_contract.rb

@@ -6,6 +6,8 @@ module Html2rss
   class Config
     # Runtime source of truth for validating auto-source config values.
     AutoSourceContract = Dry::Schema.Params do # rubocop:disable Metrics/BlockLength
+      optional(:limit).filled(:integer, gt?: 0)
+
       optional(:scraper).hash do # rubocop:disable Metrics/BlockLength
         optional(:wordpress_api).hash do
           optional(:enabled).filled(:bool)

data/lib/html2rss/config/request_controls.rb

@@ -43,6 +43,39 @@ module Html2rss
         request_config.is_a?(Hash) && request_config.key?(key)
       end
 
+      ##
+      # Builds controls for public API shortcuts ({Html2rss.auto_source}, etc.).
+      # Strategy is explicit only when non-nil and not the configured default.
+      #
+      # @param strategy [Symbol, nil]
+      # @param max_redirects [Integer, nil]
+      # @param max_requests [Integer, nil]
+      # @return [RequestControls]
+      def self.from_shortcut(strategy:, max_redirects: nil, max_requests: nil)
+        keys = []
+        keys << :strategy unless strategy.nil? || strategy == Config.default_strategy_name
+        keys << :max_redirects unless max_redirects.nil?
+        keys << :max_requests unless max_requests.nil?
+        new(strategy:, max_redirects:, max_requests:, explicit_keys: keys)
+      end
+
+      ##
+      # Builds controls from CLI option values. Strategy is explicit when the
+      # option was provided (truthy string/symbol), matching Thor option presence.
+      #
+      # @param strategy [Symbol, String, nil]
+      # @param max_redirects [Integer, nil]
+      # @param max_requests [Integer, nil]
+      # @return [RequestControls]
+      def self.from_cli_options(strategy:, max_redirects: nil, max_requests: nil)
+        strategy_sym = strategy&.to_sym
+        keys = []
+        keys << :strategy if strategy
+        keys << :max_redirects unless max_redirects.nil?
+        keys << :max_requests unless max_requests.nil?
+        new(strategy: strategy_sym, max_redirects:, max_requests:, explicit_keys: keys)
+      end
+
       private_class_method :explicit_keys_for, :request_value_for, :request_key?
 
       ##

data/lib/html2rss/config/validator.rb

@@ -24,6 +24,8 @@ module Html2rss
         optional(:language).maybe(:string, format?: LANGUAGE_FORMAT_REGEX)
         optional(:ttl).maybe(:integer, gt?: 0)
         optional(:time_zone).maybe(:string)
+        optional(:author).maybe(:string)
+        optional(:image).maybe(:string, format?: URI_REGEXP)
       end
 
       # Contract for a stylesheet entry in `stylesheets`.
@@ -132,15 +134,20 @@ module Html2rss
 
       # URL validation delegated to Url class
       rule(:channel) do
-        next unless values[:channel]&.key?(:url)
-
-        url_string = values[:channel][:url]
-        next if url_string.nil? || url_string.empty?
+        if (url_string = values.dig(:channel, :url)) && !url_string.empty?
+          begin
+            Html2rss::Url.for_channel(url_string)
+          rescue ArgumentError => error
+            key(%i[channel url]).failure(error.message)
+          end
+        end
 
-        begin
-          Html2rss::Url.for_channel(url_string)
-        rescue ArgumentError => error
-          key(%i[channel url]).failure(error.message)
+        if (image_string = values.dig(:channel, :image)) && !image_string.empty?
+          begin
+            Html2rss::Url.from_absolute(image_string)
+          rescue ArgumentError => error
+            key(%i[channel image]).failure(error.message)
+          end
         end
       end
     end

data/lib/html2rss/config.rb

@@ -120,11 +120,15 @@ module Html2rss
       # @param url [String] source page URL
       # @param items_selector [String, nil] optional selector hint for item extraction
       # @param request_controls [Html2rss::Config::RequestControls, nil] explicit request controls to write
+      # @param limit [Integer, nil] max articles to keep in the auto-sourced feed
       # @return [Hash{Symbol => Object}] feed config hash ready for {from_hash}
-      def auto_source_config(url:, items_selector: nil, request_controls: nil)
+      def auto_source_config(url:, items_selector: nil, request_controls: nil, limit: nil)
+        auto_source = AutoSource::DEFAULT_CONFIG
+        auto_source = auto_source.merge(limit:) unless limit.nil?
+
         config = {
           channel: default_config[:channel].merge(url:),
-          auto_source: AutoSource::DEFAULT_CONFIG
+          auto_source:
         }
 
         request_controls ||= RequestControls.new

data/lib/html2rss/feed_builder/rss.rb

@@ -79,7 +79,7 @@ module Html2rss
         RSS::Maker.make('2.0') do |maker|
           Stylesheet.add(maker, stylesheets)
 
-          make_channel(maker.channel)
+          make_channel(maker)
           make_items(maker)
         end
       end
@@ -96,14 +96,26 @@ module Html2rss
         @stylesheets.map { |style| Stylesheet.new(**style) }
       end
 
+      # rubocop:disable Metrics/AbcSize
       def make_channel(maker)
+        channel_maker = maker.channel
         %i[language title description ttl].each do |key|
-          maker.public_send(:"#{key}=", channel.public_send(key))
+          channel_maker.public_send(:"#{key}=", channel.public_send(key))
         end
 
-        maker.link = channel.url.to_s
-        maker.generator = generator
-        maker.updated = channel.last_build_date
+        channel_maker.managingEditor = channel.author if channel.author
+        channel_maker.author = channel.author if channel.author
+        channel_maker.link = channel.url.to_s
+        channel_maker.generator = generator
+        channel_maker.updated = channel.last_build_date
+
+        make_image(maker.image) if channel.image
+      end
+      # rubocop:enable Metrics/AbcSize
+
+      def make_image(image_maker)
+        image_maker.url = channel.image.to_s
+        image_maker.title = channel.title.to_s
       end
 
       def make_items(maker)

data/lib/html2rss/feed_pipeline.rb

@@ -81,6 +81,10 @@ module Html2rss
       request_session = request_session_for(config, strategy:, resources:)
       response = request_session.fetch_initial_response
       articles, dedup_dropped = deduplicated_articles(config:, response:, request_session:)
+      if config.auto_source && articles.empty?
+        raise NoFeedItemsExtracted.new(attempts: [{ strategy:, items_count: 0, error_class: nil }])
+      end
+
       PipelineOutcome.new(
         response:, articles:, dedup_dropped:, selected_strategy: nil, attempt_count: 0, strategy_attempts: []
       )

data/lib/html2rss/feed_result.rb

@@ -69,7 +69,7 @@ module Html2rss
 
     ##
     # @return [Html2rss::Status] frozen scraper tallies, dedup count, and generator formatter.
-    #   {#to_h} on +status+ is a stable consumer contract for observability payloads.
+    #   {Status#to_h} on +status+ is a stable consumer contract for observability payloads.
     attr_reader :status
 
     private

data/lib/html2rss/html/article_extractor/category_extractor.rb

@@ -7,20 +7,16 @@ module Html2rss
       # CategoryExtractor is responsible for extracting categories from HTML elements
       # by looking for CSS class names containing common category-related terms.
       class CategoryExtractor
-        # Common category-related terms to look for in class names
-        CATEGORY_TERMS = %w[
-          category categories tag tags topic topics section sections
-          label labels theme themes subject subjects
-        ].freeze
+        # Shared category vocabulary (owned by {ArticleRules::Category}).
+        CATEGORY_TERMS = ArticleRules::Category::CATEGORY_TERMS
+        # Shared category attribute/class pattern (owned by {ArticleRules::Category}).
+        CATEGORY_ATTR_PATTERN = ArticleRules::Category::CATEGORY_ATTR_PATTERN
 
         # CSS selectors to find elements with category-related class names or data attributes
         CATEGORY_SELECTORS = CATEGORY_TERMS.flat_map do |term|
           ["[class*=\"#{term}\"]", "[data-#{term}]", "[#{term}]"]
         end.freeze
 
-        # Regex pattern for matching category-related attribute names
-        CATEGORY_ATTR_PATTERN = /#{CATEGORY_TERMS.join('|')}/i
-
         ##
         # Extracts categories from the given article tag by looking for elements
         # with class names containing common category-related terms.
@@ -30,7 +26,6 @@ module Html2rss
         def self.call(article_tag)
           return [] unless article_tag
 
-          # Single optimized traversal that extracts all category types
           extract_all_categories(article_tag)
             .map(&:strip)
             .reject(&:empty?)
@@ -44,10 +39,7 @@ module Html2rss
         def self.extract_all_categories(article_tag)
           Set.new.tap do |categories|
             article_tag.css(CATEGORY_SELECTORS.join(',')).each do |element|
-              # Extract text categories from elements with category-related class names
-              extract_text_categories!(categories, element) if element['class']&.match?(CATEGORY_ATTR_PATTERN)
-
-              # Extract data categories from all elements
+              extract_text_categories!(categories, element) if ArticleRules::Category.class_match?(element['class'])
               extract_element_data_categories!(categories, element)
             end
           end
@@ -61,10 +53,9 @@ module Html2rss
         # @return [void]
         def self.extract_element_data_categories!(categories, element)
           element.attributes.each_value do |attr|
-            next unless attr.name.match?(CATEGORY_ATTR_PATTERN)
+            next unless ArticleRules::Category.attr_name_match?(attr.name)
 
-            value = attr.value&.strip
-            categories.add(value) if value && !value.empty?
+            ArticleRules::Category.add_text!(categories, attr.value)
           end
         end
 
@@ -85,7 +76,7 @@ module Html2rss
           if anchors.any?
             anchors.each { |node| add_text_to_categories!(categories, node) }
           else
-            extract_split_text_categories!(categories, element)
+            ArticleRules::Category.add_split_text!(categories, element.text)
           end
         end
 
@@ -96,27 +87,10 @@ module Html2rss
         # @param element [Nokogiri::XML::Element] The element to extract text from
         # @return [void]
         def self.add_text_to_categories!(categories, element)
-          text = Navigator.extract_visible_text(element)
-          categories.add(text) if text && !text.empty?
-        end
-
-        ##
-        # Extracts categories from the element's text by splitting on newlines.
-        #
-        # @param categories [Set<String>] Accumulator set
-        # @param element [Nokogiri::XML::Element] The element to extract text from
-        # @return [void]
-        def self.extract_split_text_categories!(categories, element)
-          text = element.text
-          return unless text
-
-          text.split(/\n+/).each do |line|
-            line = line.strip
-            categories.add(line) unless line.empty?
-          end
+          ArticleRules::Category.add_text!(categories, Navigator.extract_visible_text(element))
         end
 
-        private_class_method :add_text_to_categories!, :extract_split_text_categories!
+        private_class_method :add_text_to_categories!
       end
     end
   end

data/lib/html2rss/html/article_extractor/date_extractor.rb

@@ -8,13 +8,8 @@ module Html2rss
         # @param article_tag [Nokogiri::XML::Element] article container node
         # @return [DateTime, nil]
         def self.call(article_tag)
-          times = article_tag.css('[datetime]').filter_map do |tag|
-            DateTime.parse(tag['datetime'])
-          rescue ArgumentError, TypeError
-            nil
-          end
-
-          times.min
+          datetimes = article_tag.css('[datetime]').map { |tag| tag['datetime'] }
+          ArticleRules::Date.earliest(datetimes)
         end
       end
     end

data/lib/html2rss/html/article_extractor/enclosure_extractor.rb

@@ -33,62 +33,17 @@ module Html2rss
         def self.extract_from_element(element, base_url)
           case element.name
           when 'img'
-            extract_image(element, base_url)
+            ArticleRules::Enclosure.from_image(element['src'], base_url)
           when 'video', 'audio', 'source'
-            extract_media(element, base_url)
+            ArticleRules::Enclosure.from_media(element['src'], element['type'], base_url)
           when 'iframe'
-            extract_iframe(element, base_url)
+            ArticleRules::Enclosure.from_iframe(element['src'], base_url)
           when 'a'
-            extract_a(element, base_url)
+            ArticleRules::Enclosure.from_anchor(element['href'], base_url)
           end
         end
 
-        def self.extract_image(img, base_url)
-          src = img['src'].to_s
-          return if src.empty?
-
-          abs_url = Url.from_relative(src, base_url)
-          {
-            url: abs_url,
-            type: Article::Enclosure.guess_content_type_from_url(abs_url, default: 'image/jpeg')
-          }
-        end
-
-        def self.extract_media(element, base_url)
-          src = element['src'].to_s
-          return if src.empty?
-
-          {
-            url: Url.from_relative(src, base_url),
-            type: element['type']
-          }
-        end
-
-        def self.extract_iframe(iframe, base_url)
-          src = iframe['src'].to_s
-          return if src.empty?
-
-          abs_url = Url.from_relative(src, base_url)
-          {
-            url: abs_url,
-            type: Article::Enclosure.guess_content_type_from_url(abs_url, default: 'text/html')
-          }
-        end
-
-        def self.extract_a(link, base_url)
-          href = link['href'].to_s
-          return if href.empty?
-
-          abs_url = Url.from_relative(href, base_url)
-
-          if href.end_with?('.pdf')
-            { url: abs_url, type: Article::Enclosure.guess_content_type_from_url(abs_url) }
-          else
-            { url: abs_url, type: 'application/zip' }
-          end
-        end
-
-        private_class_method :extract_from_element, :extract_image, :extract_media, :extract_iframe, :extract_a
+        private_class_method :extract_from_element
       end
     end
   end

data/lib/html2rss/html/article_extractor/image_extractor.rb

@@ -32,27 +32,16 @@ module Html2rss
         # @see <https://developer.mozilla.org/en-US/docs/Learn/HTML/Multimedia_and_embedding/Responsive_images>
         # @see <https://developer.mozilla.org/en-US/docs/Web/HTML/Element/img#srcset>
         # @see <https://developer.mozilla.org/en-US/docs/Web/HTML/Element/picture>
-        def self.from_source(article_tag) # rubocop:disable Metrics/AbcSize
-          hash = article_tag.css('img[srcset], picture > source[srcset]').flat_map do |source|
-            source['srcset'].to_s.scan(/(\S+)\s+(\d+w|\d+h)[\s,]?/).map do |url, width|
-              next if url.nil? || url.start_with?('data:')
-
-              width_value = width.to_i.zero? ? 0 : width.scan(/\d+/).first.to_i
-
-              [width_value, url.strip]
-            end
-          end.compact.to_h
-
-          hash[hash.keys.max]
+        def self.from_source(article_tag)
+          srcsets = article_tag.css('img[srcset], picture > source[srcset]').map { |source| source['srcset'] }
+          ArticleRules::Image.largest_from_srcsets(srcsets)
         end
 
         # @param article_tag [Nokogiri::XML::Element] article container node
         # @return [String, nil] best style-based background image URL
         def self.from_style(article_tag)
-          article_tag.css('[style*="url"]')
-                     .filter_map { |tag| tag['style'][/url\(['"]?(.*?)['"]?\)/, 1] }
-                     .reject { |src| src.start_with?('data:') }
-                     .max_by(&:size)
+          styles = article_tag.css('[style*="url"]').map { |tag| tag['style'] }
+          ArticleRules::Image.best_from_styles(styles)
         end
       end
     end

data/lib/html2rss/html/article_rules/category.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    module ArticleRules
+      ##
+      # Shared category term vocabulary and text accumulation (DOM walk stays in adapters).
+      module Category
+        # Class/data attribute tokens that mark category metadata.
+        CATEGORY_TERMS = %w[
+          category categories tag tags topic topics section sections
+          label labels theme themes subject subjects
+        ].freeze
+
+        # Regex matching category-related attribute or class names.
+        CATEGORY_ATTR_PATTERN = /#{CATEGORY_TERMS.join('|')}/i
+
+        class << self
+          ##
+          # @param name [String]
+          # @return [Boolean]
+          def attr_name_match?(name)
+            name.to_s.match?(CATEGORY_ATTR_PATTERN)
+          end
+
+          ##
+          # @param class_attr [String, nil]
+          # @return [Boolean]
+          def class_match?(class_attr)
+            class_attr.to_s.match?(CATEGORY_ATTR_PATTERN)
+          end
+
+          ##
+          # @param categories [Set<String>]
+          # @param text [String, nil]
+          # @return [void]
+          def add_text!(categories, text)
+            value = text.to_s.strip
+            categories.add(value) unless value.empty?
+          end
+
+          ##
+          # @param categories [Set<String>]
+          # @param text [String, nil]
+          # @return [void]
+          def add_split_text!(categories, text)
+            return unless text
+
+            text.split(/\n+/).each { |line| add_text!(categories, line) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html/article_rules/date.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    module ArticleRules
+      ##
+      # DOM-agnostic datetime aggregation for article published_at.
+      module Date
+        class << self
+          ##
+          # @param datetime_strings [Array<String, nil>] raw datetime attribute values
+          # @return [DateTime, nil] earliest successfully parsed instant
+          def earliest(datetime_strings)
+            times = Array(datetime_strings).filter_map do |value|
+              DateTime.parse(value.to_s)
+            rescue ArgumentError, TypeError
+              nil
+            end
+            times.min
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html/article_rules/enclosure.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    module ArticleRules
+      ##
+      # DOM-agnostic enclosure URL/type normalization.
+      module Enclosure
+        # Href path suffixes treated as download/archive enclosure links.
+        ARCHIVE_HREF_SUFFIXES = %w[.pdf .zip .tar.gz .tgz].freeze
+
+        class << self
+          ##
+          # @param href [String]
+          # @return [Boolean]
+          def archive_href?(href)
+            ARCHIVE_HREF_SUFFIXES.any? { |suffix| href.to_s.end_with?(suffix) }
+          end
+
+          ##
+          # @param src [String]
+          # @param base_url [String, Html2rss::Url]
+          # @return [Hash{Symbol => Object}, nil]
+          def from_image(src, base_url)
+            return if src.to_s.empty?
+
+            abs = Url.from_relative(src, base_url)
+            { url: abs, type: Article::Enclosure.guess_content_type_from_url(abs, default: 'image/jpeg') }
+          end
+
+          ##
+          # @param src [String]
+          # @param type [String, nil]
+          # @param base_url [String, Html2rss::Url]
+          # @return [Hash{Symbol => Object}, nil]
+          def from_media(src, type, base_url)
+            return if src.to_s.empty?
+
+            { url: Url.from_relative(src, base_url), type: }
+          end
+
+          ##
+          # @param src [String]
+          # @param base_url [String, Html2rss::Url]
+          # @return [Hash{Symbol => Object}, nil]
+          def from_iframe(src, base_url)
+            return if src.to_s.empty?
+
+            abs = Url.from_relative(src, base_url)
+            { url: abs, type: Article::Enclosure.guess_content_type_from_url(abs, default: 'text/html') }
+          end
+
+          ##
+          # @param href [String]
+          # @param base_url [String, Html2rss::Url]
+          # @return [Hash{Symbol => Object}, nil]
+          def from_anchor(href, base_url)
+            return if href.to_s.empty?
+
+            abs = Url.from_relative(href, base_url)
+            type = if href.end_with?('.pdf')
+                     Article::Enclosure.guess_content_type_from_url(abs)
+                   else
+                     'application/zip'
+                   end
+            { url: abs, type: }
+          end
+        end
+      end
+    end
+  end
+end