html2rss 0.18.0 → 0.19.1

data/lib/html2rss/feed_pipeline.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Builds feeds from validated config through request, extraction, and rendering stages.
+  class FeedPipeline
+    ##
+    # @param raw_config [Hash{Symbol => Object}] user-provided feed config
+    def initialize(raw_config)
+      @raw_config = raw_config
+    end
+
+    ##
+    # @return [RSS::Rss] generated RSS feed
+    def to_rss
+      run do |response:, config:, articles:|
+        channel = RssBuilder::Channel.new(response, overrides: config.channel)
+        RssBuilder.new(channel:, articles:, stylesheets: config.stylesheets).call
+      end
+    end
+
+    ##
+    # @return [Hash] generated JSONFeed 1.1 payload
+    def to_json_feed
+      run do |response:, config:, articles:|
+        channel = RssBuilder::Channel.new(response, overrides: config.channel)
+        JsonFeedBuilder.new(channel:, articles:).call
+      end
+    end
+
+    private
+
+    attr_reader :raw_config
+
+    def run
+      config = Config.from_hash(raw_config, params: raw_config[:params])
+      state = pipeline_state_for(config)
+      yield response: state.fetch(:response), config:, articles: state.fetch(:articles)
+    end
+
+    def pipeline_state_for(config)
+      if config.strategy == :auto
+        run_auto_pipeline(config)
+      else
+        run_pipeline_for_strategy(config, strategy: config.strategy)
+      end
+    end
+
+    def run_pipeline_for_strategy(config, strategy:, budget: nil)
+      request_session = request_session_for(config, strategy:, budget:)
+      response = request_session.fetch_initial_response
+      articles = deduplicated_articles(response:, config:, request_session:)
+      { response:, articles: }
+    end
+
+    def request_session_for(config, strategy:, budget: nil)
+      RequestSession.from_runtime_input(runtime_input_for(config, strategy:), budget:)
+    end
+
+    def runtime_input_for(config, strategy:)
+      RequestSession::RuntimeInput.new(
+        url: config.url,
+        headers: config.headers,
+        request: config.request,
+        strategy:,
+        request_policy: RequestSession::RuntimePolicy.from_config(config)
+      )
+    end
+
+    def deduplicated_articles(response:, config:, request_session:)
+      Articles::Deduplicator.new(
+        collect_articles(response:, config:, request_session:)
+      ).call
+    end
+
+    def run_auto_pipeline(config)
+      auto_fallback_for(config).call
+    end
+
+    def auto_fallback_for(config)
+      AutoFallback.new(
+        strategies: AutoFallback::CHAIN,
+        budget: auto_pipeline_budget(config),
+        session_for: lambda do |strategy:, budget:|
+          request_session_for(config, strategy:, budget:)
+        end,
+        articles_for: lambda do |response:, request_session:|
+          deduplicated_articles(response:, config:, request_session:)
+        end
+      )
+    end
+
+    def auto_pipeline_budget(config)
+      max_requests = RequestSession::RuntimePolicy.from_config(config).max_requests
+      RequestService::Budget.new(max_requests:)
+    end
+
+    def collect_articles(response:, config:, request_session:)
+      selector_articles(response:, config:, request_session:) +
+        auto_source_articles(response:, config:, request_session:)
+    end
+
+    def selector_articles(response:, config:, request_session:) # rubocop:disable Metrics/MethodLength
+      return [] unless (selectors = config.selectors)
+
+      page_responses = if (max_pages = selectors.dig(:items, :pagination, :max_pages))
+                         RequestSession::RelNextPager.new(
+                           session: request_session,
+                           initial_response: response,
+                           max_pages:
+                         ).to_a
+                       else
+                         [response]
+                       end
+
+      page_responses.flat_map do |page_response|
+        Selectors.new(page_response, selectors:, time_zone: config.time_zone).articles
+      end
+    end
+
+    def auto_source_articles(response:, config:, request_session:)
+      return [] unless (auto_source = config.auto_source)
+
+      AutoSource.new(response, auto_source, request_session:).articles
+    end
+  end
+end

data/lib/html2rss/hash_util.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Html2rss
+  # Shared helpers for hash normalization and structural operations.
+  module HashUtil
+    module_function
+
+    # Deeply duplicates nested arrays and hashes.
+    #
+    # @param object [Object] nested value from configuration or runtime state
+    # @return [Object] deep duplicated object
+    def deep_dup(object)
+      case object
+      in Hash
+        object.transform_values { deep_dup(_1) }
+      in Array
+        object.map { deep_dup(_1) }
+      else
+        object.dup rescue StandardError # rubocop:disable Style/RescueModifier
+      end
+    end
+
+    # Deeply merges nested hashes while replacing non-hash values from override.
+    #
+    # @param base [Hash] base hash
+    # @param override [Hash] override hash
+    # @return [Hash] merged hash
+    def deep_merge(base, override)
+      base.merge(override) do |_key, old_val, new_val|
+        case [old_val, new_val]
+        in [Hash, Hash]
+          deep_merge(old_val, new_val)
+        else
+          new_val
+        end
+      end
+    end
+
+    # Converts string-keyed hashes to symbol-keyed hashes recursively.
+    #
+    # @param object [Object] value to normalize
+    # @param context [String] error context
+    # @return [Object] normalized value
+    def deep_symbolize_keys(object, context: 'hash')
+      case object
+      in Hash
+        object.each_with_object({}) do |(k, v), memo|
+          memo[symbol_key(k, context:)] = deep_symbolize_keys(v, context:)
+        end
+      in Array
+        object.map { deep_symbolize_keys(_1, context:) }
+      else
+        object
+      end
+    end
+
+    # Validates that hash keys are symbols.
+    #
+    # @param value [Object] candidate hash container whose keys must be symbols
+    # @param context [String] error context
+    # @param deep [Boolean] whether nested hashes should also be validated
+    # @return [void]
+    def assert_symbol_keys!(value, context: 'hash', deep: true)
+      return unless value in Hash
+
+      unless value.each_key.all?(Symbol)
+        invalid_key = value.keys.find { _1.class != Symbol }
+        raise ArgumentError, "#{context} must use symbol keys (found #{invalid_key.inspect})"
+      end
+
+      value.each_value { assert_symbol_keys!(_1, context:, deep:) } if deep
+    end
+
+    # Validates that hash keys are strings.
+    #
+    # @param value [Object] candidate hash container whose keys must be strings
+    # @param context [String] error context
+    # @param deep [Boolean] whether nested hashes should also be validated
+    # @return [void]
+    def assert_string_keys!(value, context: 'hash', deep: true)
+      return unless value in Hash
+
+      unless value.each_key.all?(String)
+        invalid_key = value.keys.find { _1.class != String }
+        raise ArgumentError, "#{context} must use string keys (found #{invalid_key.inspect})"
+      end
+
+      value.each_value { assert_string_keys!(_1, context:, deep:) } if deep
+    end
+
+    def symbol_key(key, context:)
+      case key
+      in Symbol then key
+      in String then key.to_sym
+      else
+        raise ArgumentError, "#{context} must use string or symbol keys (found #{key.inspect})"
+      end
+    end
+    private_class_method :symbol_key
+  end
+end

data/lib/html2rss/html_extractor/date_extractor.rb

@@ -4,6 +4,7 @@ module Html2rss
   class HtmlExtractor
     # Extracts the earliest date from an article_tag.
     class DateExtractor
+      # @param article_tag [Nokogiri::XML::Element] article container node
       # @return [DateTime, nil]
       def self.call(article_tag)
         times = article_tag.css('[datetime]').filter_map do |tag|

data/lib/html2rss/html_extractor/enclosure_extractor.rb

@@ -5,6 +5,9 @@ module Html2rss
     ##
     # Extracts enclosures from HTML tags using various strategies.
     class EnclosureExtractor
+      # @param article_tag [Nokogiri::XML::Element] article container node
+      # @param base_url [String, Html2rss::Url] base URL for relative enclosure links
+      # @return [Array<Hash{Symbol => Object}>] normalized enclosure hashes
       def self.call(article_tag, base_url)
         [
           Extractors::Image,
@@ -16,10 +19,14 @@ module Html2rss
       end
     end
 
+    # Extraction strategies for enclosure-like media/link tags.
     module Extractors
       # Extracts image enclosures from HTML tags.
       # Finds all image sources and returns them in a format suitable for RSS.
       class Image
+        # @param article_tag [Nokogiri::XML::Element] article container node
+        # @param base_url [String, Html2rss::Url] base URL for relative image sources
+        # @return [Array<Hash{Symbol => Object}>] image enclosure hashes
         def self.call(article_tag, base_url:)
           article_tag.css('img[src]:not([src^="data"])').filter_map do |img|
             src = img['src'].to_s
@@ -36,6 +43,9 @@ module Html2rss
 
       # Extracts media enclosures (video/audio) from HTML tags.
       class Media
+        # @param article_tag [Nokogiri::XML::Element] article container node
+        # @param base_url [String, Html2rss::Url] base URL for relative media sources
+        # @return [Array<Hash{Symbol => Object}>] media enclosure hashes
         def self.call(article_tag, base_url:)
           article_tag.css('video source[src], audio source[src], audio[src]').filter_map do |element|
             src = element['src'].to_s
@@ -51,6 +61,9 @@ module Html2rss
 
       # Extracts PDF enclosures from HTML tags.
       class Pdf
+        # @param article_tag [Nokogiri::XML::Element] article container node
+        # @param base_url [String, Html2rss::Url] base URL for relative PDF links
+        # @return [Array<Hash{Symbol => Object}>] PDF enclosure hashes
         def self.call(article_tag, base_url:)
           article_tag.css('a[href$=".pdf"]').filter_map do |link|
             href = link['href'].to_s
@@ -67,6 +80,9 @@ module Html2rss
 
       # Extracts iframe enclosures from HTML tags.
       class Iframe
+        # @param article_tag [Nokogiri::XML::Element] article container node
+        # @param base_url [String, Html2rss::Url] base URL for relative iframe links
+        # @return [Array<Hash{Symbol => Object}>] iframe enclosure hashes
         def self.call(article_tag, base_url:)
           article_tag.css('iframe[src]').filter_map do |iframe|
             src = iframe['src']
@@ -83,6 +99,9 @@ module Html2rss
 
       # Extracts archive enclosures (zip, tar.gz, tgz) from HTML tags.
       class Archive
+        # @param article_tag [Nokogiri::XML::Element] article container node
+        # @param base_url [String, Html2rss::Url] base URL for relative archive links
+        # @return [Array<Hash{Symbol => Object}>] archive enclosure hashes
         def self.call(article_tag, base_url:)
           article_tag.css('a[href$=".zip"], a[href$=".tar.gz"], a[href$=".tgz"]').filter_map do |link|
             href = link['href'].to_s

data/lib/html2rss/html_extractor/image_extractor.rb

@@ -5,6 +5,9 @@ module Html2rss
     ##
     # Image is responsible for extracting image URLs the article_tag.
     class ImageExtractor
+      # @param article_tag [Nokogiri::XML::Element] article container node
+      # @param base_url [String, Html2rss::Url] base URL for relative image URLs
+      # @return [Html2rss::Url, nil] best candidate image URL
       def self.call(article_tag, base_url:)
         img_src = from_source(article_tag) ||
                   from_img(article_tag) ||
@@ -13,6 +16,8 @@ module Html2rss
         Url.from_relative(img_src, base_url) if img_src
       end
 
+      # @param article_tag [Nokogiri::XML::Element] article container node
+      # @return [String, nil] src attribute from first matching image tag
       def self.from_img(article_tag)
         article_tag.at_css('img[src]:not([src^="data"])')&.[]('src')
       end
@@ -21,6 +26,8 @@ module Html2rss
       # Extracts the largest image source from the srcset attribute
       # of an img tag or a source tag inside a picture tag.
       #
+      # @param article_tag [Nokogiri::XML::Element] article container node
+      # @return [String, nil] largest srcset URL candidate
       # @see <https://developer.mozilla.org/en-US/docs/Learn/HTML/Multimedia_and_embedding/Responsive_images>
       # @see <https://developer.mozilla.org/en-US/docs/Web/HTML/Element/img#srcset>
       # @see <https://developer.mozilla.org/en-US/docs/Web/HTML/Element/picture>
@@ -38,6 +45,8 @@ module Html2rss
         hash[hash.keys.max]
       end
 
+      # @param article_tag [Nokogiri::XML::Element] article container node
+      # @return [String, nil] best style-based background image URL
       def self.from_style(article_tag)
         article_tag.css('[style*="url"]')
                    .filter_map { |tag| tag['style'][/url\(['"]?(.*?)['"]?\)/, 1] }

data/lib/html2rss/html_extractor.rb

@@ -5,10 +5,14 @@ module Html2rss
   # HtmlExtractor is responsible for extracting details (headline, url, images, etc.)
   # from an article_tag.
   class HtmlExtractor
+    # Tags ignored when extracting visible text content from article containers.
     INVISIBLE_CONTENT_TAGS = %w[svg script noscript style template].to_set.freeze
+    # Heading tags used to prioritize title extraction.
     HEADING_TAGS = %w[h1 h2 h3 h4 h5 h6].freeze
+    # Selector used to derive non-headline description nodes.
     NON_HEADLINE_SELECTOR = (HEADING_TAGS.map { |tag| ":not(#{tag})" } + INVISIBLE_CONTENT_TAGS.to_a).freeze
 
+    # Anchor selector used to identify the canonical article link element.
     MAIN_ANCHOR_SELECTOR = begin
       buf = +'a[href]:not([href=""])'
       %w[# javascript: mailto: tel: file:// sms: data:].each do |prefix|
@@ -56,6 +60,7 @@ module Html2rss
       @selected_anchor = selected_anchor
     end
 
+    # @return [Hash{Symbol => Object}] extracted article attributes
     def call
       {
         title: extract_title,

data/lib/html2rss/html_navigator.rb

@@ -23,6 +23,10 @@ module Html2rss
       ##
       # Think of it as `css_upwards` method.
       # It searches for the closest parent that matches the given selector.
+      #
+      # @param current_tag [Nokogiri::XML::Node, nil] starting node
+      # @param selector [String] CSS selector to search upwards for
+      # @return [Nokogiri::XML::Node, nil] first matching node in upward traversal
       def find_closest_selector_upwards(current_tag, selector)
         while current_tag
           found = current_tag.at_css(selector)
@@ -36,6 +40,10 @@ module Html2rss
 
       ##
       # Searches for the closest parent that matches the given tag name.
+      #
+      # @param current_tag [Nokogiri::XML::Node] starting node
+      # @param tag_name [String] tag name to find in ancestors
+      # @return [Nokogiri::XML::Node, nil] matching ancestor node
       def find_tag_in_ancestors(current_tag, tag_name)
         return current_tag if current_tag.name == tag_name
 

data/lib/html2rss/json_feed_builder.rb

@@ -6,6 +6,7 @@ module Html2rss
   #
   # @see https://www.jsonfeed.org/version/1.1/
   class JsonFeedBuilder
+    # Official JSON Feed 1.1 schema version URL.
     VERSION_URL = 'https://jsonfeed.org/version/1.1'
 
     ##

data/lib/html2rss/rendering/audio_renderer.rb

@@ -6,15 +6,20 @@ module Html2rss
   module Rendering
     # Renders an HTML <audio> tag from a URL and type.
     class AudioRenderer
+      # @param url [String, Html2rss::Url] media URL for the audio source
+      # @param type [String] MIME type for the audio source
       def initialize(url:, type:)
         @url = url
         @type = type
       end
 
+      # @return [String] HTML audio snippet for article rendering
       def to_html
-        %(<audio controls preload="none" referrerpolicy="no-referrer" crossorigin="anonymous">
-            <source src="#{escaped_url}" type="#{escaped_type}">
-          </audio>)
+        [
+          '<audio controls preload="none" referrerpolicy="no-referrer" crossorigin="anonymous">',
+          %(<source src="#{escaped_url}" type="#{escaped_type}">),
+          '</audio>'
+        ].join
       end
 
       private

data/lib/html2rss/rendering/description_builder.rb

@@ -16,7 +16,6 @@ module Html2rss
     #     image: "https://example.com/image.jpg"
     #   )
     #   description = builder.call
-    #
     class DescriptionBuilder
       # Removes the specified pattern from the beginning of the text
       # within a given range if the pattern occurs before the range's end.

data/lib/html2rss/rendering/image_renderer.rb

@@ -6,23 +6,33 @@ module Html2rss
   module Rendering
     # Renders an HTML <img> tag from a URL and title.
     class ImageRenderer
+      # @param url [String, Html2rss::Url] image URL for the src attribute
+      # @param title [String, nil] title/alt text for the image
       def initialize(url:, title:)
         @url = url
         @title = title
       end
 
+      # @return [String] HTML image snippet for article rendering
       def to_html
-        %(<img src="#{@url}"
-              alt="#{escaped_title}"
-              title="#{escaped_title}"
-              loading="lazy"
-              referrerpolicy="no-referrer"
-              decoding="async"
-              crossorigin="anonymous">).delete("\n").gsub(/\s+/, ' ')
+        attributes = [
+          %(src="#{escaped_url}"),
+          %(alt="#{escaped_title}"),
+          %(title="#{escaped_title}"),
+          'loading="lazy"',
+          'referrerpolicy="no-referrer"',
+          'decoding="async"',
+          'crossorigin="anonymous"'
+        ]
+        "<img #{attributes.join(' ')}>"
       end
 
       private
 
+      def escaped_url
+        CGI.escapeHTML(@url.to_s)
+      end
+
       def escaped_title
         CGI.escapeHTML(@title.to_s)
       end

data/lib/html2rss/rendering/media_renderer.rb

@@ -16,6 +16,10 @@ module Html2rss
       end
 
       # @private
+      # @param type [String, nil] enclosure MIME type
+      # @param url [String, Html2rss::Url] enclosure URL
+      # @param title [String, nil] title used by image renderer
+      # @return [ImageRenderer, VideoRenderer, AudioRenderer, PdfRenderer, nil]
       def self.create_renderer_for_type(type, url:, title:)
         case type
         when %r{^image/}

data/lib/html2rss/rendering/pdf_renderer.rb

@@ -6,16 +6,22 @@ module Html2rss
   module Rendering
     # Renders an HTML <iframe> for PDF documents.
     class PdfRenderer
+      # @param url [String, Html2rss::Url] PDF URL rendered in the iframe
       def initialize(url:)
         @url = url
       end
 
+      # @return [String] HTML iframe snippet for PDF rendering
       def to_html
-        %(<iframe src="#{escaped_url}" width="100%" height="75vh"
-                  sandbox=""
-                  referrerpolicy="no-referrer"
-                  loading="lazy">
-           </iframe>)
+        attributes = [
+          %(src="#{escaped_url}"),
+          'width="100%"',
+          'height="75vh"',
+          'sandbox=""',
+          'referrerpolicy="no-referrer"',
+          'loading="lazy"'
+        ]
+        "<iframe #{attributes.join(' ')}></iframe>"
       end
 
       private

data/lib/html2rss/rendering/video_renderer.rb

@@ -6,15 +6,20 @@ module Html2rss
   module Rendering
     # Renders an HTML <video> tag from a URL and type.
     class VideoRenderer
+      # @param url [String, Html2rss::Url] media URL for the video source
+      # @param type [String] MIME type for the video source
       def initialize(url:, type:)
         @url = url
         @type = type
       end
 
+      # @return [String] HTML video snippet for article rendering
       def to_html
-        %(<video controls preload="none" referrerpolicy="no-referrer" crossorigin="anonymous" playsinline>
-            <source src="#{escaped_url}" type="#{escaped_type}">
-          </video>)
+        [
+          '<video controls preload="none" referrerpolicy="no-referrer" crossorigin="anonymous" playsinline>',
+          %(<source src="#{escaped_url}" type="#{escaped_type}">),
+          '</video>'
+        ].join
       end
 
       private

data/lib/html2rss/rendering.rb

@@ -5,8 +5,17 @@ module Html2rss
   # images, audio, video, or embedded documents for feed descriptions.
   #
   # @example
-  #   Html2rss::Rendering::ImageRenderer.new(...).to_html
-  #   Html2rss::Rendering::MediaRenderer.for(...)
+  #   Html2rss::Rendering::ImageRenderer.new(
+  #     url: "https://example.com/image.jpg",
+  #     title: "Example"
+  #   ).to_html
+  #
+  # @example
+  #   Html2rss::Rendering::MediaRenderer.for(
+  #     enclosure: nil,
+  #     image: "https://example.com/image.jpg",
+  #     title: "Example"
+  #   )
   #
   # @see Html2rss::Rendering::DescriptionBuilder
   module Rendering

data/lib/html2rss/request_controls.rb

@@ -4,15 +4,20 @@ module Html2rss
   ##
   # Tracks runtime request controls together with whether each value was explicitly set.
   class RequestControls
+    # Request-control keys accepted at the top level of feed config.
     TOP_LEVEL_KEYS = %i[strategy].freeze
+    # Request-control keys accepted under the nested `request` config.
     REQUEST_KEYS = %i[max_redirects max_requests].freeze
 
     ##
-    # @param config [Hash<Symbol, Object>, Hash<String, Object>] raw config input
+    # @param config [Hash{Symbol => Object}] raw config input
     # @return [RequestControls] request controls extracted from the config hash
     def self.from_config(config)
+      HashUtil.assert_symbol_keys!(config, context: 'config', deep: false)
+      HashUtil.assert_symbol_keys!(config[:request], context: 'config[:request]') if config[:request].is_a?(Hash)
+
       new(
-        strategy: value_for(config, :strategy),
+        strategy: config[:strategy],
         max_redirects: request_value_for(config, :max_redirects),
         max_requests: request_value_for(config, :max_requests),
         explicit_keys: explicit_keys_for(config)
@@ -20,33 +25,23 @@ module Html2rss
     end
 
     def self.explicit_keys_for(config)
-      TOP_LEVEL_KEYS.filter { top_level_key?(config, _1) } +
+      TOP_LEVEL_KEYS.filter { config.key?(_1) } +
         REQUEST_KEYS.filter { request_key?(config, _1) }
     end
 
-    def self.value_for(config, key)
-      return config[key] if config.key?(key)
-      return config[key.to_s] if config.key?(key.to_s)
-
-      nil
-    end
-
     def self.request_value_for(config, key)
-      request_config = value_for(config, :request)
+      request_config = config[:request]
       return nil unless request_config.is_a?(Hash)
 
-      value_for(request_config, key)
-    end
-
-    def self.top_level_key?(config, key)
-      config.key?(key) || config.key?(key.to_s)
+      request_config[key]
     end
 
     def self.request_key?(config, key)
-      request_config = value_for(config, :request)
-      request_config.is_a?(Hash) && top_level_key?(request_config, key)
+      request_config = config[:request]
+      request_config.is_a?(Hash) && request_config.key?(key)
     end
-    private_class_method :explicit_keys_for, :request_value_for, :top_level_key?, :request_key?, :value_for
+
+    private_class_method :explicit_keys_for, :request_value_for, :request_key?
 
     ##
     # @param strategy [Symbol, nil] effective request strategy
@@ -97,8 +92,8 @@ module Html2rss
     ##
     # Applies only explicitly set controls to the provided config hash.
     #
-    # @param config [Hash<Symbol, Object>] mutable config hash
-    # @return [Hash<Symbol, Object>] the same hash with explicit controls written
+    # @param config [Hash{Symbol => Object}] mutable config hash
+    # @return [Hash{Symbol => Object}] the same hash with explicit controls written
     def apply_to(config)
       config[:strategy] = strategy if explicit?(:strategy)
       apply_request_value(config, :max_redirects, max_redirects)