html2rss 0.18.0 → 0.19.0

data/lib/html2rss/config/request_headers.rb

@@ -6,6 +6,7 @@ module Html2rss
     # Normalizes HTTP headers for outgoing requests.
     # Ensures a browser-like baseline while respecting caller overrides.
     class RequestHeaders
+      # Browser-like default `Accept` header value.
       DEFAULT_ACCEPT = %w[
         text/html
         application/xhtml+xml
@@ -16,6 +17,7 @@ module Html2rss
         */*;q=0.8
       ].join(',')
 
+      # Browser-like default `User-Agent` header value.
       DEFAULT_USER_AGENT = [
         'Mozilla/5.0 (Windows NT 10.0; Win64; x64)',
         'AppleWebKit/537.36 (KHTML, like Gecko)',
@@ -23,6 +25,7 @@ module Html2rss
         'Safari/537.36'
       ].join(' ')
 
+      # Baseline browser-like header set used for outbound requests.
       DEFAULT_HEADERS = {
         'Accept' => DEFAULT_ACCEPT,
         'Cache-Control' => 'max-age=0',
@@ -37,7 +40,7 @@ module Html2rss
 
       class << self
         ##
-        # @return [Hash<String, String>] the unmodified default header set
+        # @return [Hash{String => String}] the unmodified default header set
         def browser_defaults
           DEFAULT_HEADERS.dup
         end
@@ -48,12 +51,15 @@ module Html2rss
         # @param headers [Hash, nil] caller provided headers
         # @param channel_language [String, nil] language defined on the channel
         # @param url [String] request URL used to infer the Host header
-        # @return [Hash<String, String>] normalized HTTP headers
+        # @return [Hash{String => String}] normalized HTTP headers
         def normalize(headers, channel_language:, url:)
           new(headers || {}, channel_language:, url:).to_h
         end
       end
 
+      # @param headers [Hash{String, Symbol => String}] caller-provided headers
+      # @param channel_language [String, nil] channel language hint for Accept-Language
+      # @param url [String, Html2rss::Url, nil] request URL used to infer Host
       def initialize(headers, channel_language:, url:)
         @headers = headers
         @channel_language = channel_language
@@ -61,7 +67,7 @@ module Html2rss
       end
 
       ##
-      # @return [Hash<String, String>] normalized HTTP headers
+      # @return [Hash{String => String}] normalized HTTP headers
       def to_h
         defaults = DEFAULT_HEADERS.dup
         normalized = normalize_custom_headers(headers)

data/lib/html2rss/config/schema.rb

@@ -7,12 +7,13 @@ module Html2rss
     module Schema
       module_function
 
+      # Canonical filename for the exported config JSON schema artifact.
       SCHEMA_FILENAME = 'html2rss-config.schema.json'
 
       ##
       # Returns the exported configuration JSON Schema.
       #
-      # @return [Hash<String, Object>] JSON Schema represented as a Ruby hash
+      # @return [Hash{String => Object}] JSON Schema represented as a Ruby hash
       def json_schema
         load_json_schema_extension!
         Builder.call
@@ -38,6 +39,7 @@ module Html2rss
         File.expand_path("../../../schema/#{SCHEMA_FILENAME}", __dir__)
       end
 
+      # @return [void]
       def load_json_schema_extension!
         require 'dry/schema/extensions/json_schema'
         Dry::Schema.load_extensions(:json_schema)
@@ -48,11 +50,13 @@ module Html2rss
       # client-facing overlays.
       class Builder
         class << self
+          # @return [Hash{String => Object}] fully assembled JSON schema hash
           def call
             new.call
           end
         end
 
+        # @return [Hash{String => Object}] fully assembled JSON schema hash
         def call
           schema = validator_schema
           apply_top_level(schema)
@@ -76,6 +80,7 @@ module Html2rss
 
         def assign_properties(properties)
           properties.merge!(
+            strategy: Components.strategy,
             headers: Components.headers,
             stylesheets: Components.stylesheets,
             auto_source: Components.auto_source,
@@ -90,6 +95,15 @@ module Html2rss
       module Components
         module_function
 
+        # @return [Hash{Symbol => Object}] schema fragment for strategy selection
+        def strategy
+          {
+            type: 'string',
+            not: { type: 'null' }
+          }
+        end
+
+        # @return [Hash{Symbol => Object}] schema fragment for headers
         def headers
           {
             type: 'object',
@@ -98,6 +112,7 @@ module Html2rss
           }
         end
 
+        # @return [Hash{Symbol => Object}] schema fragment for stylesheet definitions
         def stylesheets
           {
             type: 'array',
@@ -106,12 +121,14 @@ module Html2rss
           }
         end
 
+        # @return [Hash{Symbol => Object}] schema fragment for auto_source configuration
         def auto_source
           schema = Html2rss::AutoSource::Config.json_schema(loose: true)
           schema[:default] = DeepStringifier.call(Html2rss::AutoSource::DEFAULT_CONFIG)
           schema
         end
 
+        # @return [Hash{Symbol => Object}] schema fragment for selectors configuration
         def selectors
           Selectors.schema
         end
@@ -122,8 +139,10 @@ module Html2rss
       module Selectors
         module_function
 
+        # Pattern used for dynamic selector keys excluding reserved selector names.
         RESERVED_SELECTOR_PATTERN = '^(?!items$|enclosure$|guid$|categories$).+$'
 
+        # @return [Hash{Symbol => Object}] schema fragment for selectors root object
         def schema
           {
             type: 'object',
@@ -135,6 +154,7 @@ module Html2rss
         end
 
         # rubocop:disable Layout/LineLength
+        # @return [Hash{Symbol => Object}] schema map for reserved selector properties
         def selector_properties
           {
             items: items_schema,
@@ -145,22 +165,26 @@ module Html2rss
         end
         # rubocop:enable Layout/LineLength
 
+        # @return [Hash{String => Object}] schema map for dynamic selector keys
         def pattern_properties
           { RESERVED_SELECTOR_PATTERN => dynamic_selector_schema }
         end
 
+        # @return [Hash{Symbol => Object}] schema fragment for dynamic selector entries
         def dynamic_selector_schema
           Html2rss::Selectors::Config::Selector.new.schema.json_schema(loose: true).merge(
             description: 'Dynamic selector definition keyed by attribute name.'
           )
         end
 
+        # @return [Hash{Symbol => Object}] schema fragment for `items` selector configuration
         def items_schema
           Html2rss::Selectors::Config::Items.new.schema.json_schema(loose: true).merge(
             description: 'Defines the items selector and optional enhancement settings.'
           )
         end
 
+        # @return [Hash{Symbol => Object}] schema fragment for `enclosure` selector configuration
         def enclosure_schema
           Html2rss::Selectors::Config::Enclosure.new.schema.json_schema(loose: true).merge(
             description: 'Describes enclosure extraction settings.'
@@ -170,6 +194,8 @@ module Html2rss
         # JSON Schema can enforce non-empty reference arrays, while runtime
         # validation remains authoritative for checking that each entry points
         # to an existing sibling selector key.
+        # @param description [String] human-readable description for the reference field
+        # @return [Hash{Symbol => Object}] JSON schema fragment for selector references
         def reference_array(description)
           {
             type: 'array',
@@ -188,17 +214,23 @@ module Html2rss
       module DeepStringifier
         module_function
 
+        # @param object [Hash, Array, Object] nested data to normalize
+        # @return [Hash, Array, Object] deep copy with stringified hash keys
         def call(object)
           case object
           when Hash
             stringify_hash(object)
           when Array
             object.map { |value| call(value) }
+          when Symbol
+            object.to_s
           else
             object
           end
         end
 
+        # @param object [Hash{Object => Object}] hash whose keys should become strings
+        # @return [Hash{String => Object}] hash with recursively normalized values
         def stringify_hash(object)
           object.to_h { |key, value| [key.to_s, call(value)] }
         end

data/lib/html2rss/config/validator.rb

@@ -6,11 +6,17 @@ module Html2rss
   class Config
     # Validates the configuration hash using Dry::Validation.
     # The configuration options adhere to the documented schema in README.md.
-    class Validator < Dry::Validation::Contract
+    class Validator < Dry::Validation::Contract # rubocop:disable Metrics/ClassLength
+      # URI format used for channel URL validation.
       URI_REGEXP = Url::URI_REGEXP
+      # Allowed stylesheet MIME types.
       STYLESHEET_TYPES = RssBuilder::Stylesheet::TYPES
+      # Optional language/region format (`en` or `en-US`).
       LANGUAGE_FORMAT_REGEX = /\A[a-z]{2}(-[A-Z]{2})?\z/
+      # Baseline strategy enum exported in static schema artifacts.
+      BASE_STRATEGY_OPTIONS = ([:auto] + Html2rss::RequestService.strategy_names.map(&:to_sym)).uniq.freeze
 
+      # Contract for the top-level `channel` section.
       ChannelConfig = Dry::Schema.Params do
         required(:url).filled(:string, format?: URI_REGEXP)
         optional(:title).maybe(:string)
@@ -20,41 +26,66 @@ module Html2rss
         optional(:time_zone).maybe(:string)
       end
 
+      # Contract for a stylesheet entry in `stylesheets`.
       StylesheetConfig = Dry::Schema.Params do
         required(:href).filled(:string)
         required(:type).filled(:string, included_in?: STYLESHEET_TYPES)
         optional(:media).maybe(:string)
       end
 
+      # Contract for Browserless click-preload options.
       BrowserlessPreloadClickSelectorConfig = Dry::Schema.Params do
         required(:selector).filled(:string)
         optional(:max_clicks).filled(:integer, gt?: 0)
         optional(:wait_after_ms).filled(:integer, gteq?: 0)
       end
 
+      # Contract for Browserless scroll-preload options.
       BrowserlessPreloadScrollConfig = Dry::Schema.Params do
         optional(:iterations).filled(:integer, gt?: 0)
         optional(:wait_after_ms).filled(:integer, gteq?: 0)
       end
 
+      # Contract for Browserless preload orchestration options.
       BrowserlessPreloadConfig = Dry::Schema.Params do
         optional(:wait_after_ms).filled(:integer, gteq?: 0)
         optional(:click_selectors).array(BrowserlessPreloadClickSelectorConfig)
         optional(:scroll_down).hash(BrowserlessPreloadScrollConfig)
       end
 
+      # Contract for Browserless-specific request options.
       BrowserlessRequestConfig = Dry::Schema.Params do
         optional(:preload).hash(BrowserlessPreloadConfig)
       end
 
+      # Contract for Botasaurus-specific request options.
+      BotasaurusRequestConfig = Dry::Schema.Params do
+        config.validate_keys = true
+
+        optional(:navigation_mode).filled(:string, included_in?: %w[auto get google_get google_get_bypass])
+        optional(:max_retries).filled(:integer, gteq?: 0, lteq?: 3)
+        optional(:wait_for_selector).maybe(:string)
+        optional(:wait_timeout_seconds).filled(:integer, gt?: 0)
+        optional(:block_images).filled(:bool)
+        optional(:block_images_and_css).filled(:bool)
+        optional(:wait_for_complete_page_load).filled(:bool)
+        optional(:headless).filled(:bool)
+        optional(:proxy).filled(:string)
+        optional(:user_agent).filled(:string)
+        optional(:window_size).value(:array, min_size?: 2, max_size?: 2).each(:integer, gt?: 0)
+        optional(:lang).filled(:string)
+      end
+
+      # Contract for the top-level `request` section.
       RequestConfig = Dry::Schema.Params do
         optional(:max_redirects).filled(:integer, gteq?: 0)
         optional(:max_requests).filled(:integer, gt?: 0)
         optional(:browserless).hash(BrowserlessRequestConfig)
+        optional(:botasaurus).hash(BotasaurusRequestConfig)
       end
 
       params do
-        required(:strategy).filled(:symbol)
+        optional(:strategy).filled(:symbol)
         required(:channel).hash(ChannelConfig)
         optional(:headers).hash
         optional(:stylesheets).array(StylesheetConfig)
@@ -76,6 +107,13 @@ module Html2rss
         base.failure(value) if value
       end
 
+      rule(:strategy) do
+        next if value.nil?
+        next if value == :auto || Html2rss::RequestService.strategy_registered?(value)
+
+        key.failure("must be one of: #{BASE_STRATEGY_OPTIONS.join(', ')}")
+      end
+
       # Ensure at least one of :selectors or :auto_source is present.
       rule(:selectors, :auto_source) do
         unless values.key?(:selectors) || values.key?(:auto_source)

data/lib/html2rss/config.rb

@@ -11,6 +11,7 @@ module Html2rss
   #
   # Configuration is validated during initialization.
   class Config
+    # Raised when a configuration hash fails runtime validation.
     class InvalidConfig < Html2rss::Error; end
     extend ClassMethods
 
@@ -19,7 +20,7 @@ module Html2rss
     #
     # Processes deprecated attributes, applies default values, and validates the configuration.
     #
-    # @param config [Hash<Symbol, Object>] the configuration hash.
+    # @param config [Hash{Symbol => Object}] the configuration hash.
     # @raise [InvalidConfig] if the configuration fails validation.
     def initialize(config)
       @request_controls = RequestControls.from_config(config)
@@ -34,9 +35,13 @@ module Html2rss
       )
     end
 
+    # @return [Symbol, nil] selected request strategy
     def strategy = request_controls.strategy
+    # @return [Integer, nil] configured redirect budget
     def max_redirects = request_controls.max_redirects
+    # @return [Integer, nil] configured request budget
     def max_requests = request_controls.max_requests
+    # @return [Array<Hash>] stylesheet definitions
     def stylesheets = config[:stylesheets]
 
     ##
@@ -49,14 +54,21 @@ module Html2rss
     # @return [Html2rss::RequestControls] request controls with provenance
     attr_reader :request_controls
 
+    # @return [Hash{String => String}] normalized HTTP headers
     def headers = config[:headers]
+    # @return [Hash{Symbol => Object}] channel configuration
     def channel = config[:channel]
+    # @return [String] source channel URL
     def url = config.dig(:channel, :url)
+    # @return [String, nil] configured channel time zone
     def time_zone = config.dig(:channel, :time_zone)
 
+    # @return [Hash{Symbol => Object}] request envelope configuration
     def request = config[:request]
 
+    # @return [Hash{Symbol => Object}, nil] selectors configuration
     def selectors = config[:selectors]
+    # @return [Hash{Symbol => Object}, nil] auto-source configuration
     def auto_source = config[:auto_source]
 
     private
@@ -66,8 +78,8 @@ module Html2rss
     # Normalizes raw config input before validation.
     class Preparer
       ##
-      # @param config [Hash<Symbol, Object>] raw config input
-      # @return [Hash<Symbol, Object>] config with defaults and deprecations applied
+      # @param config [Hash{Symbol => Object}] raw config input
+      # @return [Hash{Symbol => Object}] config with defaults and deprecations applied
       def call(config)
         config = config.dup if config.frozen?
 
@@ -82,7 +94,7 @@ module Html2rss
       private
 
       def handle_deprecated_channel_attributes(config)
-        { strategy: RequestService.default_strategy_name, headers: {} }.each_pair do |key, default_value|
+        { strategy: Config.default_strategy_name, headers: {} }.each_pair do |key, default_value|
           if !config[key] && (value = config.dig(:channel, key))
             Log.warn("The `channel.#{key}` key is deprecated. Please move the definition of `#{key}` to the top level.")
             config[key] = value
@@ -95,21 +107,15 @@ module Html2rss
       end
 
       def apply_default_config(config)
-        deep_merge(Config.default_config, config)
+        HashUtil.deep_merge(Config.default_config, config)
       end
 
       def apply_default_selectors_config(config)
-        deep_merge({ selectors: Selectors::DEFAULT_CONFIG }, config)
+        HashUtil.deep_merge({ selectors: Selectors::DEFAULT_CONFIG }, config)
       end
 
       def apply_default_auto_source_config(config)
-        deep_merge({ auto_source: Html2rss::AutoSource::DEFAULT_CONFIG }, config)
-      end
-
-      def deep_merge(base_config, override_config)
-        base_config.merge(override_config) do |_key, oldval, newval|
-          oldval.is_a?(Hash) && newval.is_a?(Hash) ? deep_merge(oldval, newval) : newval
-        end
+        HashUtil.deep_merge({ auto_source: Html2rss::AutoSource::DEFAULT_CONFIG }, config)
       end
     end
 

data/lib/html2rss/error.rb

@@ -3,4 +3,29 @@
 module Html2rss
   # The Html2rss::Error base class.
   class Error < StandardError; end
+
+  # Raised when auto fallback exhausts all concrete tiers and extractors find no feed items.
+  class NoFeedItemsExtracted < Error
+    ##
+    # @param attempts [Array<Hash{Symbol => Object}>] tier attempt diagnostics
+    def initialize(attempts:)
+      @attempts = attempts
+      super(build_message)
+    end
+
+    # @return [Array<Hash{Symbol => Object}>] tier attempt diagnostics
+    attr_reader :attempts
+
+    private
+
+    def build_message
+      summaries = attempts.map do |attempt|
+        details = attempt[:items_count].nil? ? "#{attempt[:error_class]} error" : "#{attempt[:items_count]} items"
+        "#{attempt[:strategy]} (#{details})"
+      end.join(', ')
+
+      "No feed items extracted after auto fallback across strategies: #{summaries}. " \
+        'Try a more specific listing URL or provide explicit selectors.'
+    end
+  end
 end

data/lib/html2rss/feed_pipeline/auto_fallback.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+##
+# The Html2rss namespace.
+module Html2rss
+  ##
+  # Coordinates feed generation pipeline stages.
+  class FeedPipeline
+    # Retries feed extraction across concrete request strategies for :auto mode.
+    class AutoFallback
+      # Ordered list of concrete request strategies attempted by auto mode.
+      CHAIN = %i[faraday botasaurus browserless].freeze
+
+      # Error classes that should abort auto fallback immediately.
+      NON_FALLBACK_ERRORS = [
+        RequestService::UnknownStrategy,
+        RequestService::InvalidUrl,
+        RequestService::UnsupportedUrlScheme,
+        RequestService::UnsupportedResponseContentType,
+        RequestService::RequestBudgetExceeded,
+        RequestService::PrivateNetworkDenied,
+        RequestService::CrossOriginFollowUpDenied,
+        RequestService::ResponseTooLarge,
+        RequestService::BrowserlessConfigurationError
+      ].freeze
+
+      ##
+      # @param strategies [Array<Symbol>] ordered concrete strategies for fallback
+      # @param budget [RequestService::Budget] shared request budget across retries
+      # @param session_for [Proc] request session factory proc
+      # @param articles_for [Proc] article extraction proc
+      # @return [void]
+      def initialize(strategies:, budget:, session_for:, articles_for:)
+        @strategies = strategies
+        @budget = budget
+        @session_for = session_for
+        @articles_for = articles_for
+      end
+
+      ##
+      # @return [Hash{Symbol => Object}] pipeline state containing :response and :articles
+      def call
+        state, attempts = run_attempts
+        return state if state
+
+        finalize_failure(attempts:)
+      end
+
+      private
+
+      attr_reader :strategies, :budget, :session_for, :articles_for
+
+      def run_attempts
+        state = { result: nil, attempts: [] }
+        strategies.each_with_index do |strategy, index|
+          run_attempt_for(strategy:, next_strategy: strategies[index + 1], state:)
+          break if state.fetch(:result)
+        end
+        [state.fetch(:result), state.fetch(:attempts)]
+      end
+
+      def run_attempt_for(strategy:, next_strategy:, state:)
+        result, attempts = attempt(
+          strategy:,
+          next_strategy:,
+          state: { attempts: state.fetch(:attempts) }
+        )
+        state[:result] = result
+        state[:attempts] = attempts
+      end
+
+      def attempt(strategy:, next_strategy:, state:)
+        request_session = session_for.call(strategy:, budget:)
+        response, state = fetch_response(
+          request_session:,
+          strategy:,
+          next_strategy:,
+          state:
+        )
+        return [nil, state.fetch(:attempts)] unless response
+
+        process_response(response:, strategy:, next_strategy:, request_session:, state:)
+      end
+
+      def fetch_response(request_session:, strategy:, next_strategy:, state:)
+        [request_session.fetch_initial_response, state]
+      rescue *NON_FALLBACK_ERRORS
+        raise
+      rescue StandardError => error
+        state[:attempts] << { strategy:, items_count: nil, error_class: error.class.name }
+        log_warn_fallback_error(strategy:, next_strategy:, error:) if next_strategy
+        Log.debug("#{self.class}: strategy=#{strategy} error=#{error.class}: #{error.message}")
+        [nil, state]
+      end
+
+      def process_response(response:, strategy:, next_strategy:, request_session:, state:)
+        articles = articles_for.call(response:, request_session:)
+        items_count = articles.size
+        state[:attempts] << { strategy:, items_count:, error_class: nil }
+        Log.debug("#{self.class}: strategy=#{strategy} items=#{items_count}")
+        return success_state(response:, strategy:, articles:, state:) if items_count.positive?
+
+        log_info_fallback_zero_items(strategy:, next_strategy:) if next_strategy
+        [nil, state.fetch(:attempts)]
+      end
+
+      def success_state(response:, strategy:, articles:, state:)
+        if state.fetch(:attempts).size > 1
+          Log.info("#{self.class}: auto selected strategy=#{strategy} after attempts=#{state.fetch(:attempts).size}")
+        end
+        [{ response:, articles: }, state.fetch(:attempts)]
+      end
+
+      def finalize_failure(attempts:)
+        raise NoFeedItemsExtracted.new(attempts:)
+      end
+
+      def log_warn_fallback_error(strategy:, next_strategy:, error:)
+        Log.warn("#{self.class}: auto fallback #{strategy} -> #{next_strategy} after error=#{error.class}")
+      end
+
+      def log_info_fallback_zero_items(strategy:, next_strategy:)
+        Log.info("#{self.class}: auto fallback #{strategy} -> #{next_strategy} after zero extracted items")
+      end
+    end
+  end
+end