html2rss 0.17.0 → 0.19.0

data/lib/html2rss/config/schema.rb

@@ -0,0 +1,240 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Config
+    ##
+    # Builds the exported configuration JSON Schema from the runtime validators.
+    module Schema
+      module_function
+
+      # Canonical filename for the exported config JSON schema artifact.
+      SCHEMA_FILENAME = 'html2rss-config.schema.json'
+
+      ##
+      # Returns the exported configuration JSON Schema.
+      #
+      # @return [Hash{String => Object}] JSON Schema represented as a Ruby hash
+      def json_schema
+        load_json_schema_extension!
+        Builder.call
+      end
+
+      ##
+      # Resolves the packaged schema path used by downstream tools.
+      #
+      # @return [String] absolute path to the packaged JSON schema file
+      def path
+        search_path = File.expand_path(__dir__)
+
+        loop do
+          candidate = File.join(search_path, 'schema', SCHEMA_FILENAME)
+          return candidate if File.exist?(candidate)
+
+          parent_path = File.dirname(search_path)
+          break if parent_path == search_path
+
+          search_path = parent_path
+        end
+
+        File.expand_path("../../../schema/#{SCHEMA_FILENAME}", __dir__)
+      end
+
+      # @return [void]
+      def load_json_schema_extension!
+        require 'dry/schema/extensions/json_schema'
+        Dry::Schema.load_extensions(:json_schema)
+      end
+
+      ##
+      # Orchestrates schema assembly from runtime validator contracts plus
+      # client-facing overlays.
+      class Builder
+        class << self
+          # @return [Hash{String => Object}] fully assembled JSON schema hash
+          def call
+            new.call
+          end
+        end
+
+        # @return [Hash{String => Object}] fully assembled JSON schema hash
+        def call
+          schema = validator_schema
+          apply_top_level(schema)
+          assign_properties(schema.fetch(:properties))
+          DeepStringifier.call(schema)
+        end
+
+        private
+
+        def validator_schema
+          Html2rss::Config::Validator.new.schema.json_schema(loose: true)
+        end
+
+        def apply_top_level(schema)
+          schema['$schema'] = 'https://json-schema.org/draft/2020-12/schema'
+          schema[:anyOf] = [
+            { 'required' => ['selectors'] },
+            { 'required' => ['auto_source'] }
+          ]
+        end
+
+        def assign_properties(properties)
+          properties.merge!(
+            strategy: Components.strategy,
+            headers: Components.headers,
+            stylesheets: Components.stylesheets,
+            auto_source: Components.auto_source,
+            selectors: Components.selectors
+          )
+          properties.delete(:dynamic_params_error)
+        end
+      end
+
+      ##
+      # Exposes schema fragments that populate the top-level configuration schema.
+      module Components
+        module_function
+
+        # @return [Hash{Symbol => Object}] schema fragment for strategy selection
+        def strategy
+          {
+            type: 'string',
+            not: { type: 'null' }
+          }
+        end
+
+        # @return [Hash{Symbol => Object}] schema fragment for headers
+        def headers
+          {
+            type: 'object',
+            description: 'HTTP headers applied to every request.',
+            additionalProperties: { type: 'string' }
+          }
+        end
+
+        # @return [Hash{Symbol => Object}] schema fragment for stylesheet definitions
+        def stylesheets
+          {
+            type: 'array',
+            description: 'Collection of stylesheets to attach to the RSS feed.',
+            items: Html2rss::Config::Validator::StylesheetConfig.json_schema(loose: true)
+          }
+        end
+
+        # @return [Hash{Symbol => Object}] schema fragment for auto_source configuration
+        def auto_source
+          schema = Html2rss::AutoSource::Config.json_schema(loose: true)
+          schema[:default] = DeepStringifier.call(Html2rss::AutoSource::DEFAULT_CONFIG)
+          schema
+        end
+
+        # @return [Hash{Symbol => Object}] schema fragment for selectors configuration
+        def selectors
+          Selectors.schema
+        end
+      end
+
+      ##
+      # Provides schema fragments that document selector configuration.
+      module Selectors
+        module_function
+
+        # Pattern used for dynamic selector keys excluding reserved selector names.
+        RESERVED_SELECTOR_PATTERN = '^(?!items$|enclosure$|guid$|categories$).+$'
+
+        # @return [Hash{Symbol => Object}] schema fragment for selectors root object
+        def schema
+          {
+            type: 'object',
+            description: 'Selectors used to extract article attributes.',
+            properties: selector_properties,
+            patternProperties: pattern_properties,
+            additionalProperties: true
+          }
+        end
+
+        # rubocop:disable Layout/LineLength
+        # @return [Hash{Symbol => Object}] schema map for reserved selector properties
+        def selector_properties
+          {
+            items: items_schema,
+            enclosure: enclosure_schema,
+            guid: reference_array('List of selector keys used to build the GUID. Each entry must reference a sibling selector key; runtime validation enforces those references.'),
+            categories: reference_array('List of selector keys whose values will be used as categories. Each entry must reference a sibling selector key; runtime validation enforces those references.')
+          }
+        end
+        # rubocop:enable Layout/LineLength
+
+        # @return [Hash{String => Object}] schema map for dynamic selector keys
+        def pattern_properties
+          { RESERVED_SELECTOR_PATTERN => dynamic_selector_schema }
+        end
+
+        # @return [Hash{Symbol => Object}] schema fragment for dynamic selector entries
+        def dynamic_selector_schema
+          Html2rss::Selectors::Config::Selector.new.schema.json_schema(loose: true).merge(
+            description: 'Dynamic selector definition keyed by attribute name.'
+          )
+        end
+
+        # @return [Hash{Symbol => Object}] schema fragment for `items` selector configuration
+        def items_schema
+          Html2rss::Selectors::Config::Items.new.schema.json_schema(loose: true).merge(
+            description: 'Defines the items selector and optional enhancement settings.'
+          )
+        end
+
+        # @return [Hash{Symbol => Object}] schema fragment for `enclosure` selector configuration
+        def enclosure_schema
+          Html2rss::Selectors::Config::Enclosure.new.schema.json_schema(loose: true).merge(
+            description: 'Describes enclosure extraction settings.'
+          )
+        end
+
+        # JSON Schema can enforce non-empty reference arrays, while runtime
+        # validation remains authoritative for checking that each entry points
+        # to an existing sibling selector key.
+        # @param description [String] human-readable description for the reference field
+        # @return [Hash{Symbol => Object}] JSON schema fragment for selector references
+        def reference_array(description)
+          {
+            type: 'array',
+            description:,
+            minItems: 1,
+            items: {
+              type: 'string',
+              description: 'Selector key defined elsewhere in this object.'
+            }
+          }
+        end
+      end
+
+      ##
+      # Converts nested hash keys to strings so the resulting schema serializes cleanly.
+      module DeepStringifier
+        module_function
+
+        # @param object [Hash, Array, Object] nested data to normalize
+        # @return [Hash, Array, Object] deep copy with stringified hash keys
+        def call(object)
+          case object
+          when Hash
+            stringify_hash(object)
+          when Array
+            object.map { |value| call(value) }
+          when Symbol
+            object.to_s
+          else
+            object
+          end
+        end
+
+        # @param object [Hash{Object => Object}] hash whose keys should become strings
+        # @return [Hash{String => Object}] hash with recursively normalized values
+        def stringify_hash(object)
+          object.to_h { |key, value| [key.to_s, call(value)] }
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/config/validator.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+require 'dry-validation'
+
+module Html2rss
+  class Config
+    # Validates the configuration hash using Dry::Validation.
+    # The configuration options adhere to the documented schema in README.md.
+    class Validator < Dry::Validation::Contract # rubocop:disable Metrics/ClassLength
+      # URI format used for channel URL validation.
+      URI_REGEXP = Url::URI_REGEXP
+      # Allowed stylesheet MIME types.
+      STYLESHEET_TYPES = RssBuilder::Stylesheet::TYPES
+      # Optional language/region format (`en` or `en-US`).
+      LANGUAGE_FORMAT_REGEX = /\A[a-z]{2}(-[A-Z]{2})?\z/
+      # Baseline strategy enum exported in static schema artifacts.
+      BASE_STRATEGY_OPTIONS = ([:auto] + Html2rss::RequestService.strategy_names.map(&:to_sym)).uniq.freeze
+
+      # Contract for the top-level `channel` section.
+      ChannelConfig = Dry::Schema.Params do
+        required(:url).filled(:string, format?: URI_REGEXP)
+        optional(:title).maybe(:string)
+        optional(:description).maybe(:string)
+        optional(:language).maybe(:string, format?: LANGUAGE_FORMAT_REGEX)
+        optional(:ttl).maybe(:integer, gt?: 0)
+        optional(:time_zone).maybe(:string)
+      end
+
+      # Contract for a stylesheet entry in `stylesheets`.
+      StylesheetConfig = Dry::Schema.Params do
+        required(:href).filled(:string)
+        required(:type).filled(:string, included_in?: STYLESHEET_TYPES)
+        optional(:media).maybe(:string)
+      end
+
+      # Contract for Browserless click-preload options.
+      BrowserlessPreloadClickSelectorConfig = Dry::Schema.Params do
+        required(:selector).filled(:string)
+        optional(:max_clicks).filled(:integer, gt?: 0)
+        optional(:wait_after_ms).filled(:integer, gteq?: 0)
+      end
+
+      # Contract for Browserless scroll-preload options.
+      BrowserlessPreloadScrollConfig = Dry::Schema.Params do
+        optional(:iterations).filled(:integer, gt?: 0)
+        optional(:wait_after_ms).filled(:integer, gteq?: 0)
+      end
+
+      # Contract for Browserless preload orchestration options.
+      BrowserlessPreloadConfig = Dry::Schema.Params do
+        optional(:wait_after_ms).filled(:integer, gteq?: 0)
+        optional(:click_selectors).array(BrowserlessPreloadClickSelectorConfig)
+        optional(:scroll_down).hash(BrowserlessPreloadScrollConfig)
+      end
+
+      # Contract for Browserless-specific request options.
+      BrowserlessRequestConfig = Dry::Schema.Params do
+        optional(:preload).hash(BrowserlessPreloadConfig)
+      end
+
+      # Contract for Botasaurus-specific request options.
+      BotasaurusRequestConfig = Dry::Schema.Params do
+        config.validate_keys = true
+
+        optional(:navigation_mode).filled(:string, included_in?: %w[auto get google_get google_get_bypass])
+        optional(:max_retries).filled(:integer, gteq?: 0, lteq?: 3)
+        optional(:wait_for_selector).maybe(:string)
+        optional(:wait_timeout_seconds).filled(:integer, gt?: 0)
+        optional(:block_images).filled(:bool)
+        optional(:block_images_and_css).filled(:bool)
+        optional(:wait_for_complete_page_load).filled(:bool)
+        optional(:headless).filled(:bool)
+        optional(:proxy).filled(:string)
+        optional(:user_agent).filled(:string)
+        optional(:window_size).value(:array, min_size?: 2, max_size?: 2).each(:integer, gt?: 0)
+        optional(:lang).filled(:string)
+      end
+
+      # Contract for the top-level `request` section.
+      RequestConfig = Dry::Schema.Params do
+        optional(:max_redirects).filled(:integer, gteq?: 0)
+        optional(:max_requests).filled(:integer, gt?: 0)
+        optional(:browserless).hash(BrowserlessRequestConfig)
+        optional(:botasaurus).hash(BotasaurusRequestConfig)
+      end
+
+      params do
+        optional(:strategy).filled(:symbol)
+        required(:channel).hash(ChannelConfig)
+        optional(:headers).hash
+        optional(:stylesheets).array(StylesheetConfig)
+        optional(:auto_source).hash(AutoSource::Config)
+        optional(:selectors).hash
+        optional(:dynamic_params_error).maybe(:string)
+        optional(:request).hash(RequestConfig)
+      end
+
+      rule(:headers) do
+        value&.each do |key, header_value|
+          unless header_value.is_a?(String)
+            key([:headers, key]).failure("must be a String, but got #{header_value.class}")
+          end
+        end
+      end
+
+      rule(:dynamic_params_error) do
+        base.failure(value) if value
+      end
+
+      rule(:strategy) do
+        next if value.nil?
+        next if value == :auto || Html2rss::RequestService.strategy_registered?(value)
+
+        key.failure("must be one of: #{BASE_STRATEGY_OPTIONS.join(', ')}")
+      end
+
+      # Ensure at least one of :selectors or :auto_source is present.
+      rule(:selectors, :auto_source) do
+        unless values.key?(:selectors) || values.key?(:auto_source)
+          base.failure("Configuration must include at least 'selectors' or 'auto_source'")
+        end
+      end
+
+      rule(:selectors) do
+        next unless value
+
+        errors = Html2rss::Selectors::Config.call(value).errors
+        errors.each { |error| key(:selectors).failure(error.text) } unless errors.empty?
+      end
+
+      # URL validation delegated to Url class
+      rule(:channel) do
+        next unless values[:channel]&.key?(:url)
+
+        url_string = values[:channel][:url]
+        next if url_string.nil? || url_string.empty?
+
+        begin
+          Html2rss::Url.for_channel(url_string)
+        rescue ArgumentError => error
+          key(%i[channel url]).failure(error.message)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/config.rb

@@ -1,82 +1,139 @@
 # frozen_string_literal: true
 
-require 'forwardable'
+require 'json'
+require 'yaml'
 
 module Html2rss
   ##
-  # The Config class abstracts from the config data structure and
-  # provides default values.
+  # The provided configuration is used to generate the RSS feed.
+  # This class provides methods to load and process configuration from a YAML file,
+  # supporting both single and multiple feed configurations.
+  #
+  # Configuration is validated during initialization.
   class Config
-    extend Forwardable
+    # Raised when a configuration hash fails runtime validation.
+    class InvalidConfig < Html2rss::Error; end
+    extend ClassMethods
 
     ##
-    # The Error class to be thrown when a feed config requires params, but none
-    # were passed to Config.
-    class ParamsMissing < Html2rss::Error; end
-
-    ##
-    # Thrown when the feed config does not contain a value at `:channel`.
-    class ChannelMissing < Html2rss::Error; end
-
-    def_delegator :@channel, :author
-    def_delegator :@channel, :ttl
-    def_delegator :@channel, :title
-    def_delegator :@channel, :language
-    def_delegator :@channel, :description
-    def_delegator :@channel, :url
-    def_delegator :@channel, :url, :link
-    def_delegator :@channel, :time_zone
-    def_delegator :@channel, :json?
-    def_delegator :@channel, :strategy
-
-    def_delegator :@selectors, :item_selector_names
-    def_delegator :@selectors, :selector?
-    def_delegator :@selectors, :category_selector_names
-    def_delegator :@selectors, :guid_selector_names
-    def_delegator :@selectors, :items_order
-    def_delegator :@selectors, :selector_string
-
-    ##
-    # Initializes the Config object with feed configuration, global settings, and parameters.
+    # Initializes the configuration object.
     #
-    # @param feed_config [Hash<Symbol, Object>] The configuration hash containing `:channel` and `:selectors`.
-    # @param global [Hash<Symbol, Object>] Global settings hash.
-    # @param params [Hash<Symbol, String>] Parameters hash.
-    def initialize(feed_config, global = {}, params = {})
-      channel_config = feed_config[:channel]
-      raise ChannelMissing, 'Channel configuration is missing in feed_config' unless channel_config
-
-      @channel = Channel.new(channel_config, params:)
-      @selectors = Selectors.new(feed_config[:selectors])
-      @global = global
+    # Processes deprecated attributes, applies default values, and validates the configuration.
+    #
+    # @param config [Hash{Symbol => Object}] the configuration hash.
+    # @raise [InvalidConfig] if the configuration fails validation.
+    def initialize(config)
+      @request_controls = RequestControls.from_config(config)
+      prepared_config = Preparer.new.call(config)
+      validated_config = validated_config_for(prepared_config)
+
+      @config = validated_config.freeze
+      @request_controls = request_controls.with_effective_values(
+        strategy: validated_config[:strategy],
+        max_redirects: validated_config.dig(:request, :max_redirects),
+        max_requests: validated_config.dig(:request, :max_requests)
+      )
     end
 
+    # @return [Symbol, nil] selected request strategy
+    def strategy = request_controls.strategy
+    # @return [Integer, nil] configured redirect budget
+    def max_redirects = request_controls.max_redirects
+    # @return [Integer, nil] configured request budget
+    def max_requests = request_controls.max_requests
+    # @return [Array<Hash>] stylesheet definitions
+    def stylesheets = config[:stylesheets]
+
     ##
-    # Retrieves selector attributes merged with channel attributes.
-    #
-    # @param name [Symbol] Selector name.
-    # @return [Hash<Symbol, Object>] Merged attributes hash.
-    def selector_attributes_with_channel(name)
-      @selectors.selector(name).to_h.merge(channel: @channel)
+    # @return [Boolean] whether max_requests was explicitly configured by the caller
+    def explicit_max_requests?
+      request_controls.explicit?(:max_requests)
     end
 
     ##
-    # Retrieves headers merged from global settings and channel headers.
-    #
-    # @return [Hash] Merged headers hash.
-    def headers
-      @global.fetch(:headers, {}).merge(@channel.headers)
+    # @return [Html2rss::RequestControls] request controls with provenance
+    attr_reader :request_controls
+
+    # @return [Hash{String => String}] normalized HTTP headers
+    def headers = config[:headers]
+    # @return [Hash{Symbol => Object}] channel configuration
+    def channel = config[:channel]
+    # @return [String] source channel URL
+    def url = config.dig(:channel, :url)
+    # @return [String, nil] configured channel time zone
+    def time_zone = config.dig(:channel, :time_zone)
+
+    # @return [Hash{Symbol => Object}] request envelope configuration
+    def request = config[:request]
+
+    # @return [Hash{Symbol => Object}, nil] selectors configuration
+    def selectors = config[:selectors]
+    # @return [Hash{Symbol => Object}, nil] auto-source configuration
+    def auto_source = config[:auto_source]
+
+    private
+
+    attr_reader :config
+
+    # Normalizes raw config input before validation.
+    class Preparer
+      ##
+      # @param config [Hash{Symbol => Object}] raw config input
+      # @return [Hash{Symbol => Object}] config with defaults and deprecations applied
+      def call(config)
+        config = config.dup if config.frozen?
+
+        config = handle_deprecated_channel_attributes(config)
+        config = apply_default_config(config)
+        config = apply_default_selectors_config(config) if config[:selectors]
+        config = apply_default_auto_source_config(config) if config[:auto_source]
+
+        config
+      end
+
+      private
+
+      def handle_deprecated_channel_attributes(config)
+        { strategy: Config.default_strategy_name, headers: {} }.each_pair do |key, default_value|
+          if !config[key] && (value = config.dig(:channel, key))
+            Log.warn("The `channel.#{key}` key is deprecated. Please move the definition of `#{key}` to the top level.")
+            config[key] = value
+          end
+
+          config[key] ||= default_value
+        end
+
+        config
+      end
+
+      def apply_default_config(config)
+        HashUtil.deep_merge(Config.default_config, config)
+      end
+
+      def apply_default_selectors_config(config)
+        HashUtil.deep_merge({ selectors: Selectors::DEFAULT_CONFIG }, config)
+      end
+
+      def apply_default_auto_source_config(config)
+        HashUtil.deep_merge({ auto_source: Html2rss::AutoSource::DEFAULT_CONFIG }, config)
+      end
     end
 
-    ##
-    # Retrieves stylesheets from global settings.
-    #
-    # @return [Array<Stylesheet>] Array of Stylesheet structs.
-    def stylesheets
-      @global.fetch(:stylesheets, []).map { |attributes| Html2rss::RssBuilder::Stylesheet.new(**attributes) }
+    def validated_config_for(config)
+      validator = Validator.new.call(config)
+
+      raise InvalidConfig, "Invalid configuration: #{validator.errors.to_h}" unless validator.success?
+
+      normalized_headers(validator.to_h)
     end
 
-    # Provides read-only access to the channel object.
-    attr_reader :channel
+    def normalized_headers(validated_config)
+      validated_config[:headers] = RequestHeaders.normalize(
+        validated_config[:headers],
+        channel_language: validated_config.dig(:channel, :language),
+        url: validated_config.dig(:channel, :url)
+      )
+      validated_config
+    end
   end
 end

data/lib/html2rss/error.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Html2rss
+  # The Html2rss::Error base class.
+  class Error < StandardError; end
+
+  # Raised when auto fallback exhausts all concrete tiers and extractors find no feed items.
+  class NoFeedItemsExtracted < Error
+    ##
+    # @param attempts [Array<Hash{Symbol => Object}>] tier attempt diagnostics
+    def initialize(attempts:)
+      @attempts = attempts
+      super(build_message)
+    end
+
+    # @return [Array<Hash{Symbol => Object}>] tier attempt diagnostics
+    attr_reader :attempts
+
+    private
+
+    def build_message
+      summaries = attempts.map do |attempt|
+        details = attempt[:items_count].nil? ? "#{attempt[:error_class]} error" : "#{attempt[:items_count]} items"
+        "#{attempt[:strategy]} (#{details})"
+      end.join(', ')
+
+      "No feed items extracted after auto fallback across strategies: #{summaries}. " \
+        'Try a more specific listing URL or provide explicit selectors.'
+    end
+  end
+end