html2rss 0.30.0 → 0.31.0

data/lib/html2rss/config/request_headers.rb

@@ -33,18 +33,17 @@ module Html2rss
       }.freeze
 
       # Hop-by-hop headers forbidden in HTTP/2 requests (RFC 7540 §8.1.2.2 / RFC 9113 §8.2.1).
-      FORBIDDEN_H2_HEADERS = %w[connection keep-alive proxy-connection transfer-encoding upgrade].to_set.freeze
+      FORBIDDEN_H2_HEADERS = Set['connection', 'keep-alive', 'proxy-connection', 'transfer-encoding', 'upgrade'].freeze
 
       class << self
         ##
-        # :reek:ManualDispatch
         # :reek:TooManyStatements
         #
         # @return [Hash{String => String}] the default header set merged with global defaults
         def browser_defaults
           defaults = DEFAULT_HEADERS.dup
           global_headers = Html2rss.defaults.headers
-          global_headers = global_headers.call if global_headers.respond_to?(:call)
+          global_headers = global_headers.call if global_headers.is_a?(Proc)
 
           if global_headers.is_a?(Hash)
             global_headers.each do |key, value|

data/lib/html2rss/config/schema.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'json'
+
 module Html2rss
   class Config
     ##
@@ -19,6 +21,15 @@ module Html2rss
         Builder.call
       end
 
+      ##
+      # Returns the exported JSON Schema as JSON.
+      #
+      # @param pretty [Boolean] whether to pretty-print the JSON output
+      # @return [String] serialized JSON Schema
+      def json_schema_json(pretty: true)
+        pretty ? JSON.pretty_generate(json_schema) : JSON.generate(json_schema)
+      end
+
       ##
       # Resolves the packaged schema path used by downstream tools.
       #
@@ -107,12 +118,16 @@ module Html2rss
         # @param registry [Hash{Symbol => Class}]
         # @return [Hash{String => Hash}]
         def catalog_from_registry(registry)
-          registry.keys.sort.to_h { |name| [name.to_s, registry.fetch(name).schema_doc] }
+          registry.keys.sort.to_h { |name| [name.to_s, registry.fetch(name).schema_export] }
         end
 
         # rubocop:disable-next Metrics/MethodLength, Metrics/AbcSize, Layout/LineLength
         def overlay
           items_schema = Html2rss::Config::SelectorsValidator::Items.new.schema.json_schema(loose: true)
+          items_schema[:required] = Array(items_schema[:required]).reject { |key| key.to_sym == :selector }
+          items_schema[:properties][:selector] = items_schema.fetch(:properties).fetch(:selector).merge(
+            default: Html2rss::Selectors::DEFAULT_ITEMS_SELECTOR
+          )
           items_schema[:properties][:enhance] = items_schema.fetch(:properties).fetch(:enhance).merge(
             description: 'List-card enrichment: run Html::ArticleExtractor on each matched item node ' \
                          'to fill missing fields from the card HTML.'

data/lib/html2rss/config/selectors_validator.rb

@@ -6,9 +6,22 @@ module Html2rss
   class Config
     ##
     # Validates the configuration hash for :selectors.
-    class SelectorsValidator < Dry::Validation::Contract
-      # Required wrapper key used to validate dynamic selector names.
-      NESTING_KEY = :dynamic_keys_workaround
+    module SelectorsValidator
+      ##
+      # One selector validation error.
+      Error = Data.define(:path, :text)
+
+      Result = Data.define(:errors)
+
+      ##
+      # Validation result containing collected selector errors.
+      class Result
+        # @return [Boolean]
+        def success? = errors.empty?
+
+        # @return [Boolean]
+        def failure? = !success?
+      end
 
       ##
       # Validates the configuration of the :items selector
@@ -75,9 +88,6 @@ module Html2rss
       ##
       # Validates the configuration of a single selector.
       class Selector < Dry::Validation::Contract
-        # Extractor Options members supplied at runtime, not from feed YAML.
-        RUNTIME_EXTRACTOR_FIELDS = %i[selector channel].to_set.freeze
-
         params do
           optional(:selector)
           optional(:extractor).filled(:string)
@@ -96,11 +106,11 @@ module Html2rss
           klass = Selectors::Extractors::NAME_TO_CLASS[value.to_sym]
           next key(:extractor).failure("unknown extractor: #{value}") unless klass
 
-          klass::Options.members.each do |field|
-            next if RUNTIME_EXTRACTOR_FIELDS.include?(field)
-            next if values[field]
+          Selectors::OptionSpec.for(klass).each do |spec|
+            next unless spec.required
+            next if values[spec.name]
 
-            key(field).failure("`#{field}` is required for extractor `#{value}`")
+            key(spec.name).failure("`#{spec.name}` is required for extractor `#{value}`")
           end
         end
 
@@ -110,49 +120,44 @@ module Html2rss
 
           klass = Selectors::PostProcessors::NAME_TO_CLASS[name.to_sym]
           next key(:post_process).failure("Unknown post_processor `name`: #{name}") unless klass
-          next unless klass.const_defined?(:Options)
 
           post_process_option_type_errors(klass, value).each do |field, message|
             key(field).failure(message)
           end
+
+          pattern_error = gsub_pattern_error(klass, value)
+          key(:pattern).failure(pattern_error) if pattern_error
         end
 
         private
 
         def post_process_option_type_errors(klass, value)
-          required_option_type_errors(klass, value) + optional_option_type_errors(klass, value)
-        end
-
-        def required_option_type_errors(klass, value)
-          option_types = klass.const_defined?(:OPTION_TYPES) ? klass::OPTION_TYPES : {}
-
-          klass::Options.members.filter_map do |field|
-            expected = option_types.fetch(field, String)
-            next if value[field].is_a?(expected)
-
-            [field, option_type_failure(field, expected)]
+          Selectors::OptionSpec.for(klass).filter_map do |spec|
+            actual = value[spec.name]
+            if actual.nil?
+              [spec.name, spec.error_message(optional: false)] if spec.required
+            elsif !spec.valid_type?(actual)
+              [spec.name, spec.error_message(optional: !spec.required)]
+            end
           end
         end
 
-        def optional_option_type_errors(klass, value)
-          return [] unless klass.const_defined?(:OPTIONAL_OPTION_TYPES)
+        # Same compile as Gsub#call. ArgumentError is the bound; Gsub maps
+        # parser errors onto that path so admit and execute still share it.
+        #
+        # @param klass [Class]
+        # @param value [Hash]
+        # @return [String, nil]
+        def gsub_pattern_error(klass, value)
+          return unless klass == Selectors::PostProcessors::Gsub
 
-          klass::OPTIONAL_OPTION_TYPES.filter_map do |field, expected|
-            actual = value[field]
-            next if actual.nil? || actual.is_a?(expected)
-
-            [field, option_type_failure(field, expected, optional: true)]
-          end
-        end
+          pattern = value[:pattern]
+          return unless pattern.is_a?(String)
 
-        def option_type_failure(field, expected, optional: false)
-          label = {
-            Integer => 'an integer',
-            String => 'a string',
-            Hash => 'a hash'
-          }.fetch(expected) { "a #{expected}" }
-          suffix = optional ? ' or omitted' : ''
-          "`#{field}` must be #{label}#{suffix}"
+          Selectors::PostProcessors::Gsub.compiled_pattern(pattern)
+          nil
+        rescue ArgumentError => error
+          error.message
         end
       end
 
@@ -164,47 +169,64 @@ module Html2rss
         end
       end
 
-      params do
-        required(NESTING_KEY).hash
-      end
+      class << self
+        ##
+        # Shortcut to validate the config.
+        # @param config [Hash] the configuration hash to validate
+        # @return [Result]
+        def call(config)
+          errors = []
+          return Result.new(errors:) unless config.is_a?(Hash)
+
+          config.each_pair do |selector_key, selector|
+            validate_entry(selector_key, selector, config, errors)
+          end
+
+          Result.new(errors:)
+        end
 
-      rule(NESTING_KEY) do
-        value.each_pair do |selector_key, selector|
+        private
+
+        def validate_entry(selector_key, selector, config, errors)
           case selector_key.to_sym
           when Selectors::ITEMS_SELECTOR_KEY
-            Items.new.call(selector).errors.each { |error| key(selector_key).failure(error.text) }
+            collect_contract_errors(Items, selector, selector_key, errors)
           when :enclosure
-            Enclosure.new.call(selector).errors.each { |error| key(selector_key).failure(error.text) }
+            collect_contract_errors(Enclosure, selector, selector_key, errors)
           when :guid, :categories
-            unless selector.is_a?(Array)
-              key(selector_key).failure("`#{selector_key}` must be an array")
-              next
-            end
+            validate_array_selector(selector_key, selector, config, errors)
+          else
+            collect_contract_errors(Selector, selector, selector_key, errors)
+          end
+        end
 
-            key(selector_key).failure("`#{selector_key}` must contain at least one element") if selector.empty?
+        def collect_contract_errors(contract_class, selector, selector_key, errors)
+          contract_class.new.call(selector).errors.each do |error|
+            errors << Error.new(path: [selector_key, *error.path], text: error.text)
+          end
+        end
 
-            selector.each do |name|
-              next if values[NESTING_KEY].key?(name.to_sym)
+        def validate_array_selector(selector_key, selector, config, errors)
+          msg = array_selector_shape_error(selector_key, selector)
+          return errors << Error.new(path: [selector_key], text: msg) if msg
 
-              key(selector_key).failure("`#{selector_key}` references unspecified `#{name}`")
-            end
-          else
-            # From here on, the selector is found under its "dynamic" selector_key
-            Selector.new.call(selector).errors.each { |error| key(selector_key).failure(error.text) }
-          end
+          check_unspecified_references(selector_key, selector, config, errors)
         end
-      end
 
-      ##
-      # Shortcut to validate the config.
-      # @param config [Hash] the configuration hash to validate
-      # @return [Dry::Validation::Result] the result of the validation
-      def self.call(config)
-        # dry-validation/schema does not support "Dynamic Keys" yet: https://github.com/dry-rb/dry-schema/issues/37
-        # But :selectors contains mostly "dynamic" keys, as the user defines them to extract article attributes.
-        # --> Validate the dynamic keys manually.
-        # To be able to specify a `rule`, nest the config under NESTING_KEY and mark that as `required`.
-        new.call(NESTING_KEY => config)
+        def array_selector_shape_error(key, selector)
+          return "`#{key}` must be an array" unless selector.is_a?(Array)
+          return "`#{key}` must contain at least one element" if selector.empty?
+
+          nil
+        end
+
+        def check_unspecified_references(selector_key, selector, config, errors)
+          selector.each do |name|
+            next if config.key?(name.to_sym) || config.key?(name.to_s)
+
+            errors << Error.new(path: [selector_key], text: "`#{selector_key}` references unspecified `#{name}`")
+          end
+        end
       end
     end
   end

data/lib/html2rss/config/validation_issue.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Config
+    ValidationIssue = Data.define(:path, :code, :message, :expected, :actual)
+
+    ##
+    # One structured validation finding for GUI/MCP/CLI/Test.
+    #
+    # +path+ is JSON-pointer-friendly segments. +code+ is closed ({CODES}).
+    # +expected+ / +actual+ are JSON-ish or +nil+ (never invented).
+    class ValidationIssue
+      # Closed set of issue codes (mapper owns Dry predicate → code).
+      CODES = Set[
+        :missing_key,
+        :type_mismatch,
+        :unknown_value,
+        :invalid_value,
+        :constraint,
+        :parse
+      ].freeze
+
+      ##
+      # @param path [Array<Symbol, Integer, String>]
+      # @param code [Symbol]
+      # @param message [String]
+      # @param expected [Hash, String, Numeric, true, false, nil]
+      # @param actual [Hash, String, Numeric, true, false, nil]
+      def initialize(path:, code:, message:, expected: nil, actual: nil)
+        raise ArgumentError, "unknown ValidationIssue code: #{code.inspect}" unless CODES.include?(code)
+
+        super(
+          path: Array(path).freeze,
+          code:,
+          message: message.to_s,
+          expected:,
+          actual:
+        )
+      end
+
+      ##
+      # Wire shape for a single issue.
+      #
+      # @return [Hash{Symbol => Object}]
+      def to_h
+        { path:, code:, message:, expected:, actual: }
+      end
+
+      ##
+      # Single renderer for CLI / exceptions (path [code] message).
+      #
+      # @return [String]
+      def to_s
+        path_s = path.join('.')
+        path_s = '(root)' if path_s.empty?
+        "#{path_s} [#{code}] #{message}"
+      end
+    end
+  end
+end

data/lib/html2rss/config/validation_report.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Config
+    ValidationReport = Data.define(:issues)
+
+    ##
+    # Public result of config validation. Dry::Validation::Result never crosses this boundary.
+    # +success+ is derived from +issues+ — invalid states like +success: true+ with issues cannot exist.
+    class ValidationReport
+      ##
+      # @return [Html2rss::Config::ValidationReport]
+      def self.ok
+        new(issues: [].freeze)
+      end
+
+      ##
+      # @param issues [Array<Html2rss::Config::ValidationIssue>]
+      # @return [Html2rss::Config::ValidationReport]
+      def self.failure(issues)
+        new(issues: Array(issues).freeze)
+      end
+
+      ##
+      # @param issues [Array<Html2rss::Config::ValidationIssue>]
+      def initialize(issues:)
+        super(issues: Array(issues).freeze)
+      end
+
+      # @return [Boolean]
+      def success? = issues.empty?
+
+      # @return [Boolean]
+      def failure? = !success?
+
+      ##
+      # Wire shape: +{ success:, issues: [...] }+ only (MCP contract v3).
+      #
+      # @return [Hash{Symbol => Object}]
+      def to_h
+        { success: success?, issues: issues.map(&:to_h) }
+      end
+
+      ##
+      # Human-readable summary for CLI / exception messages.
+      #
+      # @return [String]
+      def to_s
+        return 'valid' if success?
+
+        issues.join('; ')
+      end
+    end
+  end
+end

data/lib/html2rss/config/validator.rb

@@ -16,10 +16,10 @@ module Html2rss
       # Baseline strategy-plan enum (:auto plus concrete RequestService strategies).
       BASE_STRATEGY_OPTIONS = Html2rss::FeedPipeline::StrategyPlan.accepted_names.freeze
       # Controlled vocabulary for catalog-only `directory.topics` (not RSS channel fields).
-      DIRECTORY_TOPICS = %w[
-        sports energy tech science news entertainment jobs finance
-        security travel environment consumer civic product research
-        health culture education rights transport
+      DIRECTORY_TOPICS = Set[
+        'sports', 'energy', 'tech', 'science', 'news', 'entertainment', 'jobs', 'finance',
+        'security', 'travel', 'environment', 'consumer', 'civic', 'product', 'research',
+        'health', 'culture', 'education', 'rights', 'transport'
       ].freeze
 
       # Contract for the top-level `channel` section.
@@ -134,8 +134,12 @@ module Html2rss
       rule(:selectors) do
         next unless value
 
-        errors = Config::SelectorsValidator.call(value).errors
-        errors.each { |error| key(:selectors).failure(error.text) } unless errors.empty?
+        result = Config::SelectorsValidator.call(value)
+        next if result.success?
+
+        result.errors.each do |error|
+          key([:selectors, *error.path]).failure(error.text)
+        end
       end
 
       rule(request: :botasaurus) do

data/lib/html2rss/config.rb

@@ -13,53 +13,8 @@ module Html2rss
   class Config # rubocop:disable Metrics/ClassLength
     # Raised when a configuration hash fails runtime validation.
     class InvalidConfig < Html2rss::Error; end
-    # Sentinel to differentiate omitted params from explicit `nil`.
-    UNSET = Object.new.freeze
-
-    ValidationResult = Data.define(:success, :errors_hash)
-
-    ##
-    # Duck-compatible with +Dry::Validation::Result+ for resolve-time outcomes
-    # (+#success?+, +#errors#to_h+). Used for parse failures so callers never see a Struct snowflake.
-    class ValidationResult
-      ##
-      # @param message [String]
-      # @return [Html2rss::Config::ValidationResult]
-      def self.parse_failure(message)
-        new(success: false, errors_hash: { parse: [message] })
-      end
-
-      ##
-      # @return [Boolean]
-      def success? = success
-
-      ##
-      # @return [self]
-      def errors = self
-
-      ##
-      # @return [Hash]
-      def to_h = errors_hash
-    end
 
     class << self
-      ##
-      # Returns the exported JSON Schema for html2rss configuration.
-      #
-      # @return [Hash{String => Object}] JSON Schema represented as a Ruby hash
-      def json_schema
-        Schema.json_schema
-      end
-
-      ##
-      # Returns the exported JSON Schema as JSON.
-      #
-      # @param pretty [Boolean] whether to pretty-print the JSON output
-      # @return [String] serialized JSON Schema
-      def json_schema_json(pretty: true)
-        pretty ? JSON.pretty_generate(json_schema) : JSON.generate(json_schema)
-      end
-
       ##
       # Resolves a Hash, YAML file path, or YAML string to a working config Hash and validates it.
       # The returned Hash is a deep copy — callers may stamp strategy/params without mutating input.
@@ -67,13 +22,14 @@ module Html2rss
       # @param config_input [Hash, String] config hash, YAML string, or file path
       # @param feed_name [String, nil] optional feed name for multi-feed files
       # @param params [Hash] dynamic feed params
-      # @return [Array(Hash, Dry::Validation::Result, Html2rss::Config::ValidationResult)]
+      # @return [Array(Hash, Html2rss::Config::ValidationReport)]
       def resolve_and_validate(config_input, feed_name: nil, params: {})
-        param_arg = params.empty? ? UNSET : params
-        working = HashUtil.deep_dup(resolve_raw_hash(config_input, feed_name))
-        [working, validate(working, params: param_arg)]
-      rescue StandardError => error
-        [{}, ValidationResult.parse_failure(error.message)]
+        raw = resolve_raw_hash(config_input, feed_name)
+      rescue Psych::Exception, ArgumentError => error
+        [{}, IssueMapper.parse_failure(error.message)]
+      else
+        working = HashUtil.deep_dup(raw)
+        [working, validate(working, params:)]
       end
 
       ##
@@ -81,36 +37,15 @@ module Html2rss
       #
       # @param config [Hash{Symbol => Object}] the configuration hash
       # @param params [Hash{Symbol => Object, Hash{String => Object, nil}}] dynamic parameters for string formatting
-      # @return [Dry::Validation::Result] validation result after defaults are applied
-      def validate(config, params: UNSET)
+      # @return [Html2rss::Config::ValidationReport]
+      def validate(config, params: {})
         prepared_config = prepare_for_validation(resolve_effective_config(config, params:))
-
-        Validator.new.call(prepared_config)
+        IssueMapper.from(Validator.new.call(prepared_config), values: prepared_config)
       rescue DynamicParams::ParamsMissing => error
         prepared_config = prepare_for_validation(HashUtil.deep_symbolize_keys(config, context: 'config'))
         prepared_config[:dynamic_params_error] = error.message
 
-        Validator.new.call(prepared_config)
-      end
-
-      ##
-      # Loads and validates a YAML configuration file.
-      #
-      # @param file [String] the YAML file to load
-      # @param feed_name [String, nil] optional feed name for multi-feed files
-      # @param multiple_feeds_key [Symbol] key under which multiple feeds are defined
-      # @param params [Hash{Symbol => Object, Hash{String => Object, nil}}] dynamic parameters for string formatting
-      # @return [Dry::Validation::Result, Html2rss::Config::ValidationResult]
-      def validate_yaml(file, feed_name = nil, multiple_feeds_key: MultipleFeedsConfig::CONFIG_KEY_FEEDS, params: UNSET)
-        validate(load_yaml(file, feed_name, multiple_feeds_key:), params:)
-      end
-
-      ##
-      # Returns the packaged JSON Schema file path.
-      #
-      # @return [String] absolute path to the packaged JSON Schema file
-      def schema_path
-        Schema.path
+        IssueMapper.from(Validator.new.call(prepared_config), values: prepared_config)
       end
 
       ##
@@ -179,7 +114,7 @@ module Html2rss
       # @param config [Hash{Symbol => Object}] the configuration hash.
       # @param params [Hash{Symbol => Object, Hash{String => Object, nil}}] dynamic parameters for string formatting.
       # @return [Html2rss::Config] the configuration object.
-      def from_hash(config, params: UNSET)
+      def from_hash(config, params: {})
         new(resolve_effective_config(config, params:))
       end
 
@@ -226,6 +161,20 @@ module Html2rss
         Html2rss.defaults.default_strategy || :auto
       end
 
+      ##
+      # Normalizes raw config input with defaults.
+      #
+      # @param config [Hash{Symbol => Object}] raw config input
+      # @return [Hash{Symbol => Object}] config with defaults applied
+      def prepare_defaults(config)
+        config = HashUtil.deep_merge(default_config, config)
+        config = HashUtil.deep_merge({ selectors: Selectors::DEFAULT_CONFIG }, config) if config[:selectors]
+        if config[:auto_source]
+          config = HashUtil.deep_merge({ auto_source: Html2rss::AutoSource::DEFAULT_CONFIG }, config)
+        end
+        config
+      end
+
       private
 
       def default_request_config
@@ -239,7 +188,7 @@ module Html2rss
       def resolve_effective_config(config, params:)
         cfg = HashUtil.deep_symbolize_keys(config, context: 'config')
         p = parameter_defaults(cfg)
-        p.merge!(HashUtil.deep_symbolize_keys(params, context: 'params')) unless params.equal?(UNSET) || params.nil?
+        p.merge!(HashUtil.deep_symbolize_keys(params, context: 'params')) unless params.nil?
 
         cfg[:headers] = DynamicParams.call(cfg[:headers], p) if cfg[:headers]
         cfg[:channel] = DynamicParams.call(cfg[:channel], p) if cfg[:channel]
@@ -255,7 +204,7 @@ module Html2rss
       end
 
       def prepare_for_validation(config)
-        Config::Preparer.new.call(HashUtil.deep_dup(config))
+        prepare_defaults(HashUtil.deep_dup(config))
       end
 
       # @param config_input [Hash, String]
@@ -282,7 +231,7 @@ module Html2rss
     # @raise [InvalidConfig] if the configuration fails validation.
     def initialize(config)
       @request_controls = RequestControls.from_config(config)
-      prepared_config = Preparer.new.call(config)
+      prepared_config = self.class.prepare_defaults(config.dup)
       validated_config = validated_config_for(prepared_config)
 
       @config = validated_config.freeze
@@ -340,42 +289,15 @@ module Html2rss
 
     attr_reader :config
 
-    # Normalizes raw config input before validation.
-    class Preparer
-      ##
-      # @param config [Hash{Symbol => Object}] raw config input
-      # @return [Hash{Symbol => Object}] config with defaults applied
-      def call(config)
-        config = config.dup if config.frozen?
-
-        config = apply_default_config(config)
-        config = apply_default_selectors_config(config) if config[:selectors]
-        config = apply_default_auto_source_config(config) if config[:auto_source]
-
-        config
-      end
-
-      private
-
-      def apply_default_config(config)
-        HashUtil.deep_merge(Config.default_config, config)
-      end
-
-      def apply_default_selectors_config(config)
-        HashUtil.deep_merge({ selectors: Selectors::DEFAULT_CONFIG }, config)
-      end
-
-      def apply_default_auto_source_config(config)
-        HashUtil.deep_merge({ auto_source: Html2rss::AutoSource::DEFAULT_CONFIG }, config)
-      end
-    end
-
     def validated_config_for(config)
-      validator = Validator.new.call(config)
+      dry = Validator.new.call(config)
 
-      raise InvalidConfig, "Invalid configuration: #{validator.errors.to_h}" unless validator.success?
+      unless dry.success?
+        report = IssueMapper.from(dry, values: config)
+        raise InvalidConfig, "Invalid configuration: #{report}"
+      end
 
-      normalized_headers(validator.to_h)
+      normalized_headers(dry.to_h)
     end
 
     def normalized_headers(validated_config)