html2rss 0.29.1 → 0.31.0

data/lib/html2rss/config/issue_mapper.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Config
+    ##
+    # Maps internal Dry::Validation messages to {ValidationIssue} (predicate → code).
+    # Not a public API — Dry stays inside Validator / this mapper.
+    module IssueMapper
+      # Dry schema predicates → closed {ValidationIssue::CODES}.
+      PREDICATE_CODES = {
+        key?: :missing_key,
+        filled?: :missing_key,
+        str?: :type_mismatch,
+        int?: :type_mismatch,
+        bool?: :type_mismatch,
+        hash?: :type_mismatch,
+        array?: :type_mismatch,
+        type?: :type_mismatch,
+        format?: :invalid_value,
+        included_in?: :unknown_value,
+        gt?: :constraint,
+        gteq?: :constraint,
+        lt?: :constraint,
+        lteq?: :constraint,
+        max_size?: :constraint,
+        min_size?: :constraint
+      }.freeze
+
+      class << self
+        ##
+        # @param dry_result [Dry::Validation::Result]
+        # @param values [Hash, nil] config under validation (for actual/expected dig)
+        # @return [Html2rss::Config::ValidationReport]
+        def from(dry_result, values: nil)
+          return ValidationReport.ok if dry_result.success?
+
+          ValidationReport.failure(dry_result.errors.map { |message| issue_for(message, values:) })
+        end
+
+        ##
+        # @param message [String]
+        # @return [Html2rss::Config::ValidationReport]
+        def parse_failure(message)
+          ValidationReport.failure(
+            [
+              ValidationIssue.new(path: %i[parse], code: :parse, message:)
+            ]
+          )
+        end
+
+        private
+
+        def issue_for(message, values:)
+          path = Array(message.path).compact
+          ValidationIssue.new(
+            path:,
+            code: code_for(message.predicate),
+            message: message.text,
+            expected: expected_for(path, values),
+            actual: actual_for(path, values, message)
+          )
+        end
+
+        def code_for(predicate)
+          return :invalid_value if predicate.nil?
+
+          PREDICATE_CODES.fetch(predicate) do
+            raise ArgumentError, "unmapped Dry validation predicate: #{predicate.inspect}"
+          end
+        end
+
+        def actual_for(path, values, message)
+          dug = dig_path(values, path)
+          return dug unless dug.nil?
+          return message.input if message.is_a?(Dry::Schema::Message) && !message.input.nil?
+
+          nil
+        end
+
+        def expected_for(path, values)
+          return if values.nil? || path.empty?
+          return unless path.last.is_a?(Symbol)
+
+          parent = dig_path(values, path[0...-1])
+          return unless parent.is_a?(Hash)
+
+          option = Selectors::OptionSpec.option_for(parent, path.last)
+          return unless option
+
+          {
+            type: option.json_type,
+            required: option.required
+          }
+        end
+
+        def dig_path(values, path)
+          return if values.nil? || path.empty?
+
+          path.reduce(values) do |cursor, segment|
+            break unless cursor.is_a?(Hash) || (cursor.is_a?(Array) && segment.is_a?(Integer))
+
+            cursor[segment]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/config/request_headers.rb

@@ -24,7 +24,6 @@ module Html2rss
       DEFAULT_HEADERS = {
         'Accept' => DEFAULT_ACCEPT,
         'Cache-Control' => 'max-age=0',
-        'Connection' => 'keep-alive',
         'Sec-Fetch-Dest' => 'document',
         'Sec-Fetch-Mode' => 'navigate',
         'Sec-Fetch-Site' => 'none',
@@ -33,16 +32,18 @@ module Html2rss
         'User-Agent' => DEFAULT_USER_AGENT
       }.freeze
 
+      # Hop-by-hop headers forbidden in HTTP/2 requests (RFC 7540 §8.1.2.2 / RFC 9113 §8.2.1).
+      FORBIDDEN_H2_HEADERS = Set['connection', 'keep-alive', 'proxy-connection', 'transfer-encoding', 'upgrade'].freeze
+
       class << self
         ##
-        # :reek:ManualDispatch
         # :reek:TooManyStatements
         #
         # @return [Hash{String => String}] the default header set merged with global defaults
         def browser_defaults
           defaults = DEFAULT_HEADERS.dup
           global_headers = Html2rss.defaults.headers
-          global_headers = global_headers.call if global_headers.respond_to?(:call)
+          global_headers = global_headers.call if global_headers.is_a?(Proc)
 
           if global_headers.is_a?(Hash)
             global_headers.each do |key, value|
@@ -75,7 +76,7 @@ module Html2rss
       ##
       # @return [Hash{String => String}] normalized HTTP headers
       def to_h
-        defaults = DEFAULT_HEADERS.dup
+        defaults = self.class.browser_defaults
         normalized = normalize_custom_headers(headers)
 
         accept_override = normalized.delete('Accept')
@@ -84,6 +85,7 @@ module Html2rss
         defaults['Accept'] = normalize_accept(accept_override)
         defaults['Accept-Language'] = build_accept_language
 
+        defaults.reject! { |key, _| FORBIDDEN_H2_HEADERS.include?(key.downcase) }
         defaults.compact
       end
 

data/lib/html2rss/config/schema.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'json'
+
 module Html2rss
   class Config
     ##
@@ -19,6 +21,15 @@ module Html2rss
         Builder.call
       end
 
+      ##
+      # Returns the exported JSON Schema as JSON.
+      #
+      # @param pretty [Boolean] whether to pretty-print the JSON output
+      # @return [String] serialized JSON Schema
+      def json_schema_json(pretty: true)
+        pretty ? JSON.pretty_generate(json_schema) : JSON.generate(json_schema)
+      end
+
       ##
       # Resolves the packaged schema path used by downstream tools.
       #
@@ -107,12 +118,16 @@ module Html2rss
         # @param registry [Hash{Symbol => Class}]
         # @return [Hash{String => Hash}]
         def catalog_from_registry(registry)
-          registry.keys.sort.to_h { |name| [name.to_s, registry.fetch(name).schema_doc] }
+          registry.keys.sort.to_h { |name| [name.to_s, registry.fetch(name).schema_export] }
         end
 
         # rubocop:disable-next Metrics/MethodLength, Metrics/AbcSize, Layout/LineLength
         def overlay
           items_schema = Html2rss::Config::SelectorsValidator::Items.new.schema.json_schema(loose: true)
+          items_schema[:required] = Array(items_schema[:required]).reject { |key| key.to_sym == :selector }
+          items_schema[:properties][:selector] = items_schema.fetch(:properties).fetch(:selector).merge(
+            default: Html2rss::Selectors::DEFAULT_ITEMS_SELECTOR
+          )
           items_schema[:properties][:enhance] = items_schema.fetch(:properties).fetch(:enhance).merge(
             description: 'List-card enrichment: run Html::ArticleExtractor on each matched item node ' \
                          'to fill missing fields from the card HTML.'

data/lib/html2rss/config/selectors_validator.rb

@@ -6,9 +6,22 @@ module Html2rss
   class Config
     ##
     # Validates the configuration hash for :selectors.
-    class SelectorsValidator < Dry::Validation::Contract
-      # Required wrapper key used to validate dynamic selector names.
-      NESTING_KEY = :dynamic_keys_workaround
+    module SelectorsValidator
+      ##
+      # One selector validation error.
+      Error = Data.define(:path, :text)
+
+      Result = Data.define(:errors)
+
+      ##
+      # Validation result containing collected selector errors.
+      class Result
+        # @return [Boolean]
+        def success? = errors.empty?
+
+        # @return [Boolean]
+        def failure? = !success?
+      end
 
       ##
       # Validates the configuration of the :items selector
@@ -75,9 +88,6 @@ module Html2rss
       ##
       # Validates the configuration of a single selector.
       class Selector < Dry::Validation::Contract
-        # Extractor Options members supplied at runtime, not from feed YAML.
-        RUNTIME_EXTRACTOR_FIELDS = %i[selector channel].to_set.freeze
-
         params do
           optional(:selector)
           optional(:extractor).filled(:string)
@@ -96,11 +106,11 @@ module Html2rss
           klass = Selectors::Extractors::NAME_TO_CLASS[value.to_sym]
           next key(:extractor).failure("unknown extractor: #{value}") unless klass
 
-          klass::Options.members.each do |field|
-            next if RUNTIME_EXTRACTOR_FIELDS.include?(field)
-            next if values[field]
+          Selectors::OptionSpec.for(klass).each do |spec|
+            next unless spec.required
+            next if values[spec.name]
 
-            key(field).failure("`#{field}` is required for extractor `#{value}`")
+            key(spec.name).failure("`#{spec.name}` is required for extractor `#{value}`")
           end
         end
 
@@ -110,49 +120,44 @@ module Html2rss
 
           klass = Selectors::PostProcessors::NAME_TO_CLASS[name.to_sym]
           next key(:post_process).failure("Unknown post_processor `name`: #{name}") unless klass
-          next unless klass.const_defined?(:Options)
 
           post_process_option_type_errors(klass, value).each do |field, message|
             key(field).failure(message)
           end
+
+          pattern_error = gsub_pattern_error(klass, value)
+          key(:pattern).failure(pattern_error) if pattern_error
         end
 
         private
 
         def post_process_option_type_errors(klass, value)
-          required_option_type_errors(klass, value) + optional_option_type_errors(klass, value)
-        end
-
-        def required_option_type_errors(klass, value)
-          option_types = klass.const_defined?(:OPTION_TYPES) ? klass::OPTION_TYPES : {}
-
-          klass::Options.members.filter_map do |field|
-            expected = option_types.fetch(field, String)
-            next if value[field].is_a?(expected)
-
-            [field, option_type_failure(field, expected)]
+          Selectors::OptionSpec.for(klass).filter_map do |spec|
+            actual = value[spec.name]
+            if actual.nil?
+              [spec.name, spec.error_message(optional: false)] if spec.required
+            elsif !spec.valid_type?(actual)
+              [spec.name, spec.error_message(optional: !spec.required)]
+            end
           end
         end
 
-        def optional_option_type_errors(klass, value)
-          return [] unless klass.const_defined?(:OPTIONAL_OPTION_TYPES)
+        # Same compile as Gsub#call. ArgumentError is the bound; Gsub maps
+        # parser errors onto that path so admit and execute still share it.
+        #
+        # @param klass [Class]
+        # @param value [Hash]
+        # @return [String, nil]
+        def gsub_pattern_error(klass, value)
+          return unless klass == Selectors::PostProcessors::Gsub
 
-          klass::OPTIONAL_OPTION_TYPES.filter_map do |field, expected|
-            actual = value[field]
-            next if actual.nil? || actual.is_a?(expected)
-
-            [field, option_type_failure(field, expected, optional: true)]
-          end
-        end
+          pattern = value[:pattern]
+          return unless pattern.is_a?(String)
 
-        def option_type_failure(field, expected, optional: false)
-          label = {
-            Integer => 'an integer',
-            String => 'a string',
-            Hash => 'a hash'
-          }.fetch(expected) { "a #{expected}" }
-          suffix = optional ? ' or omitted' : ''
-          "`#{field}` must be #{label}#{suffix}"
+          Selectors::PostProcessors::Gsub.compiled_pattern(pattern)
+          nil
+        rescue ArgumentError => error
+          error.message
         end
       end
 
@@ -164,47 +169,64 @@ module Html2rss
         end
       end
 
-      params do
-        required(NESTING_KEY).hash
-      end
+      class << self
+        ##
+        # Shortcut to validate the config.
+        # @param config [Hash] the configuration hash to validate
+        # @return [Result]
+        def call(config)
+          errors = []
+          return Result.new(errors:) unless config.is_a?(Hash)
+
+          config.each_pair do |selector_key, selector|
+            validate_entry(selector_key, selector, config, errors)
+          end
+
+          Result.new(errors:)
+        end
 
-      rule(NESTING_KEY) do
-        value.each_pair do |selector_key, selector|
+        private
+
+        def validate_entry(selector_key, selector, config, errors)
           case selector_key.to_sym
           when Selectors::ITEMS_SELECTOR_KEY
-            Items.new.call(selector).errors.each { |error| key(selector_key).failure(error.text) }
+            collect_contract_errors(Items, selector, selector_key, errors)
           when :enclosure
-            Enclosure.new.call(selector).errors.each { |error| key(selector_key).failure(error.text) }
+            collect_contract_errors(Enclosure, selector, selector_key, errors)
           when :guid, :categories
-            unless selector.is_a?(Array)
-              key(selector_key).failure("`#{selector_key}` must be an array")
-              next
-            end
+            validate_array_selector(selector_key, selector, config, errors)
+          else
+            collect_contract_errors(Selector, selector, selector_key, errors)
+          end
+        end
 
-            key(selector_key).failure("`#{selector_key}` must contain at least one element") if selector.empty?
+        def collect_contract_errors(contract_class, selector, selector_key, errors)
+          contract_class.new.call(selector).errors.each do |error|
+            errors << Error.new(path: [selector_key, *error.path], text: error.text)
+          end
+        end
 
-            selector.each do |name|
-              next if values[NESTING_KEY].key?(name.to_sym)
+        def validate_array_selector(selector_key, selector, config, errors)
+          msg = array_selector_shape_error(selector_key, selector)
+          return errors << Error.new(path: [selector_key], text: msg) if msg
 
-              key(selector_key).failure("`#{selector_key}` references unspecified `#{name}`")
-            end
-          else
-            # From here on, the selector is found under its "dynamic" selector_key
-            Selector.new.call(selector).errors.each { |error| key(selector_key).failure(error.text) }
-          end
+          check_unspecified_references(selector_key, selector, config, errors)
         end
-      end
 
-      ##
-      # Shortcut to validate the config.
-      # @param config [Hash] the configuration hash to validate
-      # @return [Dry::Validation::Result] the result of the validation
-      def self.call(config)
-        # dry-validation/schema does not support "Dynamic Keys" yet: https://github.com/dry-rb/dry-schema/issues/37
-        # But :selectors contains mostly "dynamic" keys, as the user defines them to extract article attributes.
-        # --> Validate the dynamic keys manually.
-        # To be able to specify a `rule`, nest the config under NESTING_KEY and mark that as `required`.
-        new.call(NESTING_KEY => config)
+        def array_selector_shape_error(key, selector)
+          return "`#{key}` must be an array" unless selector.is_a?(Array)
+          return "`#{key}` must contain at least one element" if selector.empty?
+
+          nil
+        end
+
+        def check_unspecified_references(selector_key, selector, config, errors)
+          selector.each do |name|
+            next if config.key?(name.to_sym) || config.key?(name.to_s)
+
+            errors << Error.new(path: [selector_key], text: "`#{selector_key}` references unspecified `#{name}`")
+          end
+        end
       end
     end
   end

data/lib/html2rss/config/validation_issue.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Config
+    ValidationIssue = Data.define(:path, :code, :message, :expected, :actual)
+
+    ##
+    # One structured validation finding for GUI/MCP/CLI/Test.
+    #
+    # +path+ is JSON-pointer-friendly segments. +code+ is closed ({CODES}).
+    # +expected+ / +actual+ are JSON-ish or +nil+ (never invented).
+    class ValidationIssue
+      # Closed set of issue codes (mapper owns Dry predicate → code).
+      CODES = Set[
+        :missing_key,
+        :type_mismatch,
+        :unknown_value,
+        :invalid_value,
+        :constraint,
+        :parse
+      ].freeze
+
+      ##
+      # @param path [Array<Symbol, Integer, String>]
+      # @param code [Symbol]
+      # @param message [String]
+      # @param expected [Hash, String, Numeric, true, false, nil]
+      # @param actual [Hash, String, Numeric, true, false, nil]
+      def initialize(path:, code:, message:, expected: nil, actual: nil)
+        raise ArgumentError, "unknown ValidationIssue code: #{code.inspect}" unless CODES.include?(code)
+
+        super(
+          path: Array(path).freeze,
+          code:,
+          message: message.to_s,
+          expected:,
+          actual:
+        )
+      end
+
+      ##
+      # Wire shape for a single issue.
+      #
+      # @return [Hash{Symbol => Object}]
+      def to_h
+        { path:, code:, message:, expected:, actual: }
+      end
+
+      ##
+      # Single renderer for CLI / exceptions (path [code] message).
+      #
+      # @return [String]
+      def to_s
+        path_s = path.join('.')
+        path_s = '(root)' if path_s.empty?
+        "#{path_s} [#{code}] #{message}"
+      end
+    end
+  end
+end

data/lib/html2rss/config/validation_report.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Config
+    ValidationReport = Data.define(:issues)
+
+    ##
+    # Public result of config validation. Dry::Validation::Result never crosses this boundary.
+    # +success+ is derived from +issues+ — invalid states like +success: true+ with issues cannot exist.
+    class ValidationReport
+      ##
+      # @return [Html2rss::Config::ValidationReport]
+      def self.ok
+        new(issues: [].freeze)
+      end
+
+      ##
+      # @param issues [Array<Html2rss::Config::ValidationIssue>]
+      # @return [Html2rss::Config::ValidationReport]
+      def self.failure(issues)
+        new(issues: Array(issues).freeze)
+      end
+
+      ##
+      # @param issues [Array<Html2rss::Config::ValidationIssue>]
+      def initialize(issues:)
+        super(issues: Array(issues).freeze)
+      end
+
+      # @return [Boolean]
+      def success? = issues.empty?
+
+      # @return [Boolean]
+      def failure? = !success?
+
+      ##
+      # Wire shape: +{ success:, issues: [...] }+ only (MCP contract v3).
+      #
+      # @return [Hash{Symbol => Object}]
+      def to_h
+        { success: success?, issues: issues.map(&:to_h) }
+      end
+
+      ##
+      # Human-readable summary for CLI / exception messages.
+      #
+      # @return [String]
+      def to_s
+        return 'valid' if success?
+
+        issues.join('; ')
+      end
+    end
+  end
+end

data/lib/html2rss/config/validator.rb

@@ -16,10 +16,10 @@ module Html2rss
       # Baseline strategy-plan enum (:auto plus concrete RequestService strategies).
       BASE_STRATEGY_OPTIONS = Html2rss::FeedPipeline::StrategyPlan.accepted_names.freeze
       # Controlled vocabulary for catalog-only `directory.topics` (not RSS channel fields).
-      DIRECTORY_TOPICS = %w[
-        sports energy tech science news entertainment jobs finance
-        security travel environment consumer civic product research
-        health culture education rights transport
+      DIRECTORY_TOPICS = Set[
+        'sports', 'energy', 'tech', 'science', 'news', 'entertainment', 'jobs', 'finance',
+        'security', 'travel', 'environment', 'consumer', 'civic', 'product', 'research',
+        'health', 'culture', 'education', 'rights', 'transport'
       ].freeze
 
       # Contract for the top-level `channel` section.
@@ -134,8 +134,12 @@ module Html2rss
       rule(:selectors) do
         next unless value
 
-        errors = Config::SelectorsValidator.call(value).errors
-        errors.each { |error| key(:selectors).failure(error.text) } unless errors.empty?
+        result = Config::SelectorsValidator.call(value)
+        next if result.success?
+
+        result.errors.each do |error|
+          key([:selectors, *error.path]).failure(error.text)
+        end
       end
 
       rule(request: :botasaurus) do