html2rss 0.27.2 → 0.29.1

data/lib/html2rss/config/schema.rb

@@ -48,7 +48,7 @@ module Html2rss
       ##
       # Orchestrates schema assembly from runtime validator contracts plus
       # client-facing overlays.
-      # rubocop:disable Metrics/ClassLength -- overlay assembly stays in one builder
+      # rubocop:disable-next Metrics/ClassLength -- overlay assembly stays in one builder
       class Builder
         class << self
           # @return [Hash{String => Object}] fully assembled JSON schema hash
@@ -110,9 +110,13 @@ module Html2rss
           registry.keys.sort.to_h { |name| [name.to_s, registry.fetch(name).schema_doc] }
         end
 
-        # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Layout/LineLength
+        # rubocop:disable-next Metrics/MethodLength, Metrics/AbcSize, Layout/LineLength
         def overlay
           items_schema = Html2rss::Config::SelectorsValidator::Items.new.schema.json_schema(loose: true)
+          items_schema[:properties][:enhance] = items_schema.fetch(:properties).fetch(:enhance).merge(
+            description: 'List-card enrichment: run Html::ArticleExtractor on each matched item node ' \
+                         'to fill missing fields from the card HTML.'
+          )
           items_schema[:properties][:pagination] = {
             description: 'Pagination configuration or maximum page count integer.',
             oneOf: [
@@ -166,7 +170,7 @@ module Html2rss
               description: 'Selectors used to extract article attributes.',
               properties: {
                 items: items_schema.merge(
-                  description: 'Defines the items selector and optional enhancement settings.'
+                  description: 'Defines the items selector and list-card enhance settings.'
                 ),
                 enclosure: enclosure_schema.merge(
                   description: 'Describes enclosure extraction settings.'
@@ -183,7 +187,6 @@ module Html2rss
             }
           }
         end
-        # rubocop:enable Metrics/MethodLength, Metrics/AbcSize, Layout/LineLength
 
         def reference_array(description)
           {
@@ -197,7 +200,6 @@ module Html2rss
           }
         end
       end
-      # rubocop:enable Metrics/ClassLength
 
       ##
       # Wires extractor / post_process to thin oneOf $refs into the registry catalog.

data/lib/html2rss/config/validator.rb

@@ -19,6 +19,7 @@ module Html2rss
       DIRECTORY_TOPICS = %w[
         sports energy tech science news entertainment jobs finance
         security travel environment consumer civic product research
+        health culture education rights transport
       ].freeze
 
       # Contract for the top-level `channel` section.

data/lib/html2rss/config.rb

@@ -16,6 +16,32 @@ module Html2rss
     # Sentinel to differentiate omitted params from explicit `nil`.
     UNSET = Object.new.freeze
 
+    ValidationResult = Data.define(:success, :errors_hash)
+
+    ##
+    # Duck-compatible with +Dry::Validation::Result+ for resolve-time outcomes
+    # (+#success?+, +#errors#to_h+). Used for parse failures so callers never see a Struct snowflake.
+    class ValidationResult
+      ##
+      # @param message [String]
+      # @return [Html2rss::Config::ValidationResult]
+      def self.parse_failure(message)
+        new(success: false, errors_hash: { parse: [message] })
+      end
+
+      ##
+      # @return [Boolean]
+      def success? = success
+
+      ##
+      # @return [self]
+      def errors = self
+
+      ##
+      # @return [Hash]
+      def to_h = errors_hash
+    end
+
     class << self
       ##
       # Returns the exported JSON Schema for html2rss configuration.
@@ -34,6 +60,22 @@ module Html2rss
         pretty ? JSON.pretty_generate(json_schema) : JSON.generate(json_schema)
       end
 
+      ##
+      # Resolves a Hash, YAML file path, or YAML string to a working config Hash and validates it.
+      # The returned Hash is a deep copy — callers may stamp strategy/params without mutating input.
+      #
+      # @param config_input [Hash, String] config hash, YAML string, or file path
+      # @param feed_name [String, nil] optional feed name for multi-feed files
+      # @param params [Hash] dynamic feed params
+      # @return [Array(Hash, Dry::Validation::Result, Html2rss::Config::ValidationResult)]
+      def resolve_and_validate(config_input, feed_name: nil, params: {})
+        param_arg = params.empty? ? UNSET : params
+        working = HashUtil.deep_dup(resolve_raw_hash(config_input, feed_name))
+        [working, validate(working, params: param_arg)]
+      rescue StandardError => error
+        [{}, ValidationResult.parse_failure(error.message)]
+      end
+
       ##
       # Validates a configuration hash with the runtime validator.
       #
@@ -59,22 +101,10 @@ module Html2rss
         Schema.path
       end
 
-      ##
-      # Loads and validates a YAML configuration file.
-      #
-      # @param file [String] the YAML file to load
-      # @param feed_name [String, nil] optional feed name for multi-feed files
-      # @param multiple_feeds_key [Symbol] key under which multiple feeds are defined
-      # @param params [Hash{Symbol => Object, Hash{String => Object, nil}}] dynamic parameters for string formatting
-      # @return [Dry::Validation::Result] validation result after defaults are applied
-      def validate_yaml(file, feed_name = nil, multiple_feeds_key: MultipleFeedsConfig::CONFIG_KEY_FEEDS, params: UNSET)
-        validate(load_yaml(file, feed_name, multiple_feeds_key:), params:)
-      end
-
       ##
       # Serializes a configuration hash to string-key YAML.
       #
-      # This is the single serializer for CLI capture and MCP +capture_config+.
+      # This is the single serializer for CLI capture and MCP +capture+.
       #
       # @param hash [Hash] configuration hash (symbol or string keys)
       # @return [String] YAML document without Ruby symbol-key prefixes
@@ -109,7 +139,7 @@ module Html2rss
       # @param multiple_feeds_key [Symbol] the key under which multiple feeds are defined.
       # @return [Hash{Symbol => Object}] the configuration hash.
       # @raise [ArgumentError] if the file doesn't exist or feed is not found.
-      # rubocop:disable Metrics/MethodLength
+      # rubocop:disable-next Metrics/MethodLength
       def load_yaml(file, feed_name = nil, multiple_feeds_key: MultipleFeedsConfig::CONFIG_KEY_FEEDS)
         raise ArgumentError, "File '#{file}' does not exist" unless File.exist?(file)
         raise ArgumentError, "`#{multiple_feeds_key}` is a reserved feed name" if feed_name == multiple_feeds_key
@@ -129,7 +159,6 @@ module Html2rss
 
         MultipleFeedsConfig.to_single_feed(config, yaml, multiple_feeds_key:)
       end
-      # rubocop:enable Metrics/MethodLength
 
       ##
       # Processes the provided configuration hash, applying dynamic parameters if given,
@@ -216,6 +245,20 @@ module Html2rss
       def prepare_for_validation(config)
         Config::Preparer.new.call(HashUtil.deep_dup(config))
       end
+
+      # @param config_input [Hash, String]
+      # @param feed_name [String, nil]
+      # @return [Hash]
+      def resolve_raw_hash(config_input, feed_name)
+        if config_input.is_a?(Hash)
+          config_input
+        elsif File.file?(config_input.to_s)
+          load_yaml(config_input.to_s, feed_name)
+        else
+          from_yaml(config_input.to_s)
+        end
+      end
+      private :resolve_raw_hash
     end
 
     ##
@@ -264,8 +307,12 @@ module Html2rss
     def headers = config[:headers]
     # @return [Hash{Symbol => Object}] channel configuration
     def channel = config[:channel]
-    # @return [String] source channel URL
+    ##
+    # Source channel URL (also the default scrape URL).
+    #
+    # @return [String]
     def url = config.dig(:channel, :url)
+
     # @return [String, nil] configured channel time zone
     def time_zone = config.dig(:channel, :time_zone)
 
@@ -322,8 +369,7 @@ module Html2rss
     def normalized_headers(validated_config)
       validated_config[:headers] = RequestHeaders.normalize(
         validated_config[:headers],
-        channel_language: validated_config.dig(:channel, :language),
-        url: validated_config.dig(:channel, :url)
+        channel_language: validated_config.dig(:channel, :language)
       )
       validated_config
     end

data/lib/html2rss/doctor/botasaurus.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require 'faraday'
+require 'json'
+
+module Html2rss
+  ##
+  # CLI runtime health checks for optional scrape dependencies.
+  module Doctor
+    ##
+    # Preflight checks for Botasaurus scrape-api connectivity.
+    module Botasaurus
+      ##
+      # One named doctor check outcome.
+      Check = Data.define(:name, :ok, :detail) do
+        ##
+        # @return [Hash{Symbol => Object}]
+        def to_h
+          { name: name.to_s, ok:, detail: }
+        end
+      end
+
+      ##
+      # Aggregate doctor command outcome.
+      Result = Data.define(:ok, :checks, :message) do
+        ##
+        # @return [Hash{Symbol => Object}]
+        def to_h
+          { ok:, message:, checks: checks.map(&:to_h) }
+        end
+      end
+
+      module_function
+
+      ENV_VAR = 'BOTASAURUS_SCRAPER_URL'
+      private_constant :ENV_VAR
+
+      ##
+      # @param sample_url [String, nil] optional URL for a sample scrape smoke test
+      # @return [Result]
+      def call(sample_url: nil) # rubocop:disable Metrics/AbcSize
+        env = env_check
+        checks = [env_check_record(env)]
+        return missing_env_failure(checks) unless checks.first.ok
+
+        checks << health_check_record(env[:base_url])
+        return failure(checks, 'Botasaurus health check failed.') unless checks.last.ok
+
+        checks << sample_scrape_record(sample_url) if sample_url
+        return failure(checks, 'Sample scrape failed.') if sample_url && !checks.last.ok
+
+        Log.info('doctor botasaurus: preflight passed')
+        Result.new(ok: true, checks:, message: 'Botasaurus preflight passed.')
+      end
+
+      ##
+      # @return [Hash{Symbol => Object}]
+      def env_check
+        base = ENV.fetch(ENV_VAR, '').strip
+        {
+          configured: MCP::Runtime.botasaurus_configured?,
+          var: ENV_VAR,
+          base_url: base.empty? ? nil : base
+        }
+      end
+
+      ##
+      # @param base_url [String]
+      # @return [Hash{Symbol => Object}]
+      def health_check(base_url)
+        uri = Url.for_channel(base_url)
+        client = Faraday.new(url: uri.to_s.chomp('/'), request: { timeout: 5 })
+        response = client.get('/health')
+        body = JSON.parse(response.body)
+        { ok: response.status == 200, status: response.status, version: body['version'] }
+      rescue StandardError => error
+        { ok: false, error: "#{error.class}: #{error.message}" }
+      end
+
+      ##
+      # @param url [String]
+      # @return [Hash{Symbol => Object}]
+      def sample_scrape(url)
+        wire = PageRecon::Diagnostics.call(url:, strategy: :botasaurus).to_wire_h
+        {
+          ok: wire[:status].to_i.between?(200, 399),
+          status: wire[:status],
+          articles_count: wire[:articles_count]
+        }
+      rescue StandardError => error
+        { ok: false, error: "#{error.class}: #{error.message}" }
+      end
+
+      def env_check_record(env = env_check)
+        Check.new(name: :env, ok: env[:configured], detail: { var: env[:var], base_url_set: !env[:base_url].nil? })
+      end
+      module_function :env_check_record
+      private_class_method :env_check_record
+
+      def health_check_record(base_url)
+        return Check.new(name: :health, ok: false, detail: { error: 'missing base URL' }) unless base_url
+
+        health = health_check(base_url)
+        Check.new(name: :health, ok: health[:ok], detail: health)
+      end
+      module_function :health_check_record
+      private_class_method :health_check_record
+
+      def sample_scrape_record(url)
+        sample = sample_scrape(url)
+        Check.new(name: :sample_scrape, ok: sample[:ok], detail: sample)
+      end
+      module_function :sample_scrape_record
+      private_class_method :sample_scrape_record
+
+      def missing_env_failure(checks)
+        Log.info("doctor botasaurus: #{ENV_VAR} unset")
+        failure(checks, "Set #{ENV_VAR} to the scrape-api base URL.")
+      end
+      module_function :missing_env_failure
+      private_class_method :missing_env_failure
+
+      def failure(checks, message)
+        Result.new(ok: false, checks:, message:)
+      end
+      module_function :failure
+      private_class_method :failure
+    end
+  end
+end

data/lib/html2rss/feed_builder/rss.rb

@@ -96,7 +96,7 @@ module Html2rss
         @stylesheets.map { |style| Stylesheet.new(**style) }
       end
 
-      # rubocop:disable Metrics/AbcSize
+      # rubocop:disable-next Metrics/AbcSize
       def make_channel(maker)
         channel_maker = maker.channel
         %i[language title description ttl].each do |key|
@@ -111,7 +111,6 @@ module Html2rss
 
         make_image(maker.image) if channel.image
       end
-      # rubocop:enable Metrics/AbcSize
 
       def make_image(image_maker)
         image_maker.url = channel.image.to_s

data/lib/html2rss/feed_pipeline/README.md

@@ -1,6 +1,6 @@
 # FeedPipeline — `auto` request strategy
 
-`:auto` is the default request plan for feed builds (`auto_source`, `auto_json_feed`, Capture, and MCP `scrape_url` / `capture_config`). `FeedPipeline::StrategyPlan` resolves it; `FeedPipeline::AutoFallback` executes `AutoFallback::CHAIN`.
+`:auto` is the default request plan for feed builds (`auto_source`, `auto_json_feed`, Capture, and MCP `scrape` / `capture`). `FeedPipeline::StrategyPlan` resolves it; `FeedPipeline::AutoFallback` executes `AutoFallback::CHAIN`.
 
 Use `:auto` when you want Faraday first and a browser-backed hop only if that fetch fails or yields zero items. Pin a concrete strategy (`faraday`, `botasaurus`, `local_file`) when you need a single transport.
 
@@ -11,14 +11,16 @@ Use `:auto` when you want Faraday first and a browser-backed hop only if that fe
 1. **Faraday** — plain HTTP (faster, cheaper).
 2. **Botasaurus** — attempted when Faraday raises a fallback-eligible error (for example `BlockedSurfaceDetected` or timeout) or extracts zero feed items.
 
+Before escalating Faraday → Botasaurus on a **weak** auto-source extract (empty / below floor / high-entropy / app-shell / unsupported — not blocked), `FeedResolution` may probe up to five same-origin listing or native-feed candidates and rewrite the effective scrape URL via {ScrapeTarget}. Botasaurus then uses the resolved URL. Direct RSS/Atom entry URLs are parsed via `Syndication::Parser` without HTML AutoSource.
+
 There is no Browserless / Puppeteer-in-gem tier. Pin `botasaurus` when you want browser rendering without Faraday first. Botasaurus needs `BOTASAURUS_SCRAPER_URL`.
 
 ## Surfaces
 
-| Surface | `:auto` behavior |
-|---------|------------------|
-| Gem / CLI feed build, MCP `scrape_url`, Capture | Full AutoFallback chain (`faraday` → `botasaurus`) |
-| MCP `inspect_url` | Cheap diagnostic: `StrategyPlan.concrete_for_diagnostic` maps `auto` to Faraday (pin `botasaurus` when you need browser rendering) |
+| Surface                                  | `:auto` behavior                                                                                                                   |
+| ---------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------- |
+| Gem / CLI `apply`, MCP `scrape`, Capture | Full AutoFallback chain (`faraday` → `botasaurus`)                                                                                 |
+| MCP `inspect`                            | Cheap diagnostic: `StrategyPlan.concrete_for_diagnostic` maps `auto` to Faraday (pin `botasaurus` when you need browser rendering) |
 
 ## Fallback vs abort
 

data/lib/html2rss/feed_pipeline/auto_fallback.rb

@@ -10,6 +10,9 @@ module Html2rss
       # Ordered list of concrete request strategies attempted by the :auto plan.
       CHAIN = %i[faraday botasaurus].freeze
 
+      # Deterministic HTTP statuses that indicate permanent failure rather than anti-bot challenges.
+      DETERMINISTIC_HTTP_STATUSES = Set[400, 404, 410, 422, 451].freeze
+
       # Error classes that should abort auto fallback immediately.
       NON_FALLBACK_ERRORS = [
         RequestService::UnknownStrategy,
@@ -18,23 +21,49 @@ module Html2rss
         RequestService::RequestBudgetExceeded,
         RequestService::PrivateNetworkDenied,
         RequestService::CrossOriginFollowUpDenied,
-        RequestService::ResponseTooLarge
+        RequestService::ResponseTooLarge,
+        RequestService::RedirectLimitReached
       ].freeze
 
       ##
       # Mutable run state for one auto-fallback chain: attempts plus selected result.
       class AttemptState
-        attr_reader :attempts, :result, :last_response
+        attr_reader :attempts, :result, :last_response, :entry_resolution
 
         def initialize
           @attempts = []
           @result = nil
           @last_response = nil
+          @entry_resolution = nil
+          @resolution_tried = false
+          @aborted = false
         end
 
         # @return [Boolean] true when a strategy already yielded items
         def succeeded? = !result.nil?
 
+        # @return [Boolean] whether attempts were aborted due to deterministic errors
+        def aborted? = @aborted
+
+        # @return [void]
+        def abort!
+          @aborted = true
+        end
+
+        # @return [Boolean] whether entry resolution already ran for this chain
+        def resolution_tried? = @resolution_tried
+
+        # @return [void]
+        def mark_resolution_tried!
+          @resolution_tried = true
+        end
+
+        # @param result [Html2rss::FeedResolution::Result]
+        # @return [void]
+        def remember_entry_resolution(result)
+          @entry_resolution = FeedResolution::Diag.from_result(result)
+        end
+
         # @param strategy [Symbol] strategy that raised
         # @param error [Exception] caught error
         # @return [void]
@@ -64,10 +93,11 @@ module Html2rss
         # @param selected_strategy [Symbol] concrete strategy that produced items
         # @param attempt_count [Integer] number of attempts recorded for this chain
         # @param admission_drops [Hash{String => Integer}] Cleanup drop tallies
+        # @param scrape_target [Html2rss::ScrapeTarget]
         # @return [void]
-        # rubocop:disable Metrics/ParameterLists -- PipelineOutcome kwargs stay co-located
+        # rubocop:disable-next Metrics/ParameterLists, Metrics/MethodLength -- PipelineOutcome kwargs stay co-located
         def succeed!(response:, articles:, dedup_dropped:, selected_strategy:, attempt_count:,
-                     admission_drops: {})
+                     scrape_target:, admission_drops: {})
           @result = PipelineOutcome.new(
             response:,
             articles:,
@@ -75,10 +105,11 @@ module Html2rss
             selected_strategy:,
             attempt_count:,
             strategy_attempts: attempts,
-            admission_drops:
+            admission_drops:,
+            scrape_target:,
+            entry_resolution:
           )
         end
-        # rubocop:enable Metrics/ParameterLists
       end
 
       ##
@@ -94,6 +125,7 @@ module Html2rss
         @pipeline = pipeline
         @config = config
         @resources = resources
+        @scrape_target = ScrapeTarget.from_config(config)
       end
 
       ##
@@ -107,19 +139,21 @@ module Html2rss
 
       private
 
-      attr_reader :strategies, :budget, :pipeline, :config, :resources
+      attr_reader :strategies, :budget, :pipeline, :config, :resources, :scrape_target
 
       def run_attempts
         AttemptState.new.tap do |state|
           strategies.each_with_index do |strategy, index|
             attempt(strategy:, next_strategy: strategies[index + 1], state:)
-            break if state.succeeded?
+            break if state.succeeded? || state.aborted?
           end
         end
       end
 
       def attempt(strategy:, next_strategy:, state:)
-        request_session = pipeline.request_session_for(config, strategy:, resources:)
+        request_session = pipeline.request_session_for(
+          config, strategy:, resources:, scrape_url: scrape_target.effective_url
+        )
         response = fetch_response(request_session:, strategy:, next_strategy:, state:)
         return unless response
 
@@ -139,7 +173,8 @@ module Html2rss
         nil
       end
 
-      def process_response(response:, strategy:, next_strategy:, request_session:, state:) # rubocop:disable Metrics/MethodLength -- extract + success path
+      # rubocop:disable-next Metrics/MethodLength, Metrics/AbcSize -- extract + resolution delegate
+      def process_response(response:, strategy:, next_strategy:, request_session:, state:)
         articles, dedup_dropped, admission_drops = articles_for(response:, request_session:)
         state.remember_response(response)
         items_count = articles.size
@@ -147,39 +182,64 @@ module Html2rss
         Log.debug("#{self.class}: strategy=#{strategy} items=#{items_count} " \
                   "host=#{response.url.host} elapsed=#{format('%.3f', budget.elapsed_seconds)}s " \
                   "budget_remaining=#{budget_remaining_label}")
+
+        if (outcome = FeedResolution.try_apply!(
+          pipeline:, config:, response:, session: request_session, strategy:, resources:,
+          articles:, scrape_target:, state:, budget:
+        ))
+          @scrape_target = outcome.scrape_target
+          return
+        end
+
         if items_count.positive?
           return record_success(response:, strategy:, articles:, dedup_dropped:, admission_drops:,
                                 state:)
         end
 
+        if deterministic_status?(response)
+          state.abort!
+          Log.debug("#{self.class}: deterministic HTTP #{response.status}; aborting auto fallback")
+          return
+        end
+
         log_info_fallback_zero_items(strategy:, next_strategy:, response:) if next_strategy
       end
 
+      def deterministic_status?(response)
+        DETERMINISTIC_HTTP_STATUSES.member?(response&.status)
+      end
+
       def articles_for(response:, request_session:)
         pipeline.deduplicated_articles(config:, response:, request_session:)
       end
 
-      # rubocop:disable Metrics/ParameterLists -- success kwargs match PipelineOutcome
+      # rubocop:disable-next Metrics/ParameterLists -- success kwargs match PipelineOutcome
       def record_success(response:, strategy:, articles:, dedup_dropped:, admission_drops:, state:)
         attempt_count = state.attempts.size
-        state.succeed!(response:, articles:, dedup_dropped:, selected_strategy: strategy,
-                       attempt_count:, admission_drops:)
+        state.succeed!(
+          response:, articles:, dedup_dropped:, selected_strategy: strategy,
+          attempt_count:, admission_drops:,
+          scrape_target:
+        )
         return unless attempt_count > 1
 
         Log.info("#{self.class}: auto selected strategy=#{strategy} after attempts=#{attempt_count} " \
                  "host=#{response.url.host} elapsed=#{format('%.3f', budget.elapsed_seconds)}s " \
                  "budget_remaining=#{budget_remaining_label}")
       end
-      # rubocop:enable Metrics/ParameterLists
 
       def finalize_failure(attempts:, response:)
-        surface_category = response && AutoSource::Scraper.classify_no_scraper_surface(
-          response.parsed_body, body: response.body
-        )
+        surface_category = surface_category_for(response)
         raise NoFeedItemsExtracted.new(attempts:, surface_category:)
       end
 
-      # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      def surface_category_for(response)
+        return unless response
+
+        PageRecon.surface_category_for(response:, url: response.url)
+      end
+
+      # rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength
       def log_fallback_error(strategy:, next_strategy:, error:, request_session:)
         host = request_session.url.host
         detail = "host=#{host} elapsed=#{format('%.3f', budget.elapsed_seconds)}s " \
@@ -193,7 +253,6 @@ module Html2rss
         end
         Log.debug("#{self.class}: strategy=#{strategy} error=#{error.class}: #{error.message} #{detail}")
       end
-      # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
       def log_info_fallback_zero_items(strategy:, next_strategy:, response:)
         Log.info("#{self.class}: auto fallback #{strategy} -> #{next_strategy} after zero extracted items " \

data/lib/html2rss/feed_pipeline/runtime_policy.rb

@@ -60,6 +60,7 @@ module Html2rss
           1 +
             RequestSession::Pager.request_slots_for(config.selectors&.dig(:items, :pagination)) +
             AutoSource.request_slots_for(config.auto_source) +
+            FeedResolution.request_slots_for(config.auto_source) +
             StrategyPlan.resolve(config.strategy).request_slots
         end
       end