html2rss 0.26.0 → 0.27.0

data/lib/html2rss/capture.rb

@@ -2,20 +2,28 @@
 
 module Html2rss
   ##
-  # Analyzes a URL and produces a reusable feed config hash with derived CSS selectors.
+  # Analyzes a URL and produces a durable feed config: items selector + +enhance: true+.
   #
-  # Uses the auto-source pipeline to extract articles, then traces back through
-  # SST segments to build items + attribute selectors suitable for a static feed config.
-  class Capture # rubocop:disable Metrics/ClassLength
+  # Fetches via {FeedPipeline} (including AutoFallback for +:auto+), extracts articles,
+  # then derives a reusable items CSS selector from SST segments (list → cluster → semantic).
+  #
+  # {include:file:lib/html2rss/capture/README.md}
+  class Capture # rubocop:disable Metrics/ClassLength -- segment strategies + CSS trim stay co-located
     LEADING_TRIM_TAGS = %w[html body].freeze
     private_constant :LEADING_TRIM_TAGS
 
+    # Ordered Segmenter strategies tried until the items selector quality gate passes.
+    SEGMENT_STRATEGIES = %i[list cluster semantic].freeze
+
+    # Minimum matched segment/article pairs required to emit an items selector.
+    MIN_SELECTOR_MATCHES = 2
+
     ##
-    # Result of a capture operation.
-    # @!attribute config [Hash] feed config hash with +:channel+ and +:selectors+
-    # @!attribute articles_count [Integer] number of articles extracted
-    # @!attribute channel_title [String] derived channel title
-    CaptureResult = Data.define(:config, :articles_count, :channel_title)
+    # Result of a capture operation (config plus quality meta).
+    CaptureResult = Data.define(
+      :config, :articles_count, :channel_title, :has_selectors, :segment_strategy,
+      :admission_drops, :selected_strategy
+    )
 
     class << self
       ##
@@ -57,31 +65,47 @@ module Html2rss
     # Runs the capture pipeline.
     #
     # @return [CaptureResult]
-    def build # rubocop:disable Metrics/MethodLength
-      response = fetch_response
-      articles = extract_articles(response)
-      selectors = derive_selectors(response, articles)
+    def build # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- outcome + selector + result assembly
+      outcome = FeedPipeline.new(raw_config).to_outcome
+      selectors, segment_strategy = derive_selectors(outcome.response, outcome.articles)
 
       config = {
-        channel: build_channel(response),
+        channel: build_channel(outcome.response),
         selectors: selectors.empty? ? nil : selectors,
+        **strategy_stamp(outcome),
         **local_file_request_overlay
       }.compact
 
       CaptureResult.new(
         config:,
-        articles_count: articles.size,
-        channel_title: channel_title_from(response)
+        articles_count: outcome.articles.size,
+        channel_title: channel_title_from(outcome.response),
+        has_selectors: !selectors.empty?,
+        segment_strategy:,
+        admission_drops: outcome.admission_drops,
+        selected_strategy: outcome.selected_strategy
       )
     end
 
     private
 
+    def strategy_stamp(outcome)
+      concrete = outcome.selected_strategy || concrete_request_strategy
+      return {} if concrete.nil? || concrete == :auto
+
+      { strategy: concrete }
+    end
+
+    def concrete_request_strategy
+      plan = FeedPipeline::StrategyPlan.resolve(@strategy)
+      plan.is_a?(FeedPipeline::StrategyPlan::Concrete) ? plan.strategy : nil
+    end
+
     def raw_config
       @raw_config ||= build_raw_config
     end
 
-    def build_raw_config # rubocop:disable Metrics/MethodLength
+    def build_raw_config
       Config.auto_source_config(
         url: @url,
         items_selector: @items_selector_hint,
@@ -91,10 +115,7 @@ module Html2rss
           max_requests: @max_requests
         ),
         limit: @limit
-      ).tap do |config|
-        config[:strategy] = resolve_strategy(config[:strategy] || @strategy)
-        apply_local_file_path!(config)
-      end
+      ).tap { |config| apply_local_file_path!(config) }
     end
 
     def apply_local_file_path!(config)
@@ -111,62 +132,39 @@ module Html2rss
       { strategy: :local_file, request: { local_file_path: @local_file_path } }
     end
 
-    def resolve_strategy(strategy)
-      plan = FeedPipeline::StrategyPlan.resolve(strategy)
-      plan.is_a?(FeedPipeline::StrategyPlan::Auto) ? :faraday : plan.strategy
-    end
-
-    def fetch_response
-      config = Config.from_hash(raw_config)
-      resources = FeedPipeline::RuntimePolicy.resources_for(config)
-      session = RequestSession.build(
-        config:,
-        strategy: config.strategy,
-        budget: resources.budget,
-        policy: resources.policy
-      )
-      session.fetch_initial_response
-    end
-
-    def extract_articles(response)
-      auto_source_opts = raw_config[:auto_source] || AutoSource::DEFAULT_CONFIG
-      AutoSource.new(response, auto_source_opts).articles
-    rescue AutoSource::Scraper::NoScraperFound
-      []
-    end
-
-    def derive_selectors(response, articles) # rubocop:disable Metrics/MethodLength
-      return {} if articles.empty? || !response.html_response?
-
-      sst = normalize_sst(response)
-      return {} unless sst
+    # @return [Array(Hash, Symbol, nil)] selectors hash and winning segment strategy
+    def derive_selectors(response, articles)
+      return hint_selectors if @items_selector_hint
+      return [{}, nil] if articles.empty? || !response.html_response?
 
-      segments = discover_segments(sst)
-      return {} if segments.empty?
+      sst = SST::Normalizer.call(response.body)
+      return [{}, nil] unless sst
 
-      matched = match_segments_to_articles(segments, articles)
-      return {} if matched.empty?
-
-      build_selector_hash(matched, sst)
+      select_enhance_selectors(sst, articles)
     rescue ArgumentError => error
       Log.warn("Capture selector derivation failed: #{error.message}")
-      {}
+      [{}, nil]
     end
 
-    def normalize_sst(response)
-      SST::Normalizer.call(response.body)
+    def hint_selectors
+      [{ items: { selector: @items_selector_hint, enhance: true } }, :hint]
     end
 
-    def discover_segments(sst)
+    def select_enhance_selectors(sst, articles) # rubocop:disable Metrics/MethodLength -- strategy loop + gate
       link_resolver = Scoring::LinkResolver.new(@url)
 
-      AutoSource::Segmenter.call(
-        sst,
-        base_url: @url,
-        strategy: :list,
-        permit_unanchored: false,
-        link_resolver:
-      )
+      SEGMENT_STRATEGIES.each do |strategy|
+        segments = AutoSource::Segmenter.call(
+          sst, base_url: @url, strategy:, permit_unanchored: false, link_resolver:
+        )
+        matched = match_segments_to_articles(segments, articles)
+        items_sel = items_selector(matched)
+        next unless items_sel && matched.size >= MIN_SELECTOR_MATCHES
+
+        return [{ items: { selector: items_sel, enhance: true } }, strategy]
+      end
+
+      [{}, nil]
     end
 
     def match_segments_to_articles(segments, articles)
@@ -201,24 +199,10 @@ module Html2rss
       segment.root_node.visible_text.to_s.strip
     end
 
-    def build_selector_hash(matched, _sst)
-      items_sel = items_selector(matched)
-      return {} unless items_sel
-
-      first = matched.first
-      root = first[:segment].root_node
-
-      attrs = {}.tap do |a|
-        a[:title] = title_selector(root)
-        a[:url] = url_selector(first[:segment])
-        a[:description] = description_selector(root, first[:segment])
-      end.compact
-
-      { items: { selector: items_sel }, **attrs }
-    end
-
     def items_selector(matched)
-      roots = matched.map { |m| m[:segment].root_node }
+      return nil if matched.empty?
+
+      roots = lift_heading_link_roots(matched.map { |m| m[:segment].root_node })
       shared = shared_class_items_selector(roots)
       return shared if shared
 
@@ -230,6 +214,58 @@ module Html2rss
       css_from_trimmed_tag_path(tag_path)
     end
 
+    def lift_heading_link_roots(roots)
+      roots.map { |root| lift_heading_link_root(root, roots) }
+    end
+
+    def lift_heading_link_root(root, all_roots)
+      return root unless heading_or_inner_title_link?(root)
+
+      index = SST::Index.for_node(root)
+      return root unless index
+
+      walk_usable_card(root, index, all_roots)
+    end
+
+    def walk_usable_card(root, index, all_roots)
+      candidate = root
+      parent = index.parent_of(root)
+      while parent && Html::Navigator.usable_card_parent?(parent)
+        break if contains_other_root?(parent, root, all_roots)
+
+        candidate = parent
+        parent = index.parent_of(parent)
+      end
+      candidate
+    end
+
+    def heading_or_inner_title_link?(node)
+      name = node.name.to_s
+      return false if wrapping_anchor_root?(node)
+
+      Html::Navigator::HEADING_TAGS.include?(name) || name == 'a'
+    end
+
+    def wrapping_anchor_root?(node)
+      return false unless node.name.to_s == 'a'
+      return false unless node.respond_to?(:find)
+
+      tags = Html::Navigator::WRAPPING_ANCHOR_CHILD_TAGS
+      node.find { |child| !child.equal?(node) && tags.include?(child.name.to_s) }
+    end
+
+    def contains_other_root?(parent, root, all_roots)
+      all_roots.any? do |other|
+        next if other.equal?(root)
+
+        other.equal?(parent) || sst_descendant?(other, parent)
+      end
+    end
+
+    def sst_descendant?(child, ancestor)
+      SST::Index.for_node(child)&.descendant_of?(child, ancestor)
+    end
+
     def shared_class_items_selector(roots)
       shared = roots.map { |root| root.attrs.class_names }.reduce { |left, right| left & right }
       return nil if shared.nil? || shared.empty?
@@ -265,39 +301,6 @@ module Html2rss
       "/#{prefix.join('/')}"
     end
 
-    def title_selector(root) # rubocop:disable Metrics/CyclomaticComplexity
-      heading = root.find(&:heading?)
-      relative = css_for_path(heading.tag_path, root.tag_path) if heading
-      relative ||= begin
-        link = root.find { |n| n.link? && n.visible_text.to_s.strip.length > 3 }
-        css_for_path(link.tag_path, root.tag_path) if link
-      end
-      return nil unless relative
-
-      { selector: relative }
-    end
-
-    def url_selector(segment)
-      return nil unless segment.primary_link
-
-      relative = css_for_path(segment.primary_link.tag_path, segment.root_node.tag_path)
-      { selector: relative, extractor: 'href' }
-    end
-
-    def description_selector(root, segment)
-      relative = css_for_path(root.tag_path, segment.root_node.tag_path)
-      return nil if relative.nil? || relative.empty? || relative == '.'
-
-      { selector: relative }
-    end
-
-    def css_for_path(full_path, root_path)
-      relative = full_path.delete_prefix(root_path)
-      return '.' if relative.empty?
-
-      relative.split('/').reject(&:empty?).join(' > ')
-    end
-
     def build_channel(response)
       {
         url: @url,

data/lib/html2rss/cli.rb

@@ -2,7 +2,6 @@
 
 require 'fileutils'
 require 'json'
-require 'yaml'
 require 'thor'
 
 module Html2rss
@@ -10,10 +9,6 @@ module Html2rss
   # The Html2rss command line interface.
   class CLI < Thor # rubocop:disable Metrics/ClassLength
     check_unknown_options!
-    # Ordered fallback chain attempted by the feed-level :auto plan.
-    #
-    # @return [Array<Symbol>]
-    AUTO_FALLBACK_CHAIN = Html2rss::FeedPipeline::AutoFallback::CHAIN.freeze
     # Supported CLI strategy plan option values (:auto plus concrete strategies).
     #
     # @return [Array<String>]
@@ -24,7 +19,7 @@ module Html2rss
     # @return [String]
     STRATEGY_OPTION_DESC = [
       'Optional request strategy (defaults to auto; auto tries',
-      "#{AUTO_FALLBACK_CHAIN.join(' -> ')})"
+      "#{Html2rss::FeedPipeline::AutoFallback::CHAIN.join(' -> ')})"
     ].join(' ').freeze
 
     # @return [Boolean] whether Thor should terminate process on command failures
@@ -86,17 +81,19 @@ module Html2rss
     method_option :input,
                   type: :string,
                   desc: 'Local HTML file path to read input from'
+    method_option :explain,
+                  type: :boolean,
+                  desc: 'Print Status JSON to stderr (stdout stays the feed)',
+                  default: false
     # @param url [String, nil] source page URL for auto discovery
     # @return [void]
     def auto(url = nil)
       format = options.fetch(:format, 'rss')
       strategy, local_file_path, url = prepare_auto_inputs(url, options[:input])
+      feed_result = execute_feed { auto_feed_result_for(url, strategy, local_file_path) }
 
-      result = execute_feed do
-        source_call(url, strategy, local_file_path, format == 'jsonfeed')
-      end
-
-      puts(format == 'jsonfeed' ? JSON.pretty_generate(result) : result)
+      explain_status!(feed_result.status) if options[:explain]
+      puts(format == 'jsonfeed' ? JSON.pretty_generate(feed_result.to_json_feed) : feed_result.to_rss)
     end
 
     desc 'capture URL', 'Analyze a URL and print a reusable YAML feed config'
@@ -117,14 +114,18 @@ module Html2rss
     method_option :input,
                   type: :string,
                   desc: 'Local HTML file path to read input from'
+    method_option :explain,
+                  type: :boolean,
+                  desc: 'Print capture quality JSON to stderr (stdout stays YAML)',
+                  default: false
     ##
     # Captures a URL and prints a reusable YAML config.
     #
     # @param url [String, nil] source page URL for capture
     # @return [void]
-    def capture(url = nil) # rubocop:disable Metrics/MethodLength
+    def capture(url = nil) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- CLI option wiring
       strategy, local_file_path, url = prepare_auto_inputs(url, options[:input])
-      config = Html2rss.capture(
+      result = Html2rss::Capture.build(
         url,
         strategy:,
         items_selector: options[:items_selector],
@@ -133,7 +134,8 @@ module Html2rss
         max_requests: options[:max_requests],
         local_file_path:
       )
-      puts YAML.dump(HashUtil.deep_stringify_keys(config))
+      explain_capture!(result) if options[:explain]
+      puts Config.to_yaml(result.config)
     end
 
     desc 'schema', 'Print the exported config JSON Schema'
@@ -294,9 +296,8 @@ module Html2rss
       raise Thor::Error, error.message
     end
 
-    def source_call(url, strategy, local_file_path, is_json)
-      method = is_json ? Html2rss.method(:auto_json_feed) : Html2rss.method(:auto_source)
-      method.call(
+    def auto_feed_result_for(url, strategy, local_file_path)
+      Html2rss.auto_feed_result(
         url,
         strategy:,
         items_selector: options[:items_selector],
@@ -307,6 +308,23 @@ module Html2rss
       )
     end
 
+    def explain_status!(status)
+      $stderr.puts JSON.pretty_generate(status.to_h) # rubocop:disable Style/StderrPuts -- CLI explain contract
+    end
+
+    def explain_capture!(result)
+      $stderr.puts JSON.pretty_generate( # rubocop:disable Style/StderrPuts -- CLI explain contract
+        {
+          articles_count: result.articles_count,
+          channel_title: result.channel_title,
+          has_selectors: result.has_selectors,
+          segment_strategy: result.segment_strategy,
+          selected_strategy: result.selected_strategy,
+          admission_drops: result.admission_drops
+        }.compact
+      )
+    end
+
     def check_file_exists!(path)
       File.expand_path(path).tap do |file_path|
         raise Thor::Error, "Input file does not exist: #{path}" unless File.exist?(file_path)

data/lib/html2rss/config/schema.rb

@@ -82,6 +82,18 @@ module Html2rss
 
           topics = schema.dig(:properties, :directory, :properties, :topics)
           topics[:minItems] = 1 if topics.is_a?(Hash)
+          apply_botasaurus_schema!(schema)
+        end
+
+        def apply_botasaurus_schema!(schema)
+          request = schema.dig(:properties, :request, :properties)
+          return unless request.is_a?(Hash)
+
+          exported = Html2rss::Config::Validator::BotasaurusRequestExport
+                     .new.schema.json_schema(loose: true).except(:$schema)
+          exported[:additionalProperties] = false
+          exported.fetch(:properties).fetch(:window_size)[:additionalProperties] = false
+          request[:botasaurus] = exported
         end
 
         # @return [Hash{Symbol => Hash}] catalog under $defs.post_processors / $defs.extractors

data/lib/html2rss/config/validator.rb

@@ -45,17 +45,20 @@ module Html2rss
         optional(:media).maybe(:string)
       end
 
-      # Contract for Botasaurus-specific request options.
+      # Contract for Botasaurus-specific request options (OpenAPI ScrapeRequest minus url).
       BotasaurusRequestConfig = Dry::Schema.Params do
         config.validate_keys = true
 
-        optional(:execution_mode).filled(:string, included_in?: %w[auto request browser])
-        optional(:navigation_mode).filled(:string, included_in?: %w[auto get google_get google_get_bypass organic_get])
-        optional(:max_retries).filled(:integer, gteq?: 0, lteq?: 3)
+        botasaurus = Html2rss::RequestService::BotasaurusContract
+
+        optional(:execution_mode).filled(:string, included_in?: botasaurus::EXECUTION_MODES)
+        optional(:navigation_mode).filled(:string, included_in?: botasaurus::NAVIGATION_MODES)
+        optional(:max_retries).filled(:integer, gteq?: 0, lteq?: botasaurus::MAX_RETRIES)
         optional(:wait_for_selector).maybe(:string)
-        optional(:wait_timeout_seconds).filled(:integer, gt?: 0)
+        optional(:wait_timeout_seconds).filled(
+          :integer, gteq?: botasaurus::MIN_WAIT_TIMEOUT_SECONDS, lteq?: botasaurus::MAX_WAIT_TIMEOUT_SECONDS
+        )
         optional(:scroll).filled(:bool)
-        optional(:scroll_to_bottom).filled(:bool)
         optional(:block_images).filled(:bool)
         optional(:block_images_and_css).filled(:bool)
         optional(:block_trackers).filled(:bool)
@@ -63,18 +66,27 @@ module Html2rss
         optional(:headless).filled(:bool)
         optional(:proxy).filled(:string)
         optional(:user_agent).filled(:string)
-        optional(:window_size).value(:array, min_size?: 2, max_size?: 2).each(:integer, gt?: 0)
+        optional(:window_size).hash do
+          botasaurus::WINDOW_SIZE_PROPERTIES.each do |key|
+            required(key).filled(:integer, gt?: 0)
+          end
+        end
         optional(:lang).filled(:string)
         optional(:cookies).hash
         optional(:headers).hash
       end
 
+      # JSON Schema export adapter for +BotasaurusRequestConfig+ (params schemas have no json_schema).
+      class BotasaurusRequestExport < Dry::Validation::Contract
+        params(BotasaurusRequestConfig)
+      end
+
       # Contract for the top-level `request` section.
       RequestConfig = Dry::Schema.Params do
         optional(:max_redirects).filled(:integer, gteq?: 0)
         optional(:max_requests).filled(:integer, gt?: 0)
         optional(:total_timeout_seconds).filled(:integer, gt?: 0)
-        optional(:botasaurus).hash(BotasaurusRequestConfig)
+        optional(:botasaurus).hash
         optional(:local_file_path).filled(:string)
       end
 
@@ -123,6 +135,17 @@ module Html2rss
         errors.each { |error| key(:selectors).failure(error.text) } unless errors.empty?
       end
 
+      rule(request: :botasaurus) do
+        next unless value
+
+        result = BotasaurusRequestConfig.call(value)
+        next if result.success?
+
+        result.errors.each do |error|
+          key([:request, :botasaurus, *error.path]).failure(error.text)
+        end
+      end
+
       # URL validation delegated to Url class
       rule(:channel) do
         if (url_string = values.dig(:channel, :url)) && !url_string.empty?

data/lib/html2rss/config.rb

@@ -71,6 +71,34 @@ module Html2rss
         validate(load_yaml(file, feed_name, multiple_feeds_key:), params:)
       end
 
+      ##
+      # Serializes a configuration hash to string-key YAML.
+      #
+      # This is the single serializer for CLI capture and MCP +capture_config+.
+      #
+      # @param hash [Hash] configuration hash (symbol or string keys)
+      # @return [String] YAML document without Ruby symbol-key prefixes
+      def to_yaml(hash)
+        YAML.dump(HashUtil.deep_stringify_keys(hash))
+      end
+
+      ##
+      # Parses a YAML configuration string into a symbol-keyed hash.
+      #
+      # Does not validate. Call {validate} or {from_hash} after this.
+      #
+      # @param string [String] YAML document
+      # @return [Hash{Symbol => Object}] configuration hash
+      # @raise [ArgumentError] if +string+ is not a String or does not deserialize to a Hash
+      def from_yaml(string)
+        raise ArgumentError, 'YAML must be a String' unless string.is_a?(String)
+
+        parsed = YAML.safe_load(string)
+        raise ArgumentError, 'YAML must deserialize to a Hash' unless parsed.is_a?(Hash)
+
+        HashUtil.deep_symbolize_keys(parsed, context: 'config')
+      end
+
       ##
       # Loads the feed configuration from a YAML file.
       #

data/lib/html2rss/error.rb

@@ -7,7 +7,7 @@ module Html2rss
   # Raised when auto fallback exhausts all concrete tiers and extractors find no feed items.
   class NoFeedItemsExtracted < Error
     # Categories that append shared surface guidance to the empty-feed message.
-    SURFACE_HINT_CATEGORIES = %i[app_shell blocked_surface].freeze
+    SURFACE_HINT_CATEGORIES = %i[app_shell blocked_surface high_entropy_surface].freeze
 
     ##
     # @param attempts [Array<Hash{Symbol => Object}>] tier attempt diagnostics
@@ -27,17 +27,35 @@ module Html2rss
     private
 
     def build_message
+      [base_message, surface_guidance, botasaurus_guidance].compact.join(' ')
+    end
+
+    def base_message
       summaries = attempts.map do |attempt|
         details = attempt[:items_count].nil? ? "#{attempt[:error_class]} error" : "#{attempt[:items_count]} items"
         "#{attempt[:strategy]} (#{details})"
       end.join(', ')
 
-      message = "No feed items extracted after auto fallback across strategies: #{summaries}. " \
-                'Try a more specific listing URL or provide explicit selectors.'
-      return message unless SURFACE_HINT_CATEGORIES.include?(surface_category)
+      "No feed items extracted after auto fallback across strategies: #{summaries}. " \
+        'Try a more specific listing URL or provide explicit selectors.'
+    end
+
+    def surface_guidance
+      return unless SURFACE_HINT_CATEGORIES.include?(surface_category)
+
+      AutoSource::Scraper::NoScraperFound::CATEGORY_MESSAGES.fetch(surface_category)
+    end
+
+    def botasaurus_guidance
+      return unless botasaurus_configuration_error_attempt?
+      return if surface_guidance&.include?('BOTASAURUS_SCRAPER_URL')
+
+      RequestService::BotasaurusConfigurationError::EMPTY_FEED_HINT
+    end
 
-      guidance = AutoSource::Scraper::NoScraperFound::CATEGORY_MESSAGES.fetch(surface_category)
-      "#{message} #{guidance}"
+    def botasaurus_configuration_error_attempt?
+      error_name = RequestService::BotasaurusConfigurationError.name
+      attempts.any? { |attempt| attempt[:error_class] == error_name }
     end
   end
 end

data/lib/html2rss/feed_pipeline/README.md

@@ -0,0 +1,42 @@
+# FeedPipeline — `auto` request strategy
+
+`:auto` is the default request plan for feed builds (`auto_source`, `auto_json_feed`, Capture, and MCP `scrape_url` / `capture_config`). `FeedPipeline::StrategyPlan` resolves it; `FeedPipeline::AutoFallback` executes `AutoFallback::CHAIN`.
+
+Use `:auto` when you want Faraday first and a browser-backed hop only if that fetch fails or yields zero items. Pin a concrete strategy (`faraday`, `botasaurus`, `local_file`) when you need a single transport.
+
+## Chain
+
+`AutoFallback::CHAIN` is:
+
+1. **Faraday** — plain HTTP (faster, cheaper).
+2. **Botasaurus** — attempted when Faraday raises a fallback-eligible error (for example `BlockedSurfaceDetected` or timeout) or extracts zero feed items.
+
+There is no Browserless / Puppeteer-in-gem tier. Pin `botasaurus` when you want browser rendering without Faraday first. Botasaurus needs `BOTASAURUS_SCRAPER_URL`.
+
+## Surfaces
+
+| Surface | `:auto` behavior |
+|---------|------------------|
+| Gem / CLI feed build, MCP `scrape_url`, Capture | Full AutoFallback chain (`faraday` → `botasaurus`) |
+| MCP `inspect_url` | Cheap diagnostic: `StrategyPlan.concrete_for_diagnostic` maps `auto` to Faraday (pin `botasaurus` when you need browser rendering) |
+
+## Fallback vs abort
+
+These typically hop to the next chain member (among other `StandardError`s `AutoFallback` rescues):
+
+- `Html2rss::RequestService::BlockedSurfaceDetected`
+- `Html2rss::RequestService::RequestTimedOut`
+- Faraday connection / timeout errors
+- Empty extraction results when a later chain member may succeed
+
+These abort immediately (`AutoFallback::NON_FALLBACK_ERRORS`): unknown strategy, invalid URL, unsupported scheme, budget exceeded, private network denied, cross-origin follow-up denied, response too large.
+
+Retries share the feed's request/session policy. Pin a concrete strategy when you need a single-hop budget profile.
+
+## Success signals
+
+- `Html2rss::RequestService::Response` includes transport metadata for the strategy that produced it.
+- `Html2rss::Status` records selected strategy and attempt tallies for MCP envelope payload / CLI `--explain`.
+- Fallback hops log at info/warn (`AutoFallback`).
+
+See also {Html2rss::AutoSource} for article scraping (a different pipeline) and {Html2rss::Capture} for durable configs that stamp the selected strategy.