html2rss 0.27.2 → 0.29.1

data/lib/html2rss/mcp/server.rb

@@ -20,6 +20,68 @@ module Html2rss
       # Loopback bind for HTTP transport (local use only).
       HTTP_BIND_HOST = '127.0.0.1'
 
+      # Declarative MCP resource registrations consumed by {register_resources}.
+      RESOURCES = [
+        {
+          uri: 'html2rss://schema',
+          name: 'Configuration JSON Schema',
+          description: 'Full JSON Schema for html2rss feed configurations',
+          mime_type: 'application/json',
+          body: lambda {
+            [{ uri: 'html2rss://schema', mimeType: 'application/json',
+               text: Html2rss::Config.json_schema_json(pretty: true) }]
+          }
+        },
+        {
+          uri: 'html2rss://extractors',
+          name: 'Available Extractors',
+          description: 'Registered extractor names for selector configs ' \
+                       '(full option docs live in html2rss://schema $defs)',
+          mime_type: 'application/json',
+          body: lambda {
+            extractors = Html2rss::Selectors::Extractors::NAME_TO_CLASS.keys.map(&:to_s).sort
+            [{ uri: 'html2rss://extractors', mimeType: 'application/json',
+               text: JSON.pretty_generate(extractors) }]
+          }
+        },
+        {
+          uri: 'html2rss://strategies',
+          name: 'Available Strategies',
+          description: 'Published MCP request strategy names',
+          mime_type: 'application/json',
+          body: lambda {
+            [{ uri: 'html2rss://strategies', mimeType: 'application/json',
+               text: JSON.generate(Contract::STRATEGIES) }]
+          }
+        },
+        {
+          uri: 'html2rss://runtime',
+          name: 'Runtime capabilities',
+          description: 'Gem version, MCP contract version, catalog fingerprint, tool names, and Botasaurus config',
+          mime_type: 'application/json',
+          body: lambda {
+            [{ uri: 'html2rss://runtime', mimeType: 'application/json',
+               text: JSON.pretty_generate(Runtime.snapshot.to_h) }]
+          }
+        }
+      ].freeze
+
+      # Declarative MCP prompt registrations; SDK argument objects built at {register_prompts} time.
+      PROMPTS = [
+        {
+          name: 'scrape-webpage',
+          description: 'Guided one-shot scrape: one scrape call (auto already falls back)',
+          arguments: [{ name: 'url', description: 'URL to scrape', required: true }],
+          body: ->(args) { Outcome::Playbook.scrape_webpage_prompt(args.fetch(:url)) }
+        },
+        {
+          name: 'capture-feed-config',
+          description: 'Guided capture → test → apply; YAML draft plus catalog rewrite',
+          arguments: [{ name: 'url', description: 'URL to analyze', required: true }],
+          body: ->(args) { Outcome::Playbook.capture_feed_config_prompt(args.fetch(:url)) }
+        }
+      ].freeze
+
       class << self # rubocop:disable Metrics/ClassLength
         ##
         # Starts the MCP server with the given transport.
@@ -140,28 +202,8 @@ module Html2rss
                 "(#{error.message}). Install them or use --transport stdio."
         end
 
-        def instructions_text # rubocop:disable Metrics/MethodLength -- agent decision tree is the published contract
-          <<~TEXT.strip
-            html2rss MCP — decide which tool to call:
-
-            1. Need articles now (no saved config)? → scrape_url (1 call)
-               - strategy "auto" runs Faraday → Botasaurus AutoFallback. Do not retry with explicit faraday after auto.
-               - Empty scrape is still success (articles-now). Follow next_step / guidance (read_runtime if Botasaurus unset).
-            2. Need a reusable feed YAML? → capture_config → validate_config → apply_config
-               - capture_config returns YAML inside payload.yaml. Draft only: if destination is html2rss-configs, rewrite for directory.topics and explicit channel title/url. Strive enhance: true (false only when chrome leaks).
-               - validate_config / apply_config accept config hash XOR yaml string.
-               - apply_config isError when zero RSS items (ship gate). Confirm payload.item_count.
-            3. Weak scrape/capture or recon (final URL, status, https→http, rel=alternate feeds)? → inspect_url only if weak or recon.
-            4. Have a config already? → validate_config (must succeed) → apply_config
-            5. Schema / extractors / strategies / runtime → resources html2rss://schema|extractors|strategies|runtime
-
-            Prefer capture_config for durable config; scrape_url for one-shot extraction.
-            Follow envelope next_step and guidance. Botasaurus needs BOTASAURUS_SCRAPER_URL in this process env (boolean at html2rss://runtime; the URL is never returned).
-          TEXT
-        end
-
-        def botasaurus_configured?
-          !ENV['BOTASAURUS_SCRAPER_URL'].to_s.strip.empty?
+        def instructions_text
+          Outcome::Playbook.instructions
         end
 
         def tool_error_response(error)
@@ -175,7 +217,7 @@ module Html2rss
           tool_error_response(error)
         end
 
-        # rubocop:disable Metrics/MethodLength -- listing fields stay together
+        # rubocop:disable-next Metrics/MethodLength -- listing fields stay together
         def define_envelope_tool(server, name:, description:, input_schema:,
                                  annotations: Contract::ANNOTATIONS_OPEN_WORLD)
           run = method(:handle_tool_call)
@@ -190,201 +232,38 @@ module Html2rss
             run.call { yield(**kwargs) }
           end
         end
-        # rubocop:enable Metrics/MethodLength
 
         def register_tools(server)
-          register_scrape_url(server)
-          register_inspect_url(server)
-          register_capture_config(server)
-          register_validate_config(server)
-          register_apply_config(server)
-        end
-
-        def register_scrape_url(server)
-          define_envelope_tool(
-            server,
-            name: 'scrape_url',
-            description: 'One-shot article extraction as JSON Feed items. ' \
-                         'Use when you need articles now without a saved config. ' \
-                         'strategy "auto" triggers fallback chain (faraday → botasaurus) for JS-rendered sites.',
-            input_schema: Contract::SCRAPE_INPUT_SCHEMA
-          ) do |server_context:, url:, strategy: 'auto', limit: 25, items_selector: nil| # rubocop:disable Lint/UnusedBlockArgument
-            scrape_outcome(url:, strategy:, limit:, items_selector:)
-          end
-        end
-
-        def scrape_outcome(url:, strategy:, limit:, items_selector:)
-          plan = (strategy || :auto).to_sym
-          feed_result = Html2rss.auto_feed_result(url, strategy: plan, limit:, items_selector:)
-          feed = feed_result.to_json_feed
-          Outcome.scrape(
-            items: feed[:items] || [],
-            requested_strategy: plan,
-            channel_title: feed[:title],
-            admission_drops: feed_result.status.admission_drops,
-            botasaurus_configured: botasaurus_configured?
-          )
-        end
-
-        def register_inspect_url(server)
-          define_envelope_tool(
-            server,
-            name: 'inspect_url',
-            description: 'Diagnostic page analysis (scrapers, SST, segments) plus recon: ' \
-                         'final_url, status, scheme_downgrade, rel=alternate RSS/Atom feeds. ' \
-                         'Use when scrape/capture is weak or you need those recon facts.',
-            input_schema: Contract::INSPECT_INPUT_SCHEMA
-          ) do |server_context:, url:, strategy: 'auto'| # rubocop:disable Lint/UnusedBlockArgument
-            Outcome.inspect(payload: Inspect.call(url:, strategy:))
-          end
-        end
-
-        def register_capture_config(server) # rubocop:disable Metrics/MethodLength
-          define_envelope_tool(
-            server,
-            name: 'capture_config',
-            description: 'Derive a reusable html2rss feed config from a URL. ' \
-                         'Use when the goal is a durable YAML (then validate_config). ' \
-                         'Returns YAML inside payload.yaml (same serializer as CLI capture). ' \
-                         'Draft only — catalog feeds still need directory.topics and title/url; ' \
-                         'strive enhance: true. Full schema options live in resource html2rss://schema.',
-            input_schema: Contract::CAPTURE_INPUT_SCHEMA
-          ) do |server_context:, url:, strategy: 'auto', items_selector: nil| # rubocop:disable Lint/UnusedBlockArgument
-            capture_outcome(url:, strategy:, items_selector:)
-          end
-        end
-
-        def capture_outcome(url:, strategy:, items_selector:) # rubocop:disable Metrics/MethodLength -- CaptureResult maps 1:1 onto Outcome
-          plan = (strategy || :auto).to_sym
-          result = Html2rss::Capture.build(url, strategy: plan, items_selector:)
-          Outcome.capture(
-            yaml: Config.to_yaml(result.config),
-            articles_count: result.articles_count,
-            has_selectors: result.has_selectors,
-            channel_title: result.channel_title,
-            requested_strategy: plan,
-            segment_strategy: result.segment_strategy,
-            selected_strategy: result.selected_strategy,
-            admission_drops: result.admission_drops
-          )
-        end
-
-        def register_validate_config(server) # rubocop:disable Metrics/MethodLength -- description is the published contract
-          define_envelope_tool(
-            server,
-            name: 'validate_config',
-            description: 'Validate a feed config hash XOR yaml string against the html2rss JSON schema. ' \
-                         'Call before apply_config. Failures return isError with payload.errors. ' \
-                         'Full schema lives in resource html2rss://schema.',
-            input_schema: Contract::CONFIG_XOR_SCHEMA,
-            annotations: Contract::ANNOTATIONS_VALIDATE
-          ) do |server_context:, config: nil, yaml: nil| # rubocop:disable Lint/UnusedBlockArgument
-            validate_outcome(config:, yaml:)
-          end
-        end
-
-        def validate_outcome(config:, yaml:)
-          validation = Html2rss::Config.validate(ConfigArgument.parse(config:, yaml:).config)
-          Outcome.validate(errors: validation.success? ? nil : validation.errors.to_h)
-        end
-
-        def register_apply_config(server)
-          define_envelope_tool(
-            server,
-            name: 'apply_config',
-            description: 'Apply a validated feed config (hash XOR yaml) and return RSS XML in payload.rss. ' \
-                         'isError when the feed has zero items (ship gate). payload.item_count is RSS item count. ' \
-                         'Use after validate_config succeeds.',
-            input_schema: Contract::APPLY_INPUT_SCHEMA
-          ) do |server_context:, url:, config: nil, yaml: nil| # rubocop:disable Lint/UnusedBlockArgument
-            apply_outcome(url:, config:, yaml:)
+          Tools.register_all(server, registrar: method(:define_envelope_tool))
+        end
+
+        def register_resources(server)
+          RESOURCES.each do |entry|
+            server.define_resource(
+              uri: entry[:uri],
+              name: entry[:name],
+              description: entry[:description],
+              mime_type: entry[:mime_type]
+            ) do |server_context: nil| # rubocop:disable Lint/UnusedBlockArgument
+              entry[:body].call
+            end
           end
         end
 
-        def apply_outcome(url:, config:, yaml:)
-          feed_config = ConfigArgument.parse(config:, yaml:).config
-          feed_config[:channel] ||= {}
-          feed_config[:channel][:url] ||= url
-          feed_result = Html2rss.feed_result(feed_config)
-          rss = feed_result.to_rss
-          Outcome.apply(rss: rss.to_s, item_count: rss.items.size, empty: feed_result.empty?)
-        end
-
-        def register_resources(server) # rubocop:disable Metrics/MethodLength
-          configured = method(:botasaurus_configured?)
-          server.define_resource(
-            uri: 'html2rss://schema',
-            name: 'Configuration JSON Schema',
-            description: 'Full JSON Schema for html2rss feed configurations',
-            mime_type: 'application/json'
-          ) do |server_context: nil| # rubocop:disable Lint/UnusedBlockArgument
-            schema = Html2rss::Config.json_schema_json(pretty: true)
-            [{ uri: 'html2rss://schema', mimeType: 'application/json', text: schema }]
-          end
-
-          server.define_resource(
-            uri: 'html2rss://extractors',
-            name: 'Available Extractors',
-            description: 'Registered extractor names for selector configs ' \
-                         '(full option docs live in html2rss://schema $defs)',
-            mime_type: 'application/json'
-          ) do |server_context: nil| # rubocop:disable Lint/UnusedBlockArgument
-            extractors = Html2rss::Selectors::Extractors::NAME_TO_CLASS.keys.map(&:to_s).sort
-            [{ uri: 'html2rss://extractors', mimeType: 'application/json',
-               text: JSON.pretty_generate(extractors) }]
-          end
-
-          server.define_resource(
-            uri: 'html2rss://strategies',
-            name: 'Available Strategies',
-            description: 'Published MCP request strategy names',
-            mime_type: 'application/json'
-          ) do |server_context: nil| # rubocop:disable Lint/UnusedBlockArgument
-            [{ uri: 'html2rss://strategies', mimeType: 'application/json',
-               text: JSON.generate(Contract::STRATEGIES) }]
-          end
-
-          server.define_resource(
-            uri: 'html2rss://runtime',
-            name: 'Runtime capabilities',
-            description: 'Whether optional transports are configured in this MCP process (never leaks secrets)',
-            mime_type: 'application/json'
-          ) do |server_context: nil| # rubocop:disable Lint/UnusedBlockArgument
-            [{ uri: 'html2rss://runtime', mimeType: 'application/json',
-               text: JSON.pretty_generate(botasaurus_configured: configured.call) }]
-          end
-        end
-
-        def register_prompts(server)
-          register_scrape_webpage_prompt(server)
-          register_capture_feed_config_prompt(server)
-        end
-
-        def register_scrape_webpage_prompt(server) # rubocop:disable Metrics/MethodLength -- SDK prompt types stay together
+        def register_prompts(server) # rubocop:disable Metrics/MethodLength -- prompt argument mapping
           to_result = method(:prompt_result)
-          text_for = method(:scrape_webpage_text)
-          server.define_prompt(
-            name: 'scrape-webpage',
-            description: 'Guided one-shot scrape: one scrape_url call (auto already falls back)',
-            arguments: [
-              ::MCP::Prompt::Argument.new(name: 'url', description: 'URL to scrape', required: true)
-            ]
-          ) do |args, server_context:| # rubocop:disable Lint/UnusedBlockArgument
-            to_result.call(text_for.call(args.fetch(:url)))
-          end
-        end
-
-        def register_capture_feed_config_prompt(server) # rubocop:disable Metrics/MethodLength -- SDK prompt types stay together
-          to_result = method(:prompt_result)
-          text_for = method(:capture_feed_config_text)
-          server.define_prompt(
-            name: 'capture-feed-config',
-            description: 'Guided capture → validate → apply; YAML draft plus catalog rewrite',
-            arguments: [
-              ::MCP::Prompt::Argument.new(name: 'url', description: 'URL to analyze', required: true)
-            ]
-          ) do |args, server_context:| # rubocop:disable Lint/UnusedBlockArgument
-            to_result.call(text_for.call(args.fetch(:url)))
+          PROMPTS.each do |entry|
+            arguments = entry[:arguments].map do |spec|
+              ::MCP::Prompt::Argument.new(name: spec[:name], description: spec[:description],
+                                          required: spec.fetch(:required, false))
+            end
+            server.define_prompt(
+              name: entry[:name],
+              description: entry[:description],
+              arguments:
+            ) do |args, server_context:| # rubocop:disable Lint/UnusedBlockArgument
+              to_result.call(entry[:body].call(args))
+            end
           end
         end
 
@@ -395,26 +274,6 @@ module Html2rss
             ]
           )
         end
-
-        def scrape_webpage_text(url)
-          <<~MSG.strip
-            Scrape #{url} with scrape_url (strategy auto). One call is enough — auto already runs Faraday then Botasaurus.
-            Follow envelope next_step and guidance. Call inspect_url only if articles are empty/weak or you need recon (final_url, status, scheme_downgrade, alternate_feeds).
-            Do not retry scrape_url with explicit faraday after auto. Read html2rss://runtime if next_step is read_runtime.
-            Return payload.items (not a raw JSON array).
-          MSG
-        end
-
-        def capture_feed_config_text(url)
-          <<~MSG.strip
-            Build a reusable html2rss feed config for #{url}:
-            1) capture_config — YAML is payload.yaml. Check payload.articles_count and payload.has_selectors. Strive to keep enhance: true (false only when chrome leaks into items).
-            2) Follow next_step. If weak or you need recon, inspect_url. Auto already hops to Botasaurus; do not retry capture with botasaurus unless Faraday was blocked.
-            3) validate_config with yaml (or config hash) — must not be isError
-            4) apply_config — isError if zero items. Confirm payload.item_count before shipping.
-            If the destination is html2rss-configs, rewrite the draft for directory.topics and explicit channel title/url. Return YAML.
-          MSG
-        end
       end
     end
   end

data/lib/html2rss/mcp.rb

@@ -3,6 +3,9 @@
 module Html2rss
   ##
   # MCP server for AI client consumption.
+  #
+  # {include:file:lib/html2rss/mcp/README.md}
+  #
   # Lazy-loads the mcp gem; no cost when the server is not started.
   module MCP
     class << self

data/lib/html2rss/page_recon/README.md

@@ -0,0 +1,69 @@
+# PageRecon — inspect diagnostics
+
+`PageRecon::Diagnostics` powers the **inspect** verb (CLI, MCP, `Html2rss.inspect`). It fetches once via `PageRecon.probe`, classifies the surface, counts cheap AutoSource articles, and reports redirect facts. Full curation contract: `CONTEXT.md` § Frozen contract.
+
+## What inspect reports
+
+| Field              | Meaning                                                                     |
+| ------------------ | --------------------------------------------------------------------------- |
+| `requested_url`    | URL you passed in                                                           |
+| `final_url`        | URL after redirects (may differ from requested)                             |
+| `status`           | HTTP status of the **final** response                                       |
+| `scheme_downgrade` | `true` when HTTPS entry landed on HTTP                                      |
+| `alternate_feeds`  | `rel=alternate` RSS/Atom links found in HTML                                |
+| `surface_category` | AutoSource surface class (`high_entropy_surface`, `unsupported_surface`, …) |
+| `articles_count`   | Cheap AutoSource extract (limit 10) — diagnostic only, not ship quality     |
+| `strategy`         | Concrete transport used (`inspect` maps `auto` → Faraday)                   |
+
+CLI text output omits `requested_url`; it prints the requested URL as the card title. The `Final:` line appears **only when** `final_url` differs from `requested_url`:
+
+```text
+https://apex.example/
+        Final:    https://www.example/ (HTTP 200)
+        Surface:  high_entropy_surface (10 articles)
+```
+
+That `Final:` line means the redirect **did** happen — not that inspect stopped at the apex host.
+
+## Apex → www redirects
+
+Some publishers redirect apex domains to `www` (301). html2rss follows redirects and records the landing URL in `final_url`.
+
+Observed behavior:
+
+```bash
+bin/html2rss inspect https://apex.example
+# Final: https://www.example/ (HTTP 200), high_entropy_surface, articles
+
+bin/html2rss inspect https://www.example
+# high_entropy_surface, articles (no Final: line — requested equals final)
+```
+
+Cross-host redirects (e.g. `apex.example` → `www.example`) no longer pin a stale `Host` header from the entry URL. `Config::RequestHeaders` omits `Host` by default; Faraday/Net::HTTP sets it per hop from the current request URL.
+
+When the first streamed fetch loses the redirect payload (empty body at the final URL), `FaradayStrategy` retries once without streaming so inspect/recon see the same HTML as a direct canonical URL fetch.
+
+### What to do
+
+1. **Prefer the canonical URL** — if you know the site lives on `www`, pass that URL to inspect, recon, capture, and scrape.
+2. **Read the `Final:` line** — when `final_url` differs from what you typed and status is 4xx, retry inspect on `final_url` before assuming the site is unreachable.
+3. **Do not treat apex 403 as “redirect skipped”** — check JSON output (`--format json`) for `requested_url` vs `final_url` when text output is ambiguous.
+4. **Blocked surfaces** — if the canonical URL still fails, try `strategy: botasaurus` on inspect (MCP) or escalate to recon/capture with browser strategy; Faraday-only inspect is intentionally cheap.
+
+## inspect ≠ recon
+
+| Verb    | Adds beyond diagnostics                                              |
+| ------- | -------------------------------------------------------------------- |
+| inspect | Scraper eligibility, XHR hints (Botasaurus), surface + article count |
+| recon   | Verdict (`:build` / `:defer` / `:drop`), native feed preference      |
+
+Follow golden-path `next_step` from MCP envelopes; do not call recon when inspect already answers the question.
+
+## Ownership
+
+| Concern                             | Owner                                                                |
+| ----------------------------------- | -------------------------------------------------------------------- |
+| Diagnostic fetch + assess           | `PageRecon::Diagnostics` → `PageRecon.probe`                         |
+| Surface class + cheap article count | `PageRecon::Assessment`                                              |
+| Redirect follow + terminal retry    | `RequestService::FaradayStrategy`                                    |
+| Outbound header normalization       | `Config::RequestHeaders` (no default `Host`; explicit override only) |

data/lib/html2rss/page_recon/diagnostics.rb

@@ -0,0 +1,211 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class PageRecon
+    ##
+    # Diagnostic inspect path (not Capture or Recon ownership). Fetches via {.probe},
+    # then adds scraper/XHR diagnostics for curation inspect surfaces.
+    module Diagnostics # rubocop:disable Metrics/ModuleLength -- diagnostic wire fields stay co-located
+      ##
+      # Typed diagnostic report for inspect wire payloads and Outcome policy.
+      Report = Data.define(:data) do
+        ##
+        # @return [Boolean]
+        def alternate_feeds?
+          Array(data[:alternate_feeds]).any?
+        end
+
+        ##
+        # @return [Integer]
+        def articles_count
+          data[:articles_count].to_i
+        end
+
+        ##
+        # @return [Hash{Symbol => Object}]
+        def to_wire_h
+          data
+        end
+      end
+
+      # Minimum HTML body size to treat zero-article weak surfaces as likely JS shells.
+      JS_SHELL_MIN_BODY_BYTES = 8_192
+      private_constant :JS_SHELL_MIN_BODY_BYTES
+
+      module_function
+
+      ##
+      # @param url [String]
+      # @param strategy [String, Symbol]
+      # @param deep [Boolean] when true and strategy is auto, one Botasaurus hop if configured
+      # @return [Report]
+      def call(url:, strategy: :auto, deep: false)
+        probe = PageRecon.probe(url, strategy: resolve_inspect_strategy(strategy, deep:))
+        recon = probe.result
+        response = probe.response
+
+        Report.new(data: build_data(probe, recon, response))
+      end
+
+      ##
+      # Runs diagnostic inspect across URLs with per-URL error isolation.
+      #
+      # @param urls [Enumerable<String>]
+      # @param strategy [Symbol, String]
+      # @param concurrency [Integer]
+      # @return [Array<Report>]
+      def batch(urls:, strategy: :auto, concurrency: Batch::DEFAULT_CONCURRENCY)
+        Batch.map(Array(urls), concurrency:) do |url|
+          call(url:, strategy:)
+        rescue StandardError => error
+          error_report(url, error)
+        end
+      end
+
+      ##
+      # @param response [Html2rss::RequestService::Response]
+      # @return [Hash] redacted XHR capture diagnostics (no query strings)
+      def xhr_capture_info(response)
+        captured = response.captured_responses
+        {
+          count: captured.size,
+          sample_endpoints: captured.first(5).filter_map { |entry| redacted_endpoint(entry) },
+          candidate_articles: captured.any? { |entry| xhr_candidate_articles?(entry) }
+        }
+      end
+
+      ##
+      # @param entry [Hash] captured response hash
+      # @return [String, nil] scheme+host+path only
+      def redacted_endpoint(entry)
+        raw = entry['url'] || entry[:url]
+        return unless raw
+
+        uri = URI.parse(raw.to_s)
+        return unless uri.scheme && uri.host
+
+        "#{uri.scheme}://#{uri.host}#{uri.path}"
+      rescue URI::InvalidURIError
+        nil
+      end
+
+      ##
+      # @param entry [Hash] captured response hash
+      # @return [Boolean]
+      def xhr_candidate_articles?(entry)
+        body = entry['body'] || entry[:body]
+        return false unless body.is_a?(String)
+
+        document = JSON.parse(body, symbolize_names: true)
+        AutoSource::Scraper::JsonState::CandidateDetector.candidate_array?(document)
+      rescue JSON::ParserError
+        false
+      end
+
+      ##
+      # @param parsed [Object] parsed response body
+      # @return [Array<String>, Hash]
+      def scraper_info(parsed)
+        return { error: 'Response is not HTML' } unless parsed.is_a?(Nokogiri::HTML::Document)
+
+        begin
+          Html2rss::AutoSource::Scraper.from(parsed).map(&:name)
+        rescue Html2rss::AutoSource::Scraper::NoScraperFound => error
+          { none_found: error.category.to_s }
+        end
+      end
+
+      def build_data(probe, recon, response)
+        data = recon.to_h.merge(
+          strategy: probe.strategy,
+          scraper_eligibility: scraper_info(safe_parsed_body(response)),
+          html_present: html_present?(recon, response),
+          likely_js_shell: likely_js_shell?(recon, response),
+          redirect_summary: redirect_summary(recon)
+        )
+        data[:xhr_capture] = xhr_capture_info(response) if probe.strategy == :botasaurus
+        log_js_shell(data, response.body&.bytesize.to_i) if data[:likely_js_shell]
+        data
+      end
+      module_function :build_data
+      private_class_method :build_data
+
+      def resolve_inspect_strategy(strategy, deep:)
+        name = (strategy || :auto).to_sym
+        return :botasaurus if deep && name == :auto && MCP::Runtime.botasaurus_configured?
+
+        name
+      end
+      module_function :resolve_inspect_strategy
+      private_class_method :resolve_inspect_strategy
+
+      def html_present?(recon, response)
+        recon.html_response && !response.body.to_s.empty?
+      end
+      module_function :html_present?
+      private_class_method :html_present?
+
+      def likely_js_shell?(recon, response)
+        return false unless html_present?(recon, response)
+        return false if recon.articles_count.positive?
+        return false if recon.blocked_surface || recon.surface_category == :blocked_surface
+
+        return true if recon.surface_category == :app_shell
+
+        response.body.bytesize >= JS_SHELL_MIN_BODY_BYTES &&
+          SurfaceCategory.coerce(recon.surface_category).weak?
+      end
+      module_function :likely_js_shell?
+      private_class_method :likely_js_shell?
+
+      def redirect_summary(recon)
+        {
+          requested_url: recon.requested_url,
+          final_url: recon.final_url,
+          status: recon.status,
+          scheme_downgrade: recon.scheme_downgrade
+        }
+      end
+      module_function :redirect_summary
+      private_class_method :redirect_summary
+
+      def log_js_shell(data, body_bytesize)
+        Log.debug(
+          "Diagnostics js_shell: bytesize=#{body_bytesize} surface_category=#{data[:surface_category]}"
+        )
+      end
+      module_function :log_js_shell
+      private_class_method :log_js_shell
+
+      def safe_parsed_body(response)
+        return unless response.html_response?
+
+        response.parsed_body
+      rescue RequestService::UnsupportedResponseContentType
+        nil
+      end
+      module_function :safe_parsed_body
+      private_class_method :safe_parsed_body
+
+      def error_report(url, error) # rubocop:disable Metrics/MethodLength -- error hash mirrors success report shape
+        Report.new(
+          data: {
+            requested_url: url.to_s,
+            final_url: url.to_s,
+            status: nil,
+            scheme_downgrade: false,
+            alternate_feeds: [],
+            surface_category: :unsupported_surface,
+            articles_count: 0,
+            html_response: false,
+            content_type: nil,
+            strategy: nil,
+            scraper_eligibility: { error: "#{error.class} - #{error.message}" }
+          }
+        )
+      end
+      module_function :error_report
+      private_class_method :error_report
+    end
+  end
+end