crawlforge-mcp-server 6.14.0 → 6.16.0

package/src/tools/advanced/BrowserSessionTool.js

@@ -111,8 +111,10 @@ const BrowserSessionSchema = z.object({
   timeout: z.number().min(10000).max(120000).default(30000),
 
   // Applies to the call it is sent on: on `open` to the first load, on `act` to
-  // every navigate in that call. It is never remembered by the session, so an
-  // override has to be repeated as deliberately as it was made.
+  // every navigate in that call, and on any call to wherever the page has moved
+  // since it was last checked (a click, a form, a script). It is never
+  // remembered by the session, so an override has to be repeated as
+  // deliberately as it was made. A URL that passed is not asked about again.
   respect_robots: z.boolean().optional(),
 
   // snapshot
@@ -358,6 +360,8 @@ export class BrowserSessionTool {
 
     let session;
     try {
+      // The load was gated; a script may have moved the page on since.
+      await this.actionExecutor.assertPageAllowed(page, browserOptions);
       session = this.store.create({
         ownerId,
         page,
@@ -371,8 +375,9 @@ export class BrowserSessionTool {
         maxPerOwner: ownerId.startsWith(REST_OWNER_PREFIX) ? REST_MAX_SESSIONS_PER_OWNER : undefined
       });
     } catch (error) {
-      // A cap refusal arrives with a live page in hand. Give it back before
-      // rethrowing, or the refused call leaks the context it just pinned.
+      // A cap refusal, or a page the gate refused, arrives with a live page in
+      // hand. Give it back before rethrowing, or the refused call leaks the
+      // context it just pinned.
       await releasePage();
       throw error;
     }
@@ -438,13 +443,29 @@ export class BrowserSessionTool {
     }
   }
 
+  /**
+   * Read from a session's page once the URL it stands on has passed the gate.
+   *
+   * A session's page outlives the call that loaded it, and between calls it
+   * can go anywhere: the last `act` clicked a link, or a script redirected.
+   * ActionExecutor.readGated holds it to the SSRF guard and the blocklist/
+   * robots gate on both sides of the read, with this call's `respect_robots`.
+   */
+  readPage(session, params, read) {
+    return this.actionExecutor.readGated(
+      session.page,
+      { respectRobots: params.respect_robots, tool: 'browser_session' },
+      read
+    );
+  }
+
   async snapshotSession(params, ownerId) {
     const session = this.requireSession(params, ownerId);
 
-    const snapshot = await captureSnapshot(session.page, {
+    const snapshot = await this.readPage(session, params, () => captureSnapshot(session.page, {
       interactiveOnly: params.interactive_only,
       maxNodes: params.max_nodes
-    });
+    }));
 
     this.store.touch(session, session.page.url());
     return { success: true, operation: 'snapshot', ...sessionInfo(session), snapshot };
@@ -473,9 +494,11 @@ export class BrowserSessionTool {
     }
 
     // Every `navigate` in here re-runs the SSRF guard and the blocklist/robots
-    // gate inside executeNavigateAction — verified, and the reason the gate is
-    // not repeated here. A long-lived session is a repeatable navigation
-    // primitive, so that per-hop check is what stops it becoming an SSRF hop.
+    // gate inside executeNavigateAction, and every action is bracketed by the
+    // same check on wherever the page has moved (executeGatedAction) — the
+    // reason the gate is not repeated here. A long-lived session is a
+    // repeatable navigation primitive, so those checks are what stop it
+    // becoming an SSRF hop.
     const result = await this.actionExecutor.executeActionsOnPage(session.page, params.actions, {
       continueOnError: params.continue_on_error,
       timeout: params.timeout,
@@ -503,8 +526,10 @@ export class BrowserSessionTool {
 
   async readSession(params, ownerId) {
     const session = this.requireSession(params, ownerId);
-    const url = session.page.url();
-    const html = await session.page.content();
+    const { url, html } = await this.readPage(session, params, async () => ({
+      url: session.page.url(),
+      html: await session.page.content()
+    }));
 
     const options = {};
     if (params.formats.includes('markdown')) options.outputFormat = 'markdown';
@@ -564,12 +589,12 @@ export class BrowserSessionTool {
   async screenshotSession(params, ownerId) {
     const session = this.requireSession(params, ownerId);
 
-    const shot = await this.actionExecutor.captureScreenshot(session.page, {
+    const shot = await this.readPage(session, params, () => this.actionExecutor.captureScreenshot(session.page, {
       fullPage: params.full_page,
       format: params.format,
       quality: params.quality,
       selector: params.selector
-    });
+    }));
 
     this.store.touch(session, session.page.url());
     // The actionId is what lets the server publish the image as a

package/src/tools/advanced/batchScrape/worker.js

@@ -10,6 +10,7 @@ import { config as appConfig } from '../../../constants/config.js';
 import { ssrfGuard, isSsrfError } from '../../../utils/ssrfGuard.js';
 import { noteRetryAfter } from '../../../utils/hostRateLimiter.js';
 import { preflightFetch } from '../../../utils/robotsGate.js';
+import { fetchResigned } from '../../../utils/resignedFetch.js';
 import { htmlToMarkdown } from '../../../utils/htmlToMarkdown.js';
 import { elementText } from '../../../utils/elementText.js';
 import { pageTitle } from '../../../utils/pageTitle.js';
@@ -31,9 +32,10 @@ export async function fetchUrl(url, options = {}) {
   const controller = new AbortController();
   const timeoutId = setTimeout(() => controller.abort(), timeout);
   try {
-    const response = await fetch(url, {
+    const response = await fetchResigned(url, {
       signal: controller.signal,
       headers: { ...gate.headers, ...headers },
+      onRedirect: gate.onRedirect,
       ...guard
     });
     if (response.status === 429 || response.status === 503) {

package/src/tools/basic/_fetch.js

@@ -8,6 +8,7 @@ import { config } from '../../constants/config.js';
 import { ssrfGuard, isSsrfError } from '../../utils/ssrfGuard.js';
 import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
 import { preflightFetch } from '../../utils/robotsGate.js';
+import { fetchResigned } from '../../utils/resignedFetch.js';
 
 /**
  * Fetch a URL with a configurable timeout and body-size cap.
@@ -51,12 +52,13 @@ export async function fetchWithTimeout(url, options = {}) {
   try {
     let response;
     try {
-      response = await fetch(url, {
+      response = await fetchResigned(url, {
         signal: controller.signal,
         headers: {
           ...gate.headers,
           ...headers
         },
+        onRedirect: gate.onRedirect,
         ...guard
       });
     } catch (error) {

package/src/tools/crawl/_sessionContext.js

@@ -223,7 +223,8 @@ export class SessionContext {
     const fetchOpts = {
       method,
       headers: requestHeaders,
-      redirect: 'follow'
+      redirect: 'follow',
+      onRedirect: gate.onRedirect
     };
 
     if (body) {

package/src/tools/crawl/mapSite.js

@@ -384,7 +384,8 @@ export class MapSiteTool {
     try {
       const response = await safeFetch(url, {
         signal: controller.signal,
-        headers: gate.headers
+        headers: gate.headers,
+        onRedirect: gate.onRedirect
       });
       clearTimeout(timeoutId);
       return response;

package/src/tools/extract/_fetchAndParse.js

@@ -147,6 +147,7 @@ export async function fetchAndParse(url, options = {}) {
       ...gate.headers,
       'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
     },
+    onRedirect: gate.onRedirect,
     signal: AbortSignal.timeout(timeoutMs)
   });
 

package/src/tools/extract/extractContent.js

@@ -176,7 +176,7 @@ export class ExtractContentTool {
             enableImages: options.preserveImageInfo,
             captureScreenshot: false
           }
-        });
+        }, gate);
 
         if (!browserResult.success) {
           throw new Error(`Browser processing failed: ${browserResult.error}`);
@@ -188,6 +188,7 @@ export class ExtractContentTool {
         // Simple HTTP fetch
         const response = await safeFetch(url, {
           headers: { ...gate.headers },
+          onRedirect: gate.onRedirect,
           signal: AbortSignal.timeout(15000)
         });
 

package/src/tools/extract/extractEmbeddedState.js

@@ -1,99 +1,348 @@
 /**
  * extract_embedded_state — return the JSON state a page already ships in its
- * own HTML: __NEXT_DATA__, RSC flight chunks (self.__next_f), __NUXT__,
- * __APOLLO_STATE__, __INITIAL_STATE__, __PRELOADED_STATE__ and
- * <script type="application/json"> blocks.
+ * own HTML: __NEXT_DATA__, RSC flight chunks (self.__next_f), __NUXT__ and
+ * Nuxt 3's __NUXT_DATA__, SvelteKit, __APOLLO_STATE__, __INITIAL_STATE__,
+ * __PRELOADED_STATE__, ytInitialData, Inertia, Shopify and
+ * <script type="application/json"> blocks. `find` returns the paths where a
+ * key lives instead of the data.
  *
  * One fetch, exact values, no LLM in the extraction path — the numbers come
  * from the site's own serialized state, so they cannot be fabricated.
+ *
+ * `escalate: true` (plan Phase 3.2) adds scrape's second stage: when the plain
+ * fetch is walled, the page is rendered by the same stealth escalation `scrape`
+ * uses (injected — this module never imports the browser) and the same parser
+ * runs on the rendered document. The browser also reads the framework globals
+ * off `window`, reported under `window_state`.
  */
 
+import { STATUS_CODES } from 'node:http';
+import { z } from 'zod';
 import { fetchAndParse } from './_fetchAndParse.js';
 // Both live in crawlforge-extractors so the REST API's extract_embedded_state
 // runs this exact reader — one RSC flight-stream parser, not two.
-import { extractEmbeddedState, selectJsonPath } from 'crawlforge-extractors';
+import { extractEmbeddedState, selectJsonPath, findJsonPaths } from 'crawlforge-extractors';
+import { stealthDocumentVerdict } from '../../utils/stealthVerdict.js';
+import { pageTitle } from '../../utils/pageTitle.js';
+import { setActualCost, markPreflightRefusal } from '../../server/requestContext.js';
+import { SCRAPE_ESCALATION_SHAPE, SCRAPE_ESCALATION_CREDITS, aBrowserMightPass } from '../scrape/escalation.js';
+import { WINDOW_STATE_NOTE } from '../../core/browser/windowState.js';
+
+/** The table price in AuthManager.getToolCost; escalation adds SCRAPE_ESCALATION_CREDITS. */
+const EMBEDDED_STATE_BASE_CREDITS = 2;
 
 // Above this, an unscoped result is big enough to be a problem for the caller
-// (context window, transport) rather than just large. Warn — never truncate:
-// a half-serialized object is worse than a big one, and `path` already gives
-// the caller an exact way to ask for less.
+// (context window, transport) rather than just large. Warn with a ready path:
+// over max_inline_chars the result comes back as a preview plus a
+// result_handle (src/server/inlineThreshold.js), and `path`, `keys_only` or
+// `find` ask for less up front.
 const LARGE_RESULT_BYTES = 256_000;
 
+// A wall, as opposed to a missing page or a broken server (G4: a 404 or a 5xx
+// never escalates — see aBrowserMightPass).
+const WALL_STATUSES = new Set([403, 429, 444]);
+
 const BLOCKED_HINT =
   'stealth_mode operation:"scrape" renders the page in a browser and returns the HTML; the embedded state is in the returned html. Do not repeat this call with a different path.';
 
+const ESCALATE_HINT =
+  'Call extract_embedded_state again with escalate:true - the stealth browser re-reads the page and the same parser runs on it (projected 7 credits, charged 2 if the plain fetch gets through). Do not repeat this call with a different path.';
+
+const STAGE_DID_NOT_RUN_HINT =
+  'The stealth stage did not run (see warnings) - retry once with escalate:true; stealth_mode drives the same browser, so it is not a way around this. Do not repeat this call with a different path.';
+
+/** The tool's own params, in tools/list order; server.js adds the compliance and inline-limit ones. */
+export const EMBEDDED_STATE_INPUT_SHAPE = {
+  url: z.string().url().describe('The URL to read embedded state from'),
+  path: z.string().optional().describe('Return only this subtree instead of the whole payload. Dotted keys and array indexes, e.g. "next_data.props.pageProps" or "next_f[0].f" — not JSONPath (no wildcards, filters or recursion). State payloads are routinely over a megabyte; scope them.'),
+  keys_only: z.boolean().optional().default(false).describe('Return `keys` instead of `data`: the first two levels of keys of the selected data (after `path`), each value replaced by its type ("object", "array(<n>)", "string", "number", "boolean", "null"); an array shows its length and its first item. Cheap discovery before choosing a path. Default: false'),
+  find: z.string().min(1).max(100).optional().describe('Return `matches` instead of `data`: every property with this key name (case-insensitive) anywhere in the selected data (after `path`), in document order, each as {path, preview} with the first 200 characters of its value - at most 50, with `matches_total` and `matches_truncated`. Each match path already starts with `path`, so it can be passed straight back as `path`. Discover where a field lives without downloading the payload. Cannot be combined with keys_only'),
+  raw: z.boolean().optional().default(false).describe('Also keep the undecoded __NUXT_DATA__ devalue array under json_scripts, beside the decoded nuxt_data. Default: false'),
+  escalate: z.boolean().optional().default(false).describe('When the plain fetch comes back blocked (403/429/challenge page, or an empty shell with no state), re-read the page once in the stealth browser and run the same parser on the rendered document; the browser also reads the framework globals off window (window_state). Under escalate_engine "auto" a Chrome TLS handshake with the honest CrawlForge User-Agent (impit) is tried first, and the browser runs only when that does not get the page. A 404 or 5xx never escalates. Projected at 2+5; the actual charge stays at 2 when the plain fetch succeeded. Default: false'),
+  escalate_engine: SCRAPE_ESCALATION_SHAPE.escalate_engine,
+  wait_for: z.number().min(0).max(30000).optional().describe('Escalated render only: extra wait after page load, in ms — for state assigned after DOMContentLoaded. Ignored without escalation')
+};
+
+/** "object" | "array(<n>)" | "string" | "number" | "boolean" | "null" */
+function label(value) {
+  if (value === null) return 'null';
+  if (Array.isArray(value)) return `array(${value.length})`;
+  return typeof value;
+}
+
 /**
- * @param {{ url: string, path?: string, user_agent?: string, respect_robots?: boolean }} params
+ * The keys of `value`, `depth` levels deep, every value replaced by its type.
+ * Shared shape with the REST route — keep the two in step.
+ * @param {unknown} value
+ * @param {number} depth
  */
-export async function extractEmbeddedStateHandler({ url, path, user_agent, respect_robots }) {
-  try {
-    // The raw `html` is used, not `$`: fetchAndParse strips <script> from the
-    // parsed tree by default, and every source here lives in a script tag.
-    const { html, finalUrl, warnings: fetchWarnings } = await fetchAndParse(url, {
-      userAgent: user_agent,
-      respectRobots: respect_robots,
-      tool: 'extract_embedded_state'
-    });
-
-    const state = extractEmbeddedState(html);
-    const warnings = [...fetchWarnings, ...state.warnings];
-
-    if (state.found.length === 0) {
-      warnings.push(
-        'No embedded state found. The page may render entirely on the client, or ship its data in a format this tool does not read.'
-      );
+export function describeKeys(value, depth) {
+  if (Array.isArray(value)) {
+    if (depth <= 1) return label(value);
+    return value.length > 0
+      ? { length: value.length, first: describeKeys(value[0], depth - 1) }
+      : { length: 0 };
+  }
+  if (value !== null && typeof value === 'object') {
+    const keys = {};
+    for (const [key, child] of Object.entries(value)) {
+      keys[key] = depth > 1 ? describeKeys(child, depth - 1) : label(child);
     }
+    return keys;
+  }
+  return label(value);
+}
 
-    // A path is naturally written against the payload ("props.pageProps"),
-    // not against this tool's envelope ("next_data.props.pageProps"). When
-    // the page carries exactly one payload and the path's root is not one of
-    // the envelope keys, read it inside that payload and say so (R21,
-    // 2026-09-09: four Next.js pages in a row failed on the bare path).
-    let effectivePath = path;
-    if (path && state.found.length === 1) {
-      const root = path.split(/[.[]/)[0];
-      if (root && !(root in state.data)) {
-        effectivePath = `${state.found[0].name}.${path}`;
+function httpError(status) {
+  return new Error(`HTTP ${status}: ${STATUS_CODES[status] ?? ''}`.trimEnd());
+}
+
+function isHttpError(status) {
+  return typeof status === 'number' && (status < 200 || status >= 300);
+}
+
+/**
+ * Whether a document is a failure for this tool. A named vendor's wall and an
+ * HTTP error always are. A page the verdict calls an empty shell or an error
+ * placeholder is one only when it carried no state: a client-rendered page
+ * that ships nothing but __NEXT_DATA__ is exactly what this tool reads. Shared
+ * rule with the REST route — keep the two in step.
+ */
+function failed(verdict, status, gotState) {
+  if (verdict.blocked || isHttpError(status)) return true;
+  return !verdict.success && !gotState;
+}
+
+/**
+ * @param {{ escalateFetch?: (args: { url: string, engine: string, respectRobots?: boolean, waitFor?: number, readWindowState?: boolean }) => Promise<object> }} [deps]
+ *   the stealth escalation stage (server.js `stealthEscalation`); without it
+ *   an escalation that would run is reported as unavailable
+ */
+export function createExtractEmbeddedStateHandler({ escalateFetch } = {}) {
+  return async function extractEmbeddedStateHandler({
+    url, path, keys_only, find, raw = false, escalate, escalate_engine = 'auto', wait_for, user_agent, respect_robots
+  }) {
+    // Each replaces `data` with something else, so asking for both has no
+    // answer. Nothing was fetched, so - like a robots refusal - it costs nothing.
+    if (find !== undefined && keys_only) {
+      markPreflightRefusal('BAD_REQUEST');
+      return {
+        content: [{
+          type: 'text',
+          text: 'Failed to extract embedded state: find and keys_only cannot be combined - find returns matches, keys_only returns keys.\nNext step: Call again with only one of find or keys_only.'
+        }],
+        isError: true
+      };
+    }
+    const escalating = escalate === true;
+    let escalationRan = false;
+    // One charge report for every return path: the projection is 2+5 when
+    // escalating, and the stealth stage's 5 is owed only if it ran (G4).
+    const reportCost = () => {
+      if (escalating) setActualCost(EMBEDDED_STATE_BASE_CREDITS + (escalationRan ? SCRAPE_ESCALATION_CREDITS : 0));
+    };
+    try {
+      // The raw `html` is used, not `$`: fetchAndParse strips <script> from the
+      // parsed tree by default, and every source here lives in a script tag.
+      // A non-2xx body is read too: a wall arrives as a 403 with the challenge
+      // in it, and the verdict below needs both to name the vendor.
+      const fetched = await fetchAndParse(url, {
+        userAgent: user_agent,
+        respectRobots: respect_robots,
+        tool: 'extract_embedded_state',
+        errorDocuments: true
+      });
+      let { html, finalUrl, status } = fetched;
+      const warnings = [...fetched.warnings];
+      let state = extractEmbeddedState(html, { raw });
+      let verdict = stealthDocumentVerdict(
+        { url: finalUrl, title: pageTitle(fetched.$), text: fetched.textContent, html, status },
+        { fetcher: 'a plain fetch', rendered: false, contentReturned: false }
+      );
+
+      let escalationFields = {};
+      let windowState = null;
+      if (failed(verdict, status, state.found.length > 0)) {
+        // A 404 or a 5xx is not a wall: it fails as it always has, and never
+        // escalates — the browser would spend 5 credits to be told the same.
+        if (!WALL_STATUSES.has(status) && !aBrowserMightPass(verdict, status)) throw httpError(status);
+        // Unescalated, a wall or an empty shell is an error naming the vendor
+        // (the verdict does) and the one parameter that changes the outcome.
+        if (!escalating) {
+          return {
+            content: [{ type: 'text', text: `Failed to extract embedded state: ${verdict.error}\nNext step: ${ESCALATE_HINT}` }],
+            isError: true
+          };
+        }
+        const vendorDetected = verdict.blocked?.vendor ?? null;
+        if (!escalateFetch) {
+          warnings.push('escalate: no stealth stage is wired into this server build');
+        } else {
+          try {
+            const stealth = await escalateFetch({
+              url: finalUrl,
+              engine: escalate_engine,
+              respectRobots: respect_robots,
+              waitFor: wait_for || 0,
+              readWindowState: true
+            });
+            escalationRan = true;
+            const engine = stealth.engine || escalate_engine;
+            if (Array.isArray(stealth.warnings)) warnings.push(...stealth.warnings);
+            html = stealth.html || '';
+            finalUrl = stealth.url || finalUrl;
+            status = stealth.status ?? null;
+            // The same parser, on the document the browser ended up with.
+            state = extractEmbeddedState(html, { raw });
+            verdict = stealthDocumentVerdict(
+              { url: finalUrl, title: stealth.title || '', text: stealth.text || '', html, status },
+              { waitedMs: stealth.gracedMs || 0, fetcher: 'the stealth browser', rendered: true, contentReturned: false }
+            );
+            windowState = stealth.windowState ?? null;
+            if (engine === 'impit') {
+              warnings.push('escalate: impit (a Chrome TLS handshake, no browser) got the page, so no JavaScript ran and window_state was not read');
+            }
+            const gotIt = !failed(verdict, status, state.found.length > 0 || Object.keys(windowState ?? {}).length > 0);
+            warnings.push(
+              `escalate: the plain fetch ${vendorDetected ? `was blocked by ${vendorDetected}` : 'did not return the page'}; ` +
+              `${engine === 'impit' ? 'a Chrome TLS handshake (impit, honest User-Agent)' : `the ${engine} stealth browser`} ${gotIt ? 'returned it' : 'did not get it either'}`
+            );
+            escalationFields = { stealth: { engine, vendor_detected: vendorDetected } };
+          } catch (err) {
+            // A robots refusal inside the gate stamps markPreflightRefusal, and
+            // withAuth then bills the whole call zero: nothing was rendered.
+            warnings.push(`escalate: the stealth retry did not run — ${err.message}`);
+          }
+        }
+      }
+      // Reported only to a caller who asked to escalate.
+      if (escalating) escalationFields = { escalated: escalationRan, ...escalationFields };
+
+      // Globals the served HTML already carried are not repeated: the parsed
+      // copy under its own name is the same object, and some are megabytes.
+      const windowFound = [];
+      if (windowState) {
+        const parsedVariables = new Set(state.found.map((f) => f.variable));
+        for (const [name, value] of Object.entries(windowState)) {
+          if (parsedVariables.has(name)) {
+            delete windowState[name];
+            warnings.push(`window_state: ${name} is not repeated - the served HTML already carried it (see found)`);
+            continue;
+          }
+          windowFound.push({ name, bytes: Buffer.byteLength(JSON.stringify(value)) });
+        }
+        if (windowFound.length > 0) state.data.window_state = windowState;
+      }
+
+      // Escalation asked for and still walled, with nothing to show for it. An
+      // error that says whether the stage ran: when it did, withAuth's hint
+      // does not send the caller to the stealth browser a second time.
+      if (failed(verdict, status, state.found.length > 0 || windowFound.length > 0)) {
+        reportCost();
+        return {
+          content: [{
+            type: 'text',
+            text: JSON.stringify({
+              success: false,
+              url: finalUrl,
+              ...(typeof status === 'number' ? { status } : {}),
+              error: verdict.error,
+              ...(verdict.blocked ? { blocked: verdict.blocked } : {}),
+              ...escalationFields,
+              found: state.found,
+              warnings,
+              ...(escalationRan ? {} : { next_step: STAGE_DID_NOT_RUN_HINT })
+            }, null, 2)
+          }],
+          isError: true
+        };
+      }
+
+      if (state.found.length === 0 && windowFound.length === 0) {
         warnings.push(
-          `path "${path}" was read as "${effectivePath}": "${state.found[0].name}" is the only payload on this page, so the path is resolved inside it.`
+          'No embedded state found. The page may render entirely on the client, or ship its data in a format this tool does not read.'
         );
       }
-    }
 
-    const data = effectivePath ? selectJsonPath(state.data, effectivePath) : state.data;
-    const bytes = Buffer.byteLength(JSON.stringify(data) ?? '');
+      // A path is naturally written against the payload ("props.pageProps"),
+      // not against this tool's envelope ("next_data.props.pageProps"). When
+      // the page carries exactly one payload and the path's root is not one of
+      // the envelope keys, read it inside that payload and say so (R21,
+      // 2026-09-09: four Next.js pages in a row failed on the bare path).
+      // `window_state` is an envelope key, so a path into it is left alone.
+      // `data_rows` is an index of next_f's rows, not a payload of its own.
+      const payloads = state.found.filter((f) => f.name !== 'data_rows');
+      let effectivePath = path;
+      if (path && payloads.length === 1) {
+        const root = path.split(/[.[]/)[0];
+        if (root && !(root in state.data)) {
+          effectivePath = `${payloads[0].name}.${path}`;
+          warnings.push(
+            `path "${path}" was read as "${effectivePath}": "${payloads[0].name}" is the only payload on this page, so the path is resolved inside it.`
+          );
+        }
+      }
+
+      const data = effectivePath ? selectJsonPath(state.data, effectivePath) : state.data;
+      const bytes = Buffer.byteLength(JSON.stringify(data) ?? '');
 
-    if (!path && bytes > LARGE_RESULT_BYTES) {
-      const largest = state.found.reduce((a, b) => (b.bytes > a.bytes ? b : a));
-      warnings.push(
-        `Result is ${bytes} bytes; "${largest.name}" alone is ${largest.bytes}. Re-run with path to scope it, e.g. path:"${largest.name}.${Object.keys(state.data[largest.name])[0]}".`
-      );
-    }
+      if (!path && !keys_only && find === undefined && bytes > LARGE_RESULT_BYTES) {
+        const largest = state.found.reduce((a, b) => (b.bytes > a.bytes ? b : a), state.found[0] ?? null);
+        if (largest) {
+          warnings.push(
+            `Result is ${bytes} bytes; "${largest.name}" alone is ${largest.bytes}. Re-run with path to scope it, e.g. path:"${largest.name}.${Object.keys(state.data[largest.name])[0]}", with keys_only:true to see its keys first, or with find:"<key>" to get the paths where a field of that name lives.`
+          );
+        }
+      }
 
-    return {
-      content: [{
-        type: 'text',
-        text: JSON.stringify({
-          url: finalUrl,
-          found: state.found,
-          path: effectivePath || null,
-          bytes,
-          data,
-          warnings
-        }, null, 2)
-      }]
-    };
-  } catch (error) {
-    // A block or server error is not a path problem: the generic hint
-    // ("call again without `path`") would send the caller back into the same
-    // wall. Naming the next step here keeps withAuth from appending it.
-    const blocked = /^HTTP (403|429|444|5\d\d)\b/.test(error.message);
-    return {
-      content: [{
-        type: 'text',
-        text: `Failed to extract embedded state: ${error.message}` + (blocked ? `\nNext step: ${BLOCKED_HINT}` : '')
-      }],
-      isError: true
-    };
-  }
+      // Match paths are relative to the selected data; prefixed with the path
+      // that selected it, each one can be passed straight back as `path`.
+      let search;
+      if (find !== undefined) {
+        search = findJsonPaths(data, find);
+        if (effectivePath) {
+          for (const match of search.matches) match.path = `${effectivePath}.${match.path}`;
+        }
+      }
+
+      reportCost();
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            url: finalUrl,
+            found: state.found,
+            path: effectivePath || null,
+            bytes,
+            ...escalationFields,
+            ...(windowFound.length > 0 ? { window_state: { note: WINDOW_STATE_NOTE, found: windowFound } } : {}),
+            ...(search
+              ? { matches: search.matches, matches_total: search.total, matches_truncated: search.truncated }
+              : (keys_only ? { keys: describeKeys(data, 2) } : { data })),
+            warnings
+          }, null, 2)
+        }]
+      };
+    } catch (error) {
+      reportCost();
+      // A block or server error is not a path problem: the generic hint
+      // ("call again without `path`") would send the caller back into the same
+      // wall. Naming the next step here keeps withAuth from appending it. A
+      // wall (here only one with a binary body, which fetchAndParse still
+      // throws on) points at escalate:true; a 5xx keeps the stealth_mode hint.
+      const hint = /^HTTP (403|429|444)\b/.test(error.message)
+        ? ESCALATE_HINT
+        : (/^HTTP 5\d\d\b/.test(error.message) ? BLOCKED_HINT : null);
+      return {
+        content: [{
+          type: 'text',
+          text: `Failed to extract embedded state: ${error.message}` + (hint ? `\nNext step: ${hint}` : '')
+        }],
+        isError: true
+      };
+    }
+  };
 }
+
+/** The handler with no escalation stage wired (tests; server.js wires its own). */
+export const extractEmbeddedStateHandler = createExtractEmbeddedStateHandler();