crawlforge-mcp-server 6.14.0 → 6.16.0

package/src/core/StealthBrowserManager.js

@@ -17,11 +17,12 @@ import path from 'path';
 import { createRequire } from 'module';
 import HumanBehaviorSimulator from '../utils/HumanBehaviorSimulator.js';
 import { BrowserContextPool } from './BrowserContextPool.js';
-import { safeGoto } from '../utils/ssrfGuard.js';
+import { safeGoto, assertNavigationAllowed } from '../utils/ssrfGuard.js';
 import { looksLikeInterstitial, detectChallengePage } from '../utils/challengeDetection.js';
 import { guardFirefoxPageErrors } from '../utils/firefoxPageErrorGuard.js';
 import { serverStealthProxies } from '../constants/config.js';
 import { ClearanceJar, sharedClearanceJar } from './ClearanceJar.js';
+import { readWindowState } from './browser/windowState.js';
 
 // Grace given to a document that rendered no title and no text (see _waitOutEmptyDocument).
 export const EMPTY_DOCUMENT_GRACE_MS = 8000;
@@ -2844,7 +2845,16 @@ export class StealthBrowserManager {
     return this._clickTurnstile(page);
   }
 
-  async scrapeWithStealth({ url, engine, wait_for = 0, screenshot = false, stealthConfig = {} } = {}) {
+  /**
+   * @param {object} options
+   * @param {boolean} [options.readWindowState] also read the framework globals
+   *   off `window` once the page has settled (extract_embedded_state's
+   *   escalation, src/core/browser/windowState.js); returned as `windowState`
+   * @param {(to: string) => Promise<void>} [options.onRedirect] the caller's
+   *   gate for wherever the page is redirected to (robotsGate.js
+   *   `redirectGate`); a refusal throws and nothing of the page is returned
+   */
+  async scrapeWithStealth({ url, engine, wait_for = 0, screenshot = false, stealthConfig = {}, readWindowState: wantWindowState = false, onRedirect } = {}) {
     if (!url) throw new Error('scrapeWithStealth requires a url');
 
     const { contextId, engineFallbackWarning = null } = await this.createStealthContext({ ...stealthConfig, engine });
@@ -2870,7 +2880,8 @@ export class StealthBrowserManager {
       });
       // SSRF guard at the navigation boundary: the stealth engine resolves DNS
       // itself, so a URL/host-only check would miss rebinding to private/metadata IPs.
-      const response = await safeGoto(page, url, { waitUntil: 'domcontentloaded' });
+      const response = await safeGoto(page, url, { waitUntil: 'domcontentloaded', onRedirect });
+      const landedUrl = page.url();
       if (response && status === null) status = response.status();
       if (wait_for > 0) await page.waitForTimeout(wait_for);
       // Cloudflare's and Vercel's JavaScript challenges solve themselves in a
@@ -2888,6 +2899,10 @@ export class StealthBrowserManager {
       // a successful scrape.
       const renderMs = await this._settleRender(page);
       const gracedMs = emptyGraceMs + renderMs;
+      // The waits above are time in which a challenge or a client-side redirect
+      // can move the page on. The document read below is the one that has to
+      // pass, so wherever the page stands now is checked like a redirect hop.
+      await assertNavigationAllowed(page, landedUrl, null, onRedirect);
 
       // A failure to read the document is a failure. With every read wrapped
       // in .catch(() => ''), a renderer that crashed or a page closed during
@@ -2913,6 +2928,12 @@ export class StealthBrowserManager {
       const shot = screenshot
         ? await page.screenshot({ encoding: 'base64', fullPage: false }).catch(() => null)
         : null;
+      let windowState;
+      if (wantWindowState) {
+        const read = await readWindowState(page);
+        windowState = read.state;
+        warnings.push(...read.warnings);
+      }
 
       // Still a wall after everything above: whatever clearance this context
       // carried for the site did not work, so closeContext drops it instead of
@@ -2921,7 +2942,7 @@ export class StealthBrowserManager {
         this._markBlocked(contextId, [url, page.url()]);
       }
 
-      return { success: true, url, title, text, html, screenshot: shot, status, gracedMs, engine: engineUsed, warnings };
+      return { success: true, url, title, text, html, screenshot: shot, status, gracedMs, engine: engineUsed, warnings, ...(windowState ? { windowState } : {}) };
     } finally {
       await this.closeContext(contextId).catch(() => {});
     }

package/src/core/browser/actionQueue.js

@@ -57,7 +57,10 @@ export function createActionQueue({
         'CRAWLFORGE_MAX_ACTION_SESSIONS / CRAWLFORGE_ACTION_QUEUE_TIMEOUT_MS if the machine has room.'
       ));
     }, timeoutMs);
-    timer.unref?.();
+    // Not unref()'d: a caller is awaiting this wait, so it must keep the event
+    // loop alive. Unref'd, Node 22 could exit mid-wait with the promise still
+    // pending (it did, under node:test in CI). The timer never outlives the
+    // wait — it is cleared when the job starts.
 
     return queue.add(() => {
       clearTimeout(timer);

package/src/core/browser/windowState.js

@@ -0,0 +1,88 @@
+/**
+ * windowState — read a page's framework state straight off `window`, after
+ * its JavaScript ran (extract_embedded_state's escalated path, plan Phase 3.2).
+ *
+ * The served HTML is not always where the state ends up: Nuxt 2 ships
+ * __NUXT__ as an IIFE the HTML parser cannot evaluate, and YouTube's
+ * ytInitialData is assigned by a script. Inside the stealth render those are
+ * plain globals, so they are read as the page itself sees them.
+ *
+ * Engines: on Chromium page.evaluate runs in the page's main world, where the
+ * globals live. On camoufox it runs in an isolated world that cannot see them
+ * (window.__NEXT_DATA__ is undefined there), but Firefox's Xray waiver,
+ * `window.wrappedJSObject`, reaches the page's own window from that world
+ * without evaluating anything in the page's — measured against camoufox
+ * 0.1.19, 2026-09-29. So the read works on both engines and the page never
+ * sees a script of ours run in its world.
+ *
+ * Each value is serialised INSIDE the page with its own try/catch: a cyclic or
+ * otherwise unserialisable global is skipped with a warning instead of
+ * failing the read of the others.
+ */
+
+/** The globals read, in this order. Only the ones present come back. */
+export const WINDOW_STATE_GLOBALS = Object.freeze([
+  '__NEXT_DATA__',
+  '__NUXT__',
+  '__remixContext',
+  'ytInitialData',
+  'ytInitialPlayerResponse',
+  '__INITIAL_STATE__',
+  '__PRELOADED_STATE__',
+  '__APOLLO_STATE__',
+  '__TGT_DATA__',
+  '__PWS_DATA__'
+]);
+
+/** Said wherever these values are reported: they are not what the server sent. */
+export const WINDOW_STATE_NOTE = 'read from window after JavaScript ran, not from the served HTML';
+
+/**
+ * Runs in the page. `{ json }` for a global that serialised, `{ error }` for
+ * one that did not; absent (null/undefined) globals are left out.
+ * @param {string[]} names
+ */
+export function serialiseWindowGlobals(names) {
+  const target = window.wrappedJSObject || window;
+  const out = {};
+  for (const name of names) {
+    try {
+      const value = target[name];
+      if (value === undefined || value === null) continue;
+      const json = JSON.stringify(value);
+      out[name] = typeof json === 'string' ? { json } : { error: 'it has no JSON form' };
+    } catch (error) {
+      out[name] = { error: String((error && error.message) || error) };
+    }
+  }
+  return out;
+}
+
+/**
+ * @param {{ evaluate: Function }} page a Playwright page
+ * @returns {Promise<{ state: Record<string, unknown>, warnings: string[] }>}
+ */
+export async function readWindowState(page) {
+  const warnings = [];
+  let raw;
+  try {
+    raw = await page.evaluate(serialiseWindowGlobals, [...WINDOW_STATE_GLOBALS]);
+  } catch (error) {
+    warnings.push(`window_state could not be read: ${String(error.message).split('\n')[0]}`);
+    return { state: {}, warnings };
+  }
+  const state = {};
+  for (const [name, entry] of Object.entries(raw || {})) {
+    if (entry && typeof entry.json === 'string') {
+      try {
+        state[name] = JSON.parse(entry.json);
+        continue;
+      } catch (error) {
+        warnings.push(`window_state: ${name} skipped - its JSON did not parse (${error.message})`);
+        continue;
+      }
+    }
+    warnings.push(`window_state: ${name} skipped - not serialisable (${entry?.error ?? 'unknown reason'})`);
+  }
+  return { state, warnings };
+}

package/src/core/crawlers/BFSCrawler.js

@@ -8,9 +8,10 @@ import { LinkAnalyzer } from '../analysis/LinkAnalyzer.js';
 import { normalizeUrl, extractLinks, isValidUrl } from '../../utils/urlNormalizer.js';
 import { Logger } from '../../utils/Logger.js';
 import { safeFetch } from '../../utils/ssrfGuard.js';
-import { robotsPreflight, RobotsDisallowedError } from '../../utils/robotsGate.js';
+import { robotsPreflight, redirectGate, RobotsDisallowedError } from '../../utils/robotsGate.js';
 import { throttleHost } from '../../utils/hostRateLimiter.js';
 import { CRAWLFORGE_USER_AGENT, identityHeaders } from '../../utils/fetchIdentity.js';
+import { signRequestHeaders } from '../../utils/webBotAuth.js';
 import { pageTitle } from '../../utils/pageTitle.js';
 import { extractMainContent, isThinMainContent } from '../../tools/scrape/_mainContent.js';
 
@@ -337,7 +338,10 @@ export class BFSCrawler {
         'Accept-Language': 'en-US,en;q=0.5',
         'Accept-Encoding': 'gzip, deflate',
         'Connection': 'keep-alive',
-        'Upgrade-Insecure-Requests': '1'
+        'Upgrade-Insecure-Requests': '1',
+        // Web Bot Auth, as on every other fetching tool. crawl_deep pages went
+        // out with the identity alone: a claim with no proof.
+        ...signRequestHeaders(url)
       };
 
       let headers = { ...defaultHeaders, ...domainRules.customHeaders };
@@ -357,7 +361,12 @@ export class BFSCrawler {
 
       const response = await safeFetch(url, {
         signal: controller.signal,
-        headers
+        headers,
+        onRedirect: redirectGate(url, {
+          respectRobots: this.respectRobots,
+          userAgent: this.userAgent,
+          tool: 'crawl_deep'
+        })
       });
 
       // Capture any cookies the server sets during the crawl

package/src/core/processing/BrowserProcessor.js

@@ -9,7 +9,7 @@ import { z } from 'zod';
 import StealthBrowserManager from '../StealthBrowserManager.js';
 import HumanBehaviorSimulator from '../../utils/HumanBehaviorSimulator.js';
 import LocalizationManager from '../LocalizationManager.js';
-import { safeGoto } from '../../utils/ssrfGuard.js';
+import { safeGoto, assertNavigationAllowed } from '../../utils/ssrfGuard.js';
 
 const BrowserProcessorSchema = z.object({
   url: z.string().url(),
@@ -156,9 +156,12 @@ export class BrowserProcessor {
    * @param {Object} params - Processing parameters
    * @param {string} params.url - URL to process
    * @param {Object} params.options - Browser options
+   * @param {{ onRedirect?: (to: string) => Promise<void> }} [gate] - what the
+   *   calling tool's `preflightFetch` returned, so a redirect is gated the way
+   *   the URL was. A second argument because the schema strips it from `params`.
    * @returns {Promise<Object>} - Processing result with rendered content
    */
-  async processURL(params) {
+  async processURL(params, gate) {
     const startTime = Date.now();
     
     try {
@@ -178,7 +181,7 @@ export class BrowserProcessor {
 
       try {
         // Navigate and wait for content
-        const navigationResult = await this.navigateAndWait(page, url, processingOptions);
+        const navigationResult = await this.navigateAndWait(page, url, processingOptions, gate?.onRedirect);
         
         // Extract content and metadata
         const contentResult = await this.extractContent(page, processingOptions);
@@ -635,18 +638,23 @@ export class BrowserProcessor {
    * @param {Page} page - Playwright page
    * @param {string} url - URL to navigate to
    * @param {Object} options - Navigation options
+   * @param {(to: string) => Promise<void>} [onRedirect] - the calling tool's
+   *   redirect gate (robotsGate.js `redirectGate`)
    * @returns {Promise<Object>} - Navigation result
    */
-  async navigateAndWait(page, url, options) {
+  async navigateAndWait(page, url, options, onRedirect) {
     const startTime = Date.now();
 
     // Navigate to URL. safeGoto adds the SSRF/DNS-rebinding guard on top of the
-    // robots/blocklist preflight the calling tool already ran, and re-checks the
-    // landed URL so a redirect can't carry us into a private/metadata range.
+    // robots/blocklist preflight the calling tool already ran, and re-checks
+    // where the navigation went, so a redirect can't carry us into a
+    // private/metadata range or past the gate the URL itself went through.
     await safeGoto(page, url, {
       waitUntil: 'domcontentloaded',
-      timeout: 30000
+      timeout: 30000,
+      onRedirect
     });
+    const landedUrl = page.url();
 
     // Wait for specific selector if provided
     if (options.waitForSelector) {
@@ -687,6 +695,10 @@ export class BrowserProcessor {
       }
     }
 
+    // The waits above are time in which a client-side redirect can move the
+    // page on; the document about to be read is the one that has to pass.
+    await assertNavigationAllowed(page, landedUrl, null, onRedirect);
+
     return {
       navigationTime: Date.now() - startTime
     };

package/src/core/processing/PDFProcessor.js

@@ -10,6 +10,7 @@ import path from 'path';
 import { safeFetch } from '../../utils/ssrfGuard.js';
 import { config } from '../../constants/config.js';
 import { identityHeaders } from '../../utils/fetchIdentity.js';
+import { signRequestHeaders } from '../../utils/webBotAuth.js';
 
 const PDFProcessorSchema = z.object({
   // A Buffer is the 'buffer' source: process_document hands over a body it
@@ -85,9 +86,11 @@ export class PDFProcessor {
    * @param {string} params.source - PDF source (URL, file path, or buffer)
    * @param {string} params.sourceType - Type of source ('url', 'file', 'buffer')
    * @param {Object} params.options - Processing options
+   * @param {{ headers: Record<string,string>, onRedirect: Function }} [gate] -
+   *   what `preflightFetch` returned for a URL source
    * @returns {Promise<Object>} - Processing result with text and metadata
    */
-  async processPDF(params) {
+  async processPDF(params, gate) {
     const startTime = Date.now();
     
     try {
@@ -106,7 +109,7 @@ export class PDFProcessor {
       // Get PDF buffer based on source type
       let pdfBuffer;
       try {
-        pdfBuffer = await this.getPDFBuffer(source, sourceType);
+        pdfBuffer = await this.getPDFBuffer(source, sourceType, gate);
       } catch (error) {
         result.error = `Failed to load PDF: ${error.message}`;
         result.processingTime = Date.now() - startTime;
@@ -214,12 +217,13 @@ export class PDFProcessor {
    * Get PDF buffer from various sources
    * @param {string} source - PDF source
    * @param {string} sourceType - Source type
+   * @param {Object} [gate] - see processPDF
    * @returns {Promise<Buffer>} - PDF buffer
    */
-  async getPDFBuffer(source, sourceType) {
+  async getPDFBuffer(source, sourceType, gate) {
     switch (sourceType) {
       case 'url':
-        return await this.downloadPDFFromURL(source);
+        return await this.downloadPDFFromURL(source, gate);
       case 'file':
         return await this.readPDFFromFile(source);
       case 'buffer':
@@ -232,15 +236,20 @@ export class PDFProcessor {
   /**
    * Download PDF from URL
    * @param {string} url - PDF URL
+   * @param {Object} [gate] - see processPDF
    * @returns {Promise<Buffer>} - PDF buffer
    */
-  async downloadPDFFromURL(url) {
+  async downloadPDFFromURL(url, gate) {
     try {
       // `timeout` is not a fetch init option — undici/Node fetch silently
       // ignores unknown properties, so only `signal` actually enforces a
       // deadline here.
+      // The gate's headers are the identity the URL was cleared under plus the
+      // Web Bot Auth signature; this download used to discard them and go out
+      // unsigned. With no gate, sign as ourselves.
       const response = await safeFetch(url, {
-        headers: identityHeaders(),
+        headers: gate?.headers ?? { ...identityHeaders(), ...signRequestHeaders(url) },
+        onRedirect: gate?.onRedirect,
         signal: AbortSignal.timeout(30000)
       });
 

package/src/server/fallbackHints.js

@@ -44,8 +44,8 @@ export const FALLBACK_HINTS = Object.freeze({
 });
 
 /**
- * The second-stage hint (Phase 3, 3.4). A `scrape` that came back
- * `escalated: true` has ALREADY run the stealth browser on this URL, so the
+ * The second-stage hint (Phase 3, 3.4). A `scrape` or `extract_embedded_state`
+ * that came back `escalated: true` has ALREADY run the stealth browser on this URL, so the
  * ordinary hint would send the model back to a tool that has just failed.
  * What is left is a region-specific block, which `localization` can change,
  * or a block that is final: an Akamai-style TLS-level wall is beaten with
@@ -56,9 +56,12 @@ export const SCRAPE_ESCALATED_HINT =
   'If the block is regional, set a country with localization and try once more; otherwise the block is final - ' +
   'a TLS-level wall needs residential proxies, which CrawlForge does not offer. Get the content from another source.';
 
-/** The hint for this result: the tool's, unless a scrape already escalated. */
+/** The tools whose escalate:true runs the stealth stage itself (Phase 3; extract_embedded_state since plan Phase 3.2). */
+const ESCALATING_TOOLS = new Set(['scrape', 'extract_embedded_state']);
+
+/** The hint for this result: the tool's, unless the call already escalated. */
 function hintFor(toolName, parsed) {
-  if (toolName === 'scrape' && parsed?.escalated === true) return SCRAPE_ESCALATED_HINT;
+  if (ESCALATING_TOOLS.has(toolName) && parsed?.escalated === true) return SCRAPE_ESCALATED_HINT;
   return FALLBACK_HINTS[toolName];
 }
 

package/src/server/inlineThreshold.js

@@ -20,9 +20,9 @@ export const MAX_INLINE_CHARS_PARAM = {
 /**
  * Per-tool rule. `textPaths` are dotted paths tried in order; the first one
  * holding a string becomes the text view the preview and read_result work
- * on. An empty list means the pretty-printed JSON is the view. `truncate:
- * false` keeps the whole result inline and only adds the handle
- * (extract_embedded_state's never-truncate rule). `when` gates on params.
+ * on. An empty list means the pretty-printed JSON is the view. `when` gates
+ * on params. extract_embedded_state has a shape of its own (shapeEmbeddedState
+ * below).
  */
 /** The browser_session operations that hand back content worth shaping. */
 const BROWSER_SESSION_CONTENT_OPERATIONS = new Set(['snapshot', 'act', 'read']);
@@ -52,7 +52,10 @@ export const INLINE_THRESHOLD_TOOLS = Object.freeze({
   },
   process_document: { textPaths: ['content.text'], truncate: true },
   deep_research: { textPaths: [], truncate: true },
-  extract_embedded_state: { textPaths: [], truncate: false }
+  // Truncated since plan Phase 3.1: returned whole, producthunt.com and
+  // zappos.com overflowed the client. Shaped by shapeEmbeddedState, which never
+  // cuts inside a JSON value.
+  extract_embedded_state: { textPaths: [], truncate: true }
 });
 
 /** param -> env (an int of at least 1,000) -> default. */
@@ -147,21 +150,8 @@ export function applyInlineThreshold(toolName, resultObject, params, { store, en
   const viewDesc = view === 'text' ? `the ${view_path} text` : 'the pretty-printed JSON';
   const readWith = 'read it with read_result (1 credit) - operation "search" (query), "slice" (offset, length), "lines" or "json_path" (path) - and do not fetch the page again';
 
-  if (!config.truncate) {
-    const hint = `Result is ${json.length} chars as JSON, over the inline limit of ${maxInline}; it is returned whole (this tool never truncates) and is also kept for 1 hour under result_handle ${handle}: ${readWith}.`;
-    return {
-      result: {
-        ...resultObject,
-        result_handle: handle,
-        total_chars: text.length,
-        view,
-        view_path,
-        truncated: false,
-        expires_at,
-        warnings: [...warningsOf(resultObject), hint]
-      },
-      stored: true
-    };
+  if (toolName === 'extract_embedded_state') {
+    return { result: shapeEmbeddedState(resultObject, { json, text, maxInline, handle, expires_at, readWith }), stored: true };
   }
 
   const preview = text.slice(0, maxInline);
@@ -200,3 +190,75 @@ export function applyInlineThreshold(toolName, resultObject, params, { store, en
   });
   return { result: shaped, stored: true };
 }
+
+/** Object.keys(data) for a plain object, "array(<n>)" for an array, null otherwise. */
+export function embeddedDataKeys(data) {
+  if (Array.isArray(data)) return `array(${data.length})`;
+  if (data !== null && typeof data === 'object') return Object.keys(data);
+  return null;
+}
+
+/**
+ * The longest run of whole lines from the start of `text` whose JSON-string
+ * form fits in `budget` characters. A line that does not fit ends the preview,
+ * so it never stops inside a value; with a budget too small for the first
+ * line it is empty.
+ */
+export function wholeLinePreview(text, budget) {
+  const room = budget - 2; // the string's own quotes
+  let used = 0;
+  let end = 0;
+  let start = 0;
+  while (start < text.length) {
+    const newline = text.indexOf('\n', start);
+    const stop = newline === -1 ? text.length : newline;
+    // A line's escaped length, plus the escaped newline that joins it to the last.
+    const cost = JSON.stringify(text.slice(start, stop)).length - 2 + (end > 0 ? 2 : 0);
+    if (used + cost > room) break;
+    used += cost;
+    end = stop;
+    start = stop + 1;
+  }
+  return text.slice(0, end);
+}
+
+/**
+ * extract_embedded_state over the limit (plan Phase 3.1). What stays inline is
+ * everything that says what the page carries — url, path, bytes, the whole
+ * `found` list, data_keys, the escalation report and window_state's list —
+ * then a preview of the pretty-printed JSON cut at a line boundary. The
+ * preview gets whatever max_inline_chars leaves after the rest, measured as
+ * compact JSON, so the whole inline result stays within the limit whenever
+ * the rest fits. The full result is under the handle for read_result
+ * json_path. The REST route shapes it the same way; keep the two in step.
+ */
+function shapeEmbeddedState(resultObject, { json, text, maxInline, handle, expires_at, readWith }) {
+  const dataKeys = embeddedDataKeys(resultObject.data);
+  const firstKey = Array.isArray(dataKeys) && dataKeys.length > 0 ? dataKeys[0] : null;
+  const example = firstKey && /^[A-Za-z_$][\w$]*$/.test(firstKey) ? ` Read one payload with operation "json_path", path "data.${firstKey}".` : '';
+  const hint = `Result is ${json.length} chars as JSON, over the inline limit of ${maxInline}; preview holds the pretty-printed JSON up to the last whole line that fits (${text.length} chars in total), data_keys lists the top-level keys of data, and the full result is kept for 1 hour under result_handle ${handle}: ${readWith}.${example} Or re-run with path, or keys_only:true, to ask for less.`;
+
+  const shaped = {};
+  for (const key of ['url', 'path', 'bytes', 'found']) {
+    if (key in resultObject) shaped[key] = resultObject[key];
+  }
+  shaped.data_keys = dataKeys;
+  if ('escalated' in resultObject) shaped.escalated = resultObject.escalated;
+  if (resultObject.stealth) shaped.stealth = resultObject.stealth;
+  if (resultObject.window_state && typeof resultObject.window_state === 'object') {
+    shaped.window_state = { note: resultObject.window_state.note, found: resultObject.window_state.found };
+  }
+  const tail = {
+    result_handle: handle,
+    total_chars: text.length,
+    view: 'json',
+    view_path: null,
+    truncated: true,
+    expires_at,
+    warnings: [...warningsOf(resultObject), hint]
+  };
+  // `"preview":"",` is what the key itself adds once the value is in.
+  const envelope = JSON.stringify({ ...shaped, preview: '', ...tail }).length;
+  const preview = wholeLinePreview(text, Math.max(0, maxInline - envelope + 2));
+  return { ...shaped, preview, ...tail };
+}

package/src/skills/agent-skills/crawlforge-getting-started/references/credits.md

@@ -22,7 +22,7 @@ metered; there is no free tier. Tools marked "scales" cost more as work grows.
 |------|-------|
 | `scrape` | Unified multi-format single fetch. A `highlights` or `question` format adds 1 once per call; `mode:"model"` on one of them adds 3 once. `escalate:true` adds 5 as the projected ceiling; the actual charge drops back to the base when the plain fetch succeeded and no escalation ran. |
 | `scrape_structured` | CSS-selector extraction. |
-| `extract_embedded_state` | Embedded JS state (`__NEXT_DATA__`, RSC, Nuxt, Apollo, Redux). |
+| `extract_embedded_state` | Embedded JS state (`__NEXT_DATA__`, RSC, Nuxt, Apollo, Redux). `escalate:true` adds 5 as the projected ceiling for the stealth re-read; the actual charge drops back to 2 when the plain fetch succeeded and no escalation ran. `keys_only` and `find` cost nothing extra. |
 | `extract_content` | Readability-cleaned article. |
 | `map_site` | URL discovery / sitemap. |
 | `process_document` | PDF / DOCX / TXT parsing. |

package/src/skills/agent-skills/crawlforge-structured-extraction/SKILL.md

@@ -36,15 +36,34 @@ already, exact and typed — that beats asking a model to read them off the rend
 { "tool": "extract_embedded_state", "params": { "url": "https://www.ticketmaster.com/discover/concerts", "path": "next_data.props.pageProps" } }
 ```
 
-Finds `__NEXT_DATA__`, `self.__next_f` (React Server Component payloads),
-`window.__NUXT__`, `__APOLLO_STATE__`, `__INITIAL_STATE__`, `__PRELOADED_STATE__`
-and `<script type="application/json">` blocks, keyed by source name. No LLM in
-the path, so values are exact rather than inferred.
+Finds `__NEXT_DATA__`, `self.__next_f` (React Server Component payloads, with
+`$<id>` references between rows resolved and a `data_rows` index of the rows
+carrying data), `window.__NUXT__` and Nuxt 3's `__NUXT_DATA__` (decoded as
+`nuxt_data`; `raw: true` also keeps the undecoded array), SvelteKit `kit.start`
+data, `__APOLLO_STATE__`, `__INITIAL_STATE__`, `__PRELOADED_STATE__`,
+`ytInitialData`, `ytInitialPlayerResponse`, Inertia `data-page`, Shopify meta and
+product JSON, JSON `data-*` attributes and `<script type="application/json">`
+blocks, keyed by source name. No LLM in the path, so values are exact rather
+than inferred.
 
 These payloads are routinely over a megabyte — pass `path` (dotted keys and
 array indexes, e.g. `next_data.props.pageProps` or `next_f[0].f`) to return one
-subtree. Without it, a large result comes back with a warning naming the biggest
-source and a ready-to-paste path.
+subtree, or `keys_only: true` first to see the first two levels of keys (types,
+no values) before choosing one. When you know a field's name but not where it
+is, `find: "<key>"` returns every path holding it (case-insensitive, up to 50,
+each with a 200-character preview) instead of the data; each path can be passed
+straight back as `path`. `find` and `keys_only` cannot be combined. A result
+over `max_inline_chars` comes back as a `preview` (whole lines of the JSON),
+`data_keys` and a `result_handle`: read the
+rest with `read_result` operation `json_path` (e.g. path `data.next_data.props`)
+instead of fetching again.
+
+On a site that blocks a plain fetch (403/429/challenge page), pass
+`escalate: true` (projected 7, charged 2 when the plain fetch works): the stealth
+browser re-reads the page, the same parser runs on it, and the framework globals
+are read off `window` after JavaScript ran — `ytInitialData`, `__NUXT__`,
+`__remixContext` and others, under `window_state` (path
+`window_state.ytInitialData.contents`). A 404 or 5xx never escalates.
 
 ## scrape_template — known sites, zero selectors (cost: 1)