crawlforge-mcp-server 6.12.1 → 6.16.0

package/src/server/fallbackHints.js

@@ -44,8 +44,8 @@ export const FALLBACK_HINTS = Object.freeze({
 });
 
 /**
- * The second-stage hint (Phase 3, 3.4). A `scrape` that came back
- * `escalated: true` has ALREADY run the stealth browser on this URL, so the
+ * The second-stage hint (Phase 3, 3.4). A `scrape` or `extract_embedded_state`
+ * that came back `escalated: true` has ALREADY run the stealth browser on this URL, so the
  * ordinary hint would send the model back to a tool that has just failed.
  * What is left is a region-specific block, which `localization` can change,
  * or a block that is final: an Akamai-style TLS-level wall is beaten with
@@ -56,9 +56,12 @@ export const SCRAPE_ESCALATED_HINT =
   'If the block is regional, set a country with localization and try once more; otherwise the block is final - ' +
   'a TLS-level wall needs residential proxies, which CrawlForge does not offer. Get the content from another source.';
 
-/** The hint for this result: the tool's, unless a scrape already escalated. */
+/** The tools whose escalate:true runs the stealth stage itself (Phase 3; extract_embedded_state since plan Phase 3.2). */
+const ESCALATING_TOOLS = new Set(['scrape', 'extract_embedded_state']);
+
+/** The hint for this result: the tool's, unless the call already escalated. */
 function hintFor(toolName, parsed) {
-  if (toolName === 'scrape' && parsed?.escalated === true) return SCRAPE_ESCALATED_HINT;
+  if (ESCALATING_TOOLS.has(toolName) && parsed?.escalated === true) return SCRAPE_ESCALATED_HINT;
   return FALLBACK_HINTS[toolName];
 }
 

package/src/server/inlineThreshold.js

@@ -20,9 +20,9 @@ export const MAX_INLINE_CHARS_PARAM = {
 /**
  * Per-tool rule. `textPaths` are dotted paths tried in order; the first one
  * holding a string becomes the text view the preview and read_result work
- * on. An empty list means the pretty-printed JSON is the view. `truncate:
- * false` keeps the whole result inline and only adds the handle
- * (extract_embedded_state's never-truncate rule). `when` gates on params.
+ * on. An empty list means the pretty-printed JSON is the view. `when` gates
+ * on params. extract_embedded_state has a shape of its own (shapeEmbeddedState
+ * below).
  */
 /** The browser_session operations that hand back content worth shaping. */
 const BROWSER_SESSION_CONTENT_OPERATIONS = new Set(['snapshot', 'act', 'read']);
@@ -52,7 +52,10 @@ export const INLINE_THRESHOLD_TOOLS = Object.freeze({
   },
   process_document: { textPaths: ['content.text'], truncate: true },
   deep_research: { textPaths: [], truncate: true },
-  extract_embedded_state: { textPaths: [], truncate: false }
+  // Truncated since plan Phase 3.1: returned whole, producthunt.com and
+  // zappos.com overflowed the client. Shaped by shapeEmbeddedState, which never
+  // cuts inside a JSON value.
+  extract_embedded_state: { textPaths: [], truncate: true }
 });
 
 /** param -> env (an int of at least 1,000) -> default. */
@@ -147,21 +150,8 @@ export function applyInlineThreshold(toolName, resultObject, params, { store, en
   const viewDesc = view === 'text' ? `the ${view_path} text` : 'the pretty-printed JSON';
   const readWith = 'read it with read_result (1 credit) - operation "search" (query), "slice" (offset, length), "lines" or "json_path" (path) - and do not fetch the page again';
 
-  if (!config.truncate) {
-    const hint = `Result is ${json.length} chars as JSON, over the inline limit of ${maxInline}; it is returned whole (this tool never truncates) and is also kept for 1 hour under result_handle ${handle}: ${readWith}.`;
-    return {
-      result: {
-        ...resultObject,
-        result_handle: handle,
-        total_chars: text.length,
-        view,
-        view_path,
-        truncated: false,
-        expires_at,
-        warnings: [...warningsOf(resultObject), hint]
-      },
-      stored: true
-    };
+  if (toolName === 'extract_embedded_state') {
+    return { result: shapeEmbeddedState(resultObject, { json, text, maxInline, handle, expires_at, readWith }), stored: true };
   }
 
   const preview = text.slice(0, maxInline);
@@ -200,3 +190,75 @@ export function applyInlineThreshold(toolName, resultObject, params, { store, en
   });
   return { result: shaped, stored: true };
 }
+
+/** Object.keys(data) for a plain object, "array(<n>)" for an array, null otherwise. */
+export function embeddedDataKeys(data) {
+  if (Array.isArray(data)) return `array(${data.length})`;
+  if (data !== null && typeof data === 'object') return Object.keys(data);
+  return null;
+}
+
+/**
+ * The longest run of whole lines from the start of `text` whose JSON-string
+ * form fits in `budget` characters. A line that does not fit ends the preview,
+ * so it never stops inside a value; with a budget too small for the first
+ * line it is empty.
+ */
+export function wholeLinePreview(text, budget) {
+  const room = budget - 2; // the string's own quotes
+  let used = 0;
+  let end = 0;
+  let start = 0;
+  while (start < text.length) {
+    const newline = text.indexOf('\n', start);
+    const stop = newline === -1 ? text.length : newline;
+    // A line's escaped length, plus the escaped newline that joins it to the last.
+    const cost = JSON.stringify(text.slice(start, stop)).length - 2 + (end > 0 ? 2 : 0);
+    if (used + cost > room) break;
+    used += cost;
+    end = stop;
+    start = stop + 1;
+  }
+  return text.slice(0, end);
+}
+
+/**
+ * extract_embedded_state over the limit (plan Phase 3.1). What stays inline is
+ * everything that says what the page carries — url, path, bytes, the whole
+ * `found` list, data_keys, the escalation report and window_state's list —
+ * then a preview of the pretty-printed JSON cut at a line boundary. The
+ * preview gets whatever max_inline_chars leaves after the rest, measured as
+ * compact JSON, so the whole inline result stays within the limit whenever
+ * the rest fits. The full result is under the handle for read_result
+ * json_path. The REST route shapes it the same way; keep the two in step.
+ */
+function shapeEmbeddedState(resultObject, { json, text, maxInline, handle, expires_at, readWith }) {
+  const dataKeys = embeddedDataKeys(resultObject.data);
+  const firstKey = Array.isArray(dataKeys) && dataKeys.length > 0 ? dataKeys[0] : null;
+  const example = firstKey && /^[A-Za-z_$][\w$]*$/.test(firstKey) ? ` Read one payload with operation "json_path", path "data.${firstKey}".` : '';
+  const hint = `Result is ${json.length} chars as JSON, over the inline limit of ${maxInline}; preview holds the pretty-printed JSON up to the last whole line that fits (${text.length} chars in total), data_keys lists the top-level keys of data, and the full result is kept for 1 hour under result_handle ${handle}: ${readWith}.${example} Or re-run with path, or keys_only:true, to ask for less.`;
+
+  const shaped = {};
+  for (const key of ['url', 'path', 'bytes', 'found']) {
+    if (key in resultObject) shaped[key] = resultObject[key];
+  }
+  shaped.data_keys = dataKeys;
+  if ('escalated' in resultObject) shaped.escalated = resultObject.escalated;
+  if (resultObject.stealth) shaped.stealth = resultObject.stealth;
+  if (resultObject.window_state && typeof resultObject.window_state === 'object') {
+    shaped.window_state = { note: resultObject.window_state.note, found: resultObject.window_state.found };
+  }
+  const tail = {
+    result_handle: handle,
+    total_chars: text.length,
+    view: 'json',
+    view_path: null,
+    truncated: true,
+    expires_at,
+    warnings: [...warningsOf(resultObject), hint]
+  };
+  // `"preview":"",` is what the key itself adds once the value is in.
+  const envelope = JSON.stringify({ ...shaped, preview: '', ...tail }).length;
+  const preview = wholeLinePreview(text, Math.max(0, maxInline - envelope + 2));
+  return { ...shaped, preview, ...tail };
+}

package/src/skills/agent-skills/crawlforge-getting-started/references/credits.md

@@ -22,7 +22,7 @@ metered; there is no free tier. Tools marked "scales" cost more as work grows.
 |------|-------|
 | `scrape` | Unified multi-format single fetch. A `highlights` or `question` format adds 1 once per call; `mode:"model"` on one of them adds 3 once. `escalate:true` adds 5 as the projected ceiling; the actual charge drops back to the base when the plain fetch succeeded and no escalation ran. |
 | `scrape_structured` | CSS-selector extraction. |
-| `extract_embedded_state` | Embedded JS state (`__NEXT_DATA__`, RSC, Nuxt, Apollo, Redux). |
+| `extract_embedded_state` | Embedded JS state (`__NEXT_DATA__`, RSC, Nuxt, Apollo, Redux). `escalate:true` adds 5 as the projected ceiling for the stealth re-read; the actual charge drops back to 2 when the plain fetch succeeded and no escalation ran. `keys_only` and `find` cost nothing extra. |
 | `extract_content` | Readability-cleaned article. |
 | `map_site` | URL discovery / sitemap. |
 | `process_document` | PDF / DOCX / TXT parsing. |

package/src/skills/agent-skills/crawlforge-structured-extraction/SKILL.md

@@ -36,15 +36,34 @@ already, exact and typed — that beats asking a model to read them off the rend
 { "tool": "extract_embedded_state", "params": { "url": "https://www.ticketmaster.com/discover/concerts", "path": "next_data.props.pageProps" } }
 ```
 
-Finds `__NEXT_DATA__`, `self.__next_f` (React Server Component payloads),
-`window.__NUXT__`, `__APOLLO_STATE__`, `__INITIAL_STATE__`, `__PRELOADED_STATE__`
-and `<script type="application/json">` blocks, keyed by source name. No LLM in
-the path, so values are exact rather than inferred.
+Finds `__NEXT_DATA__`, `self.__next_f` (React Server Component payloads, with
+`$<id>` references between rows resolved and a `data_rows` index of the rows
+carrying data), `window.__NUXT__` and Nuxt 3's `__NUXT_DATA__` (decoded as
+`nuxt_data`; `raw: true` also keeps the undecoded array), SvelteKit `kit.start`
+data, `__APOLLO_STATE__`, `__INITIAL_STATE__`, `__PRELOADED_STATE__`,
+`ytInitialData`, `ytInitialPlayerResponse`, Inertia `data-page`, Shopify meta and
+product JSON, JSON `data-*` attributes and `<script type="application/json">`
+blocks, keyed by source name. No LLM in the path, so values are exact rather
+than inferred.
 
 These payloads are routinely over a megabyte — pass `path` (dotted keys and
 array indexes, e.g. `next_data.props.pageProps` or `next_f[0].f`) to return one
-subtree. Without it, a large result comes back with a warning naming the biggest
-source and a ready-to-paste path.
+subtree, or `keys_only: true` first to see the first two levels of keys (types,
+no values) before choosing one. When you know a field's name but not where it
+is, `find: "<key>"` returns every path holding it (case-insensitive, up to 50,
+each with a 200-character preview) instead of the data; each path can be passed
+straight back as `path`. `find` and `keys_only` cannot be combined. A result
+over `max_inline_chars` comes back as a `preview` (whole lines of the JSON),
+`data_keys` and a `result_handle`: read the
+rest with `read_result` operation `json_path` (e.g. path `data.next_data.props`)
+instead of fetching again.
+
+On a site that blocks a plain fetch (403/429/challenge page), pass
+`escalate: true` (projected 7, charged 2 when the plain fetch works): the stealth
+browser re-reads the page, the same parser runs on it, and the framework globals
+are read off `window` after JavaScript ran — `ytInitialData`, `__NUXT__`,
+`__remixContext` and others, under `window_state` (path
+`window_state.ytInitialData.contents`). A 404 or 5xx never escalates.
 
 ## scrape_template — known sites, zero selectors (cost: 1)
 

package/src/tools/advanced/BrowserSessionTool.js

@@ -42,6 +42,7 @@ import { isRemoteTransport } from '../../utils/remoteMode.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
 import { stealthDocumentVerdict } from '../../utils/stealthVerdict.js';
 import { resolveStealthEngine } from '../../core/StealthBrowserManager.js';
+import { actionQueue as sharedActionQueue } from '../../core/browser/actionQueue.js';
 
 const SECOND = 1000;
 
@@ -110,8 +111,10 @@ const BrowserSessionSchema = z.object({
   timeout: z.number().min(10000).max(120000).default(30000),
 
   // Applies to the call it is sent on: on `open` to the first load, on `act` to
-  // every navigate in that call. It is never remembered by the session, so an
-  // override has to be repeated as deliberately as it was made.
+  // every navigate in that call, and on any call to wherever the page has moved
+  // since it was last checked (a click, a form, a script). It is never
+  // remembered by the session, so an override has to be repeated as
+  // deliberately as it was made. A URL that passed is not asked about again.
   respect_robots: z.boolean().optional(),
 
   // snapshot
@@ -207,7 +210,10 @@ export class BrowserSessionTool {
       extractContentTool = null,
       store = null,
       storeOptions = {},
-      enableLogging = true
+      enableLogging = true,
+      // The process-wide browser-slot limiter shared with scrape_with_actions.
+      // Injectable for tests only.
+      actionQueue = sharedActionQueue
     } = options;
 
     // An injected executor belongs to whoever built it (server.js hands us
@@ -218,6 +224,7 @@ export class BrowserSessionTool {
     this.extractContentTool = extractContentTool || new ExtractContentTool();
     this.storeOptions = storeOptions;
     this.store = store || new BrowserSessionStore(storeOptions);
+    this.actionQueue = actionQueue;
   }
 
   async execute(params) {
@@ -337,11 +344,24 @@ export class BrowserSessionTool {
     // only then creates a page and navigates — closing the page itself if any
     // of that fails. That is the whole of 2.5's gating on `open`, which is why
     // this is not page.goto() behind a gate written here.
-    const page = await this.actionExecutor.initializePage(params.url, browserOptions);
+    //
+    // Page creation and the first navigation take a slot from the browser
+    // budget scrape_with_actions draws on, so a burst of opens queues behind
+    // (and alongside) action chains instead of piling onto the box. The slot is
+    // released as soon as the page is up — a live session must NOT hold it for
+    // its TTL, or three open sessions would starve every scrape_with_actions
+    // call for up to an hour. How many sessions may stay open is the store's
+    // per-owner and total caps below, which still refuse rather than queue.
+    const page = await this.actionQueue.run(
+      () => this.actionExecutor.initializePage(params.url, browserOptions),
+      { label: 'browser_session open' }
+    );
     const releasePage = this.releaser(page, params.stealth);
 
     let session;
     try {
+      // The load was gated; a script may have moved the page on since.
+      await this.actionExecutor.assertPageAllowed(page, browserOptions);
       session = this.store.create({
         ownerId,
         page,
@@ -355,8 +375,9 @@ export class BrowserSessionTool {
         maxPerOwner: ownerId.startsWith(REST_OWNER_PREFIX) ? REST_MAX_SESSIONS_PER_OWNER : undefined
       });
     } catch (error) {
-      // A cap refusal arrives with a live page in hand. Give it back before
-      // rethrowing, or the refused call leaks the context it just pinned.
+      // A cap refusal, or a page the gate refused, arrives with a live page in
+      // hand. Give it back before rethrowing, or the refused call leaks the
+      // context it just pinned.
       await releasePage();
       throw error;
     }
@@ -422,13 +443,29 @@ export class BrowserSessionTool {
     }
   }
 
+  /**
+   * Read from a session's page once the URL it stands on has passed the gate.
+   *
+   * A session's page outlives the call that loaded it, and between calls it
+   * can go anywhere: the last `act` clicked a link, or a script redirected.
+   * ActionExecutor.readGated holds it to the SSRF guard and the blocklist/
+   * robots gate on both sides of the read, with this call's `respect_robots`.
+   */
+  readPage(session, params, read) {
+    return this.actionExecutor.readGated(
+      session.page,
+      { respectRobots: params.respect_robots, tool: 'browser_session' },
+      read
+    );
+  }
+
   async snapshotSession(params, ownerId) {
     const session = this.requireSession(params, ownerId);
 
-    const snapshot = await captureSnapshot(session.page, {
+    const snapshot = await this.readPage(session, params, () => captureSnapshot(session.page, {
       interactiveOnly: params.interactive_only,
       maxNodes: params.max_nodes
-    });
+    }));
 
     this.store.touch(session, session.page.url());
     return { success: true, operation: 'snapshot', ...sessionInfo(session), snapshot };
@@ -457,9 +494,11 @@ export class BrowserSessionTool {
     }
 
     // Every `navigate` in here re-runs the SSRF guard and the blocklist/robots
-    // gate inside executeNavigateAction — verified, and the reason the gate is
-    // not repeated here. A long-lived session is a repeatable navigation
-    // primitive, so that per-hop check is what stops it becoming an SSRF hop.
+    // gate inside executeNavigateAction, and every action is bracketed by the
+    // same check on wherever the page has moved (executeGatedAction) — the
+    // reason the gate is not repeated here. A long-lived session is a
+    // repeatable navigation primitive, so those checks are what stop it
+    // becoming an SSRF hop.
     const result = await this.actionExecutor.executeActionsOnPage(session.page, params.actions, {
       continueOnError: params.continue_on_error,
       timeout: params.timeout,
@@ -487,8 +526,10 @@ export class BrowserSessionTool {
 
   async readSession(params, ownerId) {
     const session = this.requireSession(params, ownerId);
-    const url = session.page.url();
-    const html = await session.page.content();
+    const { url, html } = await this.readPage(session, params, async () => ({
+      url: session.page.url(),
+      html: await session.page.content()
+    }));
 
     const options = {};
     if (params.formats.includes('markdown')) options.outputFormat = 'markdown';
@@ -548,12 +589,12 @@ export class BrowserSessionTool {
   async screenshotSession(params, ownerId) {
     const session = this.requireSession(params, ownerId);
 
-    const shot = await this.actionExecutor.captureScreenshot(session.page, {
+    const shot = await this.readPage(session, params, () => this.actionExecutor.captureScreenshot(session.page, {
       fullPage: params.full_page,
       format: params.format,
       quality: params.quality,
       selector: params.selector
-    });
+    }));
 
     this.store.touch(session, session.page.url());
     // The actionId is what lets the server publish the image as a

package/src/tools/advanced/ScrapeWithActionsTool.js

@@ -15,6 +15,7 @@ import { elementText } from '../../utils/elementText.js';
 import { pageTitle } from '../../utils/pageTitle.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
 import { resolveStealthEngine } from '../../core/StealthBrowserManager.js';
+import { actionQueue as sharedActionQueue } from '../../core/browser/actionQueue.js';
 
 // Recording / replay helpers
 import {
@@ -198,7 +199,11 @@ const ScrapeWithActionsSchema = z.object({
     // — the standard pool is Chromium and camoufox exists only on the stealth
     // path — so "camoufox" without stealth is refused rather than quietly
     // downgraded (see executeSession).
-    engine: z.enum(['auto', 'chromium', 'camoufox', 'playwright']).default('auto')
+    engine: z.enum(['auto', 'chromium', 'camoufox', 'playwright']).default('auto'),
+    // Answer a cookie/consent wall (autoconsent) after the initial load and
+    // after each navigate action: "reject" opts out, "accept" opts in. Off by
+    // default so a chain that clicks the banner itself keeps working.
+    consent: z.enum(['off', 'reject', 'accept']).default('off')
   }).optional(),
 
   // Content extraction options
@@ -255,7 +260,9 @@ export class ScrapeWithActionsTool extends EventEmitter {
       extractContentTool = null,
       enableLogging = true,
       enableCaching = false,
-      maxConcurrentSessions = 3,
+      // The process-wide browser-slot limiter shared with browser_session open.
+      // Injectable for tests only; CRAWLFORGE_MAX_ACTION_SESSIONS sizes it.
+      actionQueue = sharedActionQueue,
       defaultBrowserOptions = {
         viewportWidth: 1280,
         viewportHeight: 720,
@@ -274,7 +281,7 @@ export class ScrapeWithActionsTool extends EventEmitter {
     this.extractContentTool = extractContentTool || new ExtractContentTool();
     this.enableLogging = enableLogging;
     this.enableCaching = enableCaching;
-    this.maxConcurrentSessions = maxConcurrentSessions;
+    this.actionQueue = actionQueue;
     this.defaultBrowserOptions = defaultBrowserOptions;
 
     // Active sessions tracking
@@ -322,69 +329,80 @@ export class ScrapeWithActionsTool extends EventEmitter {
 
       this.stats.totalSessions++;
       const sessionId = this.generateSessionId();
-      const startTime = Date.now();
+      const enqueuedAt = Date.now();
 
       if (this.enableLogging) {
         console.error(`Starting scrape session ${sessionId} with ${validated.actions.length} actions on ${validated.url}`);
       }
 
-      // Check concurrent sessions limit
-      if (this.activeSessions.size >= this.maxConcurrentSessions) {
-        throw new Error(`Maximum concurrent sessions (${this.maxConcurrentSessions}) reached`);
-      }
-
-      // Create session context
-      const sessionContext = {
-        id: sessionId,
-        url: validated.url,
-        startTime,
-        params: validated,
-        states: [],
-        screenshots: [],
-        actionResults: [],
-        errors: [],
-        status: 'initializing'
-      };
+      // The whole chain runs inside a slot of the shared browser budget. Over
+      // the limit a call waits for one rather than being refused, and fails
+      // only after CRAWLFORGE_ACTION_QUEUE_TIMEOUT_MS with the queue depth.
+      return await this.actionQueue.run(
+        () => this.runSession(sessionId, validated, enqueuedAt),
+        { label: 'scrape_with_actions' }
+      );
 
-      this.activeSessions.set(sessionId, sessionContext);
-      this.emit('sessionStarted', sessionContext);
+    } catch (error) {
+      this.log('error', `Scrape with actions failed: ${error.message}`);
+      throw new Error(`Scrape with actions failed: ${error.message}`);
+    }
+  }
 
-      try {
-        const result = await this.executeSession(sessionContext);
-        
-        this.stats.successfulSessions++;
-        this.stats.totalActions += validated.actions.length;
-        this.stats.successfulActions += result.actionResults.filter(r => r.success).length;
-        this.stats.failedActions += result.actionResults.filter(r => !r.success).length;
-        
-        const executionTime = Date.now() - startTime;
-        this.updateAverageSessionTime(executionTime);
-        this.updateAverageActionsPerSession(validated.actions.length);
-        this.updateStats();
-
-        if (this.enableCaching) {
-          this.sessionResults.set(sessionId, {
-            result,
-            timestamp: Date.now(),
-            ttl: 3600000
-          });
-        }
+  async runSession(sessionId, validated, enqueuedAt) {
+    // The clock starts once the slot is held, so executionTime and the
+    // recording's timings measure the run, not the wait before it.
+    const startTime = Date.now();
+    const queuedMs = startTime - enqueuedAt;
+
+    // Create session context
+    const sessionContext = {
+      id: sessionId,
+      url: validated.url,
+      startTime,
+      params: validated,
+      states: [],
+      screenshots: [],
+      actionResults: [],
+      errors: [],
+      status: 'initializing'
+    };
 
-        this.activeSessions.delete(sessionId);
-        this.emit('sessionCompleted', result);
+    this.activeSessions.set(sessionId, sessionContext);
+    this.emit('sessionStarted', sessionContext);
 
-        return result;
+    try {
+      const result = await this.executeSession(sessionContext);
+      result.queued_ms = queuedMs;
 
-      } catch (error) {
-        this.stats.failedSessions++;
-        this.activeSessions.delete(sessionId);
-        this.emit('sessionFailed', { sessionId, url: validated.url, error });
-        throw error;
+      this.stats.successfulSessions++;
+      this.stats.totalActions += validated.actions.length;
+      this.stats.successfulActions += result.actionResults.filter(r => r.success).length;
+      this.stats.failedActions += result.actionResults.filter(r => !r.success).length;
+      
+      const executionTime = Date.now() - startTime;
+      this.updateAverageSessionTime(executionTime);
+      this.updateAverageActionsPerSession(validated.actions.length);
+      this.updateStats();
+
+      if (this.enableCaching) {
+        this.sessionResults.set(sessionId, {
+          result,
+          timestamp: Date.now(),
+          ttl: 3600000
+        });
       }
 
+      this.activeSessions.delete(sessionId);
+      this.emit('sessionCompleted', result);
+
+      return result;
+
     } catch (error) {
-      this.log('error', `Scrape with actions failed: ${error.message}`);
-      throw new Error(`Scrape with actions failed: ${error.message}`);
+      this.stats.failedSessions++;
+      this.activeSessions.delete(sessionId);
+      this.emit('sessionFailed', { sessionId, url: validated.url, error });
+      throw error;
     }
   }
 
@@ -532,6 +550,9 @@ export class ScrapeWithActionsTool extends EventEmitter {
       // was chosen (an "auto" that fell back to Chromium says so here).
       engine: browserOptions.stealthMode?.engine || 'chromium',
       ...(warnings.length ? { warnings } : {}),
+      // The landing page's consent result; each navigate action's own is on
+      // its result. Absent when consent handling is off.
+      ...(chainResult.consent ? { consent: chainResult.consent } : {}),
 
       actionResults,
       attempt: chainResult.attempt ?? attempts.length,

package/src/tools/advanced/batchScrape/worker.js

@@ -10,6 +10,7 @@ import { config as appConfig } from '../../../constants/config.js';
 import { ssrfGuard, isSsrfError } from '../../../utils/ssrfGuard.js';
 import { noteRetryAfter } from '../../../utils/hostRateLimiter.js';
 import { preflightFetch } from '../../../utils/robotsGate.js';
+import { fetchResigned } from '../../../utils/resignedFetch.js';
 import { htmlToMarkdown } from '../../../utils/htmlToMarkdown.js';
 import { elementText } from '../../../utils/elementText.js';
 import { pageTitle } from '../../../utils/pageTitle.js';
@@ -31,9 +32,10 @@ export async function fetchUrl(url, options = {}) {
   const controller = new AbortController();
   const timeoutId = setTimeout(() => controller.abort(), timeout);
   try {
-    const response = await fetch(url, {
+    const response = await fetchResigned(url, {
       signal: controller.signal,
       headers: { ...gate.headers, ...headers },
+      onRedirect: gate.onRedirect,
       ...guard
     });
     if (response.status === 429 || response.status === 503) {

package/src/tools/basic/_fetch.js

@@ -8,6 +8,7 @@ import { config } from '../../constants/config.js';
 import { ssrfGuard, isSsrfError } from '../../utils/ssrfGuard.js';
 import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
 import { preflightFetch } from '../../utils/robotsGate.js';
+import { fetchResigned } from '../../utils/resignedFetch.js';
 
 /**
  * Fetch a URL with a configurable timeout and body-size cap.
@@ -51,12 +52,13 @@ export async function fetchWithTimeout(url, options = {}) {
   try {
     let response;
     try {
-      response = await fetch(url, {
+      response = await fetchResigned(url, {
         signal: controller.signal,
         headers: {
           ...gate.headers,
           ...headers
         },
+        onRedirect: gate.onRedirect,
         ...guard
       });
     } catch (error) {

package/src/tools/crawl/_sessionContext.js

@@ -223,7 +223,8 @@ export class SessionContext {
     const fetchOpts = {
       method,
       headers: requestHeaders,
-      redirect: 'follow'
+      redirect: 'follow',
+      onRedirect: gate.onRedirect
     };
 
     if (body) {

package/src/tools/crawl/mapSite.js

@@ -384,7 +384,8 @@ export class MapSiteTool {
     try {
       const response = await safeFetch(url, {
         signal: controller.signal,
-        headers: gate.headers
+        headers: gate.headers,
+        onRedirect: gate.onRedirect
       });
       clearTimeout(timeoutId);
       return response;

package/src/tools/extract/_fetchAndParse.js

@@ -147,6 +147,7 @@ export async function fetchAndParse(url, options = {}) {
       ...gate.headers,
       'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
     },
+    onRedirect: gate.onRedirect,
     signal: AbortSignal.timeout(timeoutMs)
   });
 

package/src/tools/extract/extractContent.js

@@ -176,7 +176,7 @@ export class ExtractContentTool {
             enableImages: options.preserveImageInfo,
             captureScreenshot: false
           }
-        });
+        }, gate);
 
         if (!browserResult.success) {
           throw new Error(`Browser processing failed: ${browserResult.error}`);
@@ -188,6 +188,7 @@ export class ExtractContentTool {
         // Simple HTTP fetch
         const response = await safeFetch(url, {
           headers: { ...gate.headers },
+          onRedirect: gate.onRedirect,
           signal: AbortSignal.timeout(15000)
         });