crawlforge-mcp-server 6.6.1 → 6.7.0

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "6.6.1",
+  "version": "6.7.0",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
-  "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 30 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
+  "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 31 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
   "bin": {
     "crawlforge": "src/cli/index.js",

package/server.js

@@ -61,7 +61,7 @@ import { REDACT_PII_PARAM } from "./src/server/redaction.js"; // Phase 5 (5.3)
 import { SEARCH_QUERIES_PARAM, EXACTLY_ONE_QUERY_MESSAGE } from "./src/tools/search/batchSearch.js"; // Phase 5 (5.1)
 import { markPreflightRefusal } from "./src/server/requestContext.js";
 // D1.1 Resources + D1.2 Prompts + D1.4 Elicitation
-import { ResourceRegistry } from "./src/resources/ResourceRegistry.js";
+import { ResourceRegistry, MAX_RESOURCE_BLOB_BYTES } from "./src/resources/ResourceRegistry.js";
 import { PROMPTS, getPromptMessages } from "./src/prompts/PromptRegistry.js";
 import { ElicitationHelper } from "./src/core/ElicitationHelper.js";
 // Phase 6: MCP-spec adoption — structured output, tool filtering, spec hygiene
@@ -108,7 +108,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "6.6.1",
+  version: "6.7.0",
   description: "Production-ready MCP server with 31 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, stateful browser sessions with element refs, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -1063,9 +1063,24 @@ registerToolIfEnabled("browser_session", {
     // is megabytes beside a few lines of JSON (R21, 2026-09-09).
     const publish = (shot) => {
       if (!shot?.actionId || !shot?.data) return shot;
-      resourceRegistry.storeScreenshot(shot.actionId, shot.data);
+      const { bytes, withinInlineBudget } = resourceRegistry.storeScreenshot(shot.actionId, shot.data);
       const { data, ...rest } = shot;
-      return { ...rest, resourceUri: `crawlforge://screenshot/${shot.actionId}` };
+      return {
+        ...rest,
+        resourceUri: `crawlforge://screenshot/${shot.actionId}`,
+        bytes,
+        // full_page is a knob this tool hands the caller, and on a long page it
+        // produces an image no MCP message can carry. Saying so here costs one
+        // field; learning it from the read costs a wasted call (R23).
+        ...(withinInlineBudget
+          ? {}
+          : {
+              warning: `This image is ${bytes} bytes, too large to read back over MCP ` +
+                `(limit ${MAX_RESOURCE_BLOB_BYTES} bytes) — reading the resource will be refused. ` +
+                `Take it again without full_page, or as format:"jpeg" with a lower quality, or ` +
+                `scoped to one element with selector.`
+            })
+      };
     };
     if (result.screenshot) result.screenshot = publish(result.screenshot);
     if (Array.isArray(result.screenshots)) result.screenshots = result.screenshots.map(publish);
@@ -1348,7 +1363,7 @@ registerToolIfEnabled("stealth_mode", {
         enabled: z.boolean().default(false),
         proxies: z.array(z.string()).optional(),
         rotationInterval: z.number().default(300000)
-      }).optional(),
+      }).optional().describe("Route the browser through your own proxies. Each entry is a proxy URL — \"http://user:pass@host:port\" (percent-encode a password containing @ : or /), or a bare \"host:port\" for an unauthenticated HTTP proxy; http, https, socks4 and socks5 are accepted. rotationInterval is the minimum ms on one proxy before the list advances. Cloudflare scores the IP before it serves a challenge, so a residential proxy is what gets past a block that no fingerprint fixes. CrawlForge supplies no proxies."),
       antiDetection: z.object({
         cloudflareBypass: z.boolean().default(true),
         recaptchaHandling: z.boolean().default(true),

package/src/core/ActionExecutor.js

@@ -1184,12 +1184,13 @@ export class ActionExecutor extends EventEmitter {
     const timeout = this.actionTimeout(action);
 
     await assertUrlAllowed(action.url, { resolveDns: true });
-    await this.assertRobotsAllowed(action.url, executionContext?.browserOptions);
+    const gateWarnings = await this.assertRobotsAllowed(action.url, executionContext?.browserOptions);
 
     await this.navigateToUrl(page, action.url, {
       waitUntil: action.waitUntil,
       timeout
     });
+    page.__crawlforgeGateWarnings = gateWarnings;
 
     return {
       url: action.url,
@@ -1380,9 +1381,13 @@ export class ActionExecutor extends EventEmitter {
    * @throws {BlockedHostError|RobotsDisallowedError}
    */
   async assertRobotsAllowed(url, browserOptions = {}) {
-    await browserPreflight(url, {
+    return await browserPreflight(url, {
       respectRobots: browserOptions?.respectRobots,
-      tool: 'scrape_with_actions'
+      // The audit row is the record of the CUSTOMER's decision (G5), so it has
+      // to name the tool that actually made it. browser_session borrows this
+      // executor, and until R23 every session's override was filed against
+      // scrape_with_actions.
+      tool: browserOptions?.tool || 'scrape_with_actions'
     });
   }
 
@@ -1402,13 +1407,17 @@ export class ActionExecutor extends EventEmitter {
     // or a disallowed path never costs a Chromium process. preflightFetch is
     // deliberately not used here: its identity/signature headers belong on an
     // HTTP fetch, not on a browser context.
-    await this.assertRobotsAllowed(url, browserOptions);
+    const gateWarnings = await this.assertRobotsAllowed(url, browserOptions);
 
     const isStealth = !!browserOptions.stealthMode?.enabled;
 
     // Use the enhanced BrowserProcessor initialization that supports stealth mode
     const page = await this.browserProcessor.initializePage(browserOptions);
 
+    // Stamped on the page for the same reason __crawlforgeNavigation is: the
+    // caller's result is assembled a layer up, and the gate ran a layer down.
+    page.__crawlforgeGateWarnings = gateWarnings;
+
     try {
       // Apply CloudFlare and reCAPTCHA detection if stealth mode is enabled
       if (isStealth && this.browserProcessor.stealthManager) {

package/src/core/ResearchOrchestrator.js

@@ -10,6 +10,7 @@ import { Logger } from '../utils/Logger.js';
 import { LLMManager } from './llm/LLMManager.js';
 import { safeFetch, safeGoto } from '../utils/ssrfGuard.js';
 import { preflightFetch, browserPreflight } from '../utils/robotsGate.js';
+import { guardFirefoxPageErrors } from '../utils/firefoxPageErrorGuard.js';
 import { noteRetryAfter } from '../utils/hostRateLimiter.js';
 import {
   isAdmissibleClaim,
@@ -956,6 +957,9 @@ export class ResearchOrchestrator extends EventEmitter {
             const require = createRequire(import.meta.url);
             const camoufox = require('camoufox'); // CJS build — ESM build is broken
             await this._ensureCamoufoxLayout(camoufox);
+            // A page whose own JavaScript throws must not take the process with
+            // it — see src/utils/firefoxPageErrorGuard.js.
+            guardFirefoxPageErrors();
             this._stealthBrowser = await camoufox.Camoufox({ headless: true });
             this._stealthEngineActive = 'camoufox';
             this.logger.info('Stealth fallback using Camoufox (Firefox) engine');

package/src/core/StealthBrowserManager.js

@@ -15,10 +15,15 @@ import HumanBehaviorSimulator from '../utils/HumanBehaviorSimulator.js';
 import { BrowserContextPool } from './BrowserContextPool.js';
 import { safeGoto } from '../utils/ssrfGuard.js';
 import { detectChallengePage } from '../utils/challengeDetection.js';
+import { guardFirefoxPageErrors } from '../utils/firefoxPageErrorGuard.js';
 
 // Grace given to a document that rendered no title and no text (see _waitOutEmptyDocument).
 export const EMPTY_DOCUMENT_GRACE_MS = 8000;
 
+// The proxy a camoufox browser was launched with, kept on the browser itself so
+// it survives being parked and restored by an engine switch.
+const CAMOUFOX_PROXY = Symbol('crawlforge.camoufoxProxy');
+
 const StealthConfigSchema = z.object({
   level: z.enum(['basic', 'medium', 'advanced']).default('medium'),
   randomizeFingerprint: z.boolean().default(true),
@@ -342,10 +347,28 @@ export class StealthBrowserManager {
           'camoufox is not installed. Run: npm install camoufox to use the Firefox-based stealth engine.'
         );
       }
-      this.browser = await adapter.launch({
+      // camoufox fixes its fingerprint — and, with geoip, its geolocation,
+      // timezone and locale — at launch, from the proxy it is launched with.
+      // So the proxy is resolved once here and reused for every context this
+      // browser serves (see createStealthContext): rotating underneath it would
+      // leave camoufox reporting the first proxy's city behind the second
+      // proxy's exit IP, which is a worse signal than not rotating at all.
+      // A rotation takes effect on the next launch, after cleanup().
+      const proxy = this.resolveProxy(validatedConfig);
+      const browser = await adapter.launch({
         headless: true,
+        proxy,
+        // Only ask for geoip when there is a proxy to derive it from. Without
+        // one it would look up this machine's own public IP — a network call,
+        // and an external service learning our address, to confirm a location
+        // the browser is already in.
+        geoip: !!proxy,
+        blockWebRTC: validatedConfig.blockWebRTC,
+        humanize: validatedConfig.simulateHumanBehavior,
         launchOptions: {}
       });
+      browser[CAMOUFOX_PROXY] = proxy;
+      this.browser = browser;
       this._launchedEngine = 'camoufox';
       return this.browser;
     }
@@ -424,11 +447,11 @@ export class StealthBrowserManager {
       );
     }
 
-    // Handle proxy configuration
-    const currentProxy = await this.rotateProxy(validatedConfig);
-    if (currentProxy) {
-      stealthArgs.push(`--proxy-server=${currentProxy}`);
-    }
+    // No proxy argument here. Chromium's --proxy-server= has no field for the
+    // user:pass every residential proxy requires, and a proxy fixed at launch
+    // could never rotate: the browser it was baked into is cached for the life
+    // of the process. Both engines take the proxy per context instead — see
+    // createStealthContext.
 
     const browser = await chromium.launch({
       headless: true,
@@ -507,19 +530,69 @@ export class StealthBrowserManager {
       serviceWorkers: 'block'
     };
 
-    // camoufox's Firefox build predates the Browser.setDefaultViewport fields
-    // playwright-core 1.62 sends (screenSize, isMobile, ...) and rejects unknown
-    // properties, so any fixed viewport fails. viewport:null skips that protocol
-    // call entirely (deviceScaleFactor/isMobile/hasTouch/screen are invalid or
-    // meaningless without a viewport). window.screen is still spoofed via
-    // addInitScript in applyAdvancedStealthConfigurations.
+    // The proxy rides on the context, credentials included: Chromium takes it
+    // on Target.createBrowserContext and camoufox's Juggler on
+    // Browser.setContextProxy. Both were verified against a local authenticating
+    // proxy. camoufox additionally gets one at launch, because its geoip lookup
+    // runs there — but a launch-time proxy routes traffic and drops the
+    // credentials, so the context is what actually authenticates, on both
+    // engines. camoufox stays on the proxy it was launched with: its geolocation,
+    // timezone and locale were derived from that exit IP, and rotating
+    // underneath it would leave the first proxy's city behind the second
+    // proxy's address.
+    const proxy = this._launchedEngine === 'camoufox'
+      ? (this.browser[CAMOUFOX_PROXY] || null)
+      : this.resolveProxy(validatedConfig);
+    if (proxy) {
+      contextOptions.proxy = proxy;
+    }
+
     if (this._launchedEngine === 'camoufox') {
+      // camoufox's Firefox build predates the Browser.setDefaultViewport fields
+      // playwright-core 1.62 sends (screenSize, isMobile, ...) and rejects
+      // unknown properties, so any fixed viewport fails. viewport:null skips
+      // that protocol call entirely (deviceScaleFactor/isMobile/hasTouch/screen
+      // are invalid or meaningless without a viewport). camoufox generates its
+      // own screen and window and spoofs them below the JS layer.
       contextOptions.viewport = null;
       delete contextOptions.deviceScaleFactor;
       delete contextOptions.isMobile;
       delete contextOptions.hasTouch;
       delete contextOptions.screen;
       delete contextOptions.serviceWorkers;
+
+      // camoufox arrives with a complete Firefox identity of its own. Ours was
+      // overwriting it, and not with a Firefox one: for this engine the browser
+      // distribution is left open, so about two thirds of camoufox contexts were
+      // handed a Chrome User-Agent on a Gecko engine. Every one of them also
+      // sent sec-ch-ua, sec-ch-ua-mobile and sec-ch-ua-platform — client hints
+      // Firefox has never implemented and never sends. That is a decision a
+      // detector can make from the request headers alone, before a line of
+      // script runs. Playwright still derives a correctly shaped Accept-Language
+      // from `locale`, so dropping these loses nothing real.
+      delete contextOptions.userAgent;
+      delete contextOptions.extraHTTPHeaders;
+
+      // Behind a proxy, camoufox has already derived locale, timezone and
+      // geolocation from the exit IP. Those three agree with the address the
+      // site sees; a persona drawn here does not, so it must not override them.
+      if (proxy) {
+        delete contextOptions.locale;
+        delete contextOptions.timezoneId;
+        delete contextOptions.geolocation;
+      }
+
+      // Keep what we hand back honest. create_context returns this fingerprint,
+      // and reporting a Chrome user agent and a persona the browser never uses
+      // is worse than reporting nothing: null reads as "the engine owns this".
+      fingerprint.userAgent = null;
+      fingerprint.headers = {};
+      fingerprint.viewport = null;
+      fingerprint.hardware = { ...fingerprint.hardware, platform: null };
+      if (proxy) {
+        fingerprint.locale = null;
+        fingerprint.timezone = null;
+      }
     }
 
     const context = await this.browser.newContext(contextOptions);
@@ -590,8 +663,11 @@ export class StealthBrowserManager {
       platform: fingerprint.hardware.platform,
       locale: fingerprint.locale,
       timezone: fingerprint.timezone,
-      // width/height only — the pool's selection weight is an internal.
-      viewport: { width: fingerprint.viewport.width, height: fingerprint.viewport.height }
+      // width/height only — the pool's selection weight is an internal. null on
+      // camoufox, which sizes its own window.
+      viewport: fingerprint.viewport
+        ? { width: fingerprint.viewport.width, height: fingerprint.viewport.height }
+        : null
     };
   }
 
@@ -1249,6 +1325,24 @@ export class StealthBrowserManager {
    * Apply advanced stealth configurations to browser context
    */
   async applyAdvancedStealthConfigurations(context, config, fingerprint) {
+    // Nothing is injected into camoufox.
+    //
+    // Every script below is Chromium-shaped — it deletes navigator.webdriver,
+    // installs a window.chrome, and patches getContext, AudioContext and font
+    // metrics from the main world. camoufox does all of that in its own
+    // C++/Juggler layer, where there is no JS seam to find, and a page that
+    // compares property descriptors, Function.prototype.toString output or the
+    // main thread against a Worker sees ours and not camoufox's. Injecting on
+    // top of an engine built to need no injection only adds back the tells the
+    // engine was chosen to avoid.
+    //
+    // This does not clear deviceandbrowserinfo.com's hasInconsistentWorkerValues
+    // on camoufox: that flag stayed set with every one of these disabled, so it
+    // is camoufox's own worker leak, not ours. What it removes is our share.
+    if (this._launchedEngine === 'camoufox') {
+      return;
+    }
+
     // Enhanced initialization script with comprehensive stealth measures
     await context.addInitScript((locale) => {
       // Remove webdriver property completely
@@ -1636,8 +1730,33 @@ export class StealthBrowserManager {
       // script; relative importScripts/fetch inside such a worker resolve
       // against the original script URL. Module workers, data:/blob: worker
       // URLs and a CSP that refuses blob workers fall through to the native
-      // constructor untouched. camoufox spoofs workers itself.
-      if (config.level === 'advanced' && this._launchedEngine !== 'camoufox') {
+      // constructor untouched. (camoufox never reaches here — this whole method
+      // returns early for it.)
+      //
+      // MEASURED COVERAGE, 2026-09-16, chromium, comparing a worker's navigator
+      // with the main thread's:
+      //
+      //   level      new Worker('/w.js')   new Worker(blob:)
+      //   medium     leaks all four        leaks all four
+      //   advanced   matches               leaks all four
+      //
+      // The four are platform, hardwareConcurrency, deviceMemory and languages;
+      // WebGL's unmasked vendor/renderer leaks the same way, because the
+      // prototype patch above lives in the window, not in a worker scope. So a
+      // detector that builds its worker from a Blob — which needs no second
+      // request and is the usual shape — reads straight past this at every
+      // level, and deviceandbrowserinfo.com's hasInconsistentWorkerValues
+      // stays set. Every one of those mismatches is ours: a worker reports the
+      // truth, and only the main thread was spoofed.
+      //
+      // Closing it properly means one of two things, and both are trades:
+      // rewrite blob/data worker sources so the patch reaches them (more
+      // surface, and it still misses module workers and SharedWorker), or stop
+      // spoofing in the window what cannot be spoofed in a worker (consistent,
+      // but then the real values show — 32 cores and a SwiftShader GPU, which
+      // is its own datacenter tell). Left as it is pending that call; do not
+      // read the `advanced` gate as coverage.
+      if (config.level === 'advanced') {
         await context.addInitScript(({ hardware, locale }) => {
           const NativeWorker = window.Worker;
           if (typeof NativeWorker !== 'function') return;
@@ -1960,25 +2079,93 @@ export class StealthBrowserManager {
   }
 
   /**
-   * Proxy rotation management
+   * Parse one proxyRotation entry into the { server, username, password } shape
+   * Playwright takes.
+   *
+   * Residential proxies — the only kind that helps against Cloudflare's IP
+   * reputation check — are issued as `http://user:pass@host:port`. Those
+   * credentials are the whole point: the previous code pushed the entry into
+   * `--proxy-server=`, a Chromium flag with nowhere to put them, so every
+   * authenticating proxy answered 407 and the request failed. Splitting them out
+   * here is what lets both engines authenticate.
+   *
+   * A malformed entry throws rather than returning null. A proxy that silently
+   * does not apply is the failure mode this whole path is being fixed for: the
+   * caller believes their traffic is proxied and it is not.
    */
-  async rotateProxy(config) {
-    if (!config.proxyRotation?.enabled || !config.proxyRotation?.proxies?.length) {
+  parseProxyEntry(entry) {
+    if (typeof entry !== 'string' || !entry.trim()) {
+      throw new Error('proxyRotation.proxies entries must be non-empty strings');
+    }
+    const raw = entry.trim();
+    // `host:port` with no scheme parses as protocol "host:" and an empty host,
+    // so give the bare form the http:// every proxy list assumes.
+    const withScheme = /^[a-z][a-z0-9+.-]*:\/\//i.test(raw) ? raw : `http://${raw}`;
+
+    let url;
+    try {
+      url = new URL(withScheme);
+    } catch {
+      throw new Error(`Invalid proxy "${this.redactProxy(raw)}": expected host:port or scheme://user:pass@host:port`);
+    }
+
+    const scheme = url.protocol.replace(':', '').toLowerCase();
+    if (!['http', 'https', 'socks4', 'socks5'].includes(scheme)) {
+      throw new Error(`Invalid proxy scheme "${scheme}": expected http, https, socks4 or socks5`);
+    }
+    if (!url.hostname) {
+      throw new Error(`Invalid proxy "${this.redactProxy(raw)}": no host`);
+    }
+
+    // url.origin is the string "null" for socks4/socks5 — they are not special
+    // schemes — so the server is rebuilt from protocol and host.
+    const proxy = { server: `${url.protocol}//${url.host}` };
+    // A password with a "@" or ":" in it must arrive percent-encoded to parse at
+    // all; the proxy expects the decoded value.
+    if (url.username) proxy.username = decodeURIComponent(url.username);
+    if (url.password) proxy.password = decodeURIComponent(url.password);
+    return proxy;
+  }
+
+  /** A proxy entry with its credentials removed, for logs and get_stats. */
+  redactProxy(entry) {
+    return String(entry).replace(/\/\/[^/@]*@/, '//');
+  }
+
+  /**
+   * Pick the proxy this context should use, advancing the rotation when its
+   * interval has elapsed.
+   *
+   * Called per context rather than per launch. The old call site ran once, at
+   * browser launch, and the browser is cached for the life of the process — so
+   * rotationInterval could never elapse anywhere that mattered and the second
+   * proxy in a list was never reached.
+   */
+  resolveProxy(config) {
+    const proxies = config.proxyRotation?.enabled ? (config.proxyRotation.proxies || []) : [];
+    if (!proxies.length) {
       return null;
     }
-    
+
     const now = Date.now();
-    const { rotationInterval, proxies } = config.proxyRotation;
-    
-    if (now - this.proxyManager.lastRotation > rotationInterval) {
+    if (this.proxyManager.currentProxy === null) {
+      // First use takes proxies[0]. The old code advanced the index before its
+      // first read, so a single-proxy list worked by wrapping to 0 and every
+      // longer list silently started at the second entry.
+      this.proxyManager.proxyIndex = 0;
+      this.proxyManager.lastRotation = now;
+    } else if (now - this.proxyManager.lastRotation > config.proxyRotation.rotationInterval) {
       this.proxyManager.proxyIndex = (this.proxyManager.proxyIndex + 1) % proxies.length;
-      this.proxyManager.currentProxy = proxies[this.proxyManager.proxyIndex];
       this.proxyManager.lastRotation = now;
-      
-      console.error('Rotated to proxy:', this.proxyManager.currentProxy);
     }
-    
-    return this.proxyManager.currentProxy;
+
+    const entry = proxies[this.proxyManager.proxyIndex % proxies.length];
+    const parsed = this.parseProxyEntry(entry);
+    // Only ever hold the redacted form: getStats() returns currentProxy to the
+    // caller, and these strings carry a password.
+    this.proxyManager.currentProxy = this.redactProxy(entry);
+    this.proxyManager.activeProxies = proxies.map((p) => this.redactProxy(p));
+    return parsed;
   }
 
   /**
@@ -2056,6 +2243,80 @@ export class StealthBrowserManager {
     return Date.now() - started;
   }
 
+  /**
+   * Give the page the chance to finish rendering before it is read.
+   *
+   * Navigation returns at DOMContentLoaded, and everything before this only
+   * waits for the document to become non-empty (_waitOutEmptyDocument) or to
+   * stop being an interstitial (_waitOutChallenge). A page that already has
+   * prose and writes the part the caller came for in a load handler passed all
+   * of those as finished, and the read landed mid-render.
+   *
+   * The 6.6.2 bench recorded that as a format bug — "markdown silently dropped
+   * the verdict, text captured it" — but its two calls were two page loads and
+   * only one of them raced. One call asking for markdown and text returns the
+   * same content in both, because both are built from one render. The defect
+   * was the timing.
+   *
+   * Two bounded waits: the page's own load event, then quiet in the DOM. What
+   * this cannot do is predict a payload injected into a still page some
+   * arbitrary time later — that is what the caller's `wait_for` is for.
+   *
+   * @returns {Promise<number>} milliseconds actually waited, 0 for a page that
+   *   was already finished.
+   */
+  async _settleRender(page, { loadTimeoutMs = 3000, quietMs = 400, capMs = 2500 } = {}) {
+    const started = Date.now();
+    // Subresources are often what the render waits on. Bounded, because a page
+    // with a hanging tracker request never fires load at all.
+    await page.waitForLoadState('load', { timeout: loadTimeoutMs }).catch(() => {});
+    const loadMs = Date.now() - started;
+    return loadMs + await this._settleDom(page, { quietMs, capMs });
+  }
+
+  /**
+   * Resolve once the DOM has been unchanged for `quietMs`, or at `capMs`.
+   *
+   * A MutationObserver in the page, so this is one round trip and returns
+   * immediately on a page that was already still. The cap bounds a page that
+   * never stops animating.
+   *
+   * @returns {Promise<number>} milliseconds the DOM went on changing for — 0
+   *   when the page was already still, so a settled page reports no extra wait.
+   */
+  async _settleDom(page, { quietMs = 400, capMs = 2500 } = {}) {
+    try {
+      return await page.evaluate(({ quiet, cap }) => new Promise((resolve) => {
+        if (!document.documentElement) {
+          resolve(0);
+          return;
+        }
+        const started = Date.now();
+        let quietTimer;
+        const finish = () => {
+          observer.disconnect();
+          clearTimeout(quietTimer);
+          clearTimeout(capTimer);
+          // Subtract the quiet window itself: what is worth reporting is how
+          // long the page kept changing, not the time spent confirming it had
+          // stopped.
+          resolve(Math.max(0, Date.now() - started - quiet));
+        };
+        const observer = new MutationObserver(() => {
+          clearTimeout(quietTimer);
+          quietTimer = setTimeout(finish, quiet);
+        });
+        const capTimer = setTimeout(finish, cap);
+        quietTimer = setTimeout(finish, quiet);
+        observer.observe(document.documentElement, { childList: true, subtree: true, characterData: true });
+      }), { quiet: quietMs, cap: capMs });
+    } catch {
+      // Navigated away, closed or crashed mid-settle. The read that follows
+      // reports that properly; there is nothing to add here.
+      return 0;
+    }
+  }
+
   async _waitOutChallenge(page, { timeoutMs = 8000 } = {}) {
     let title;
     try {
@@ -2100,7 +2361,13 @@ export class StealthBrowserManager {
       // up, so the interstitial came back as success:true (R17, 2026-09-04).
       // An auto-solving challenge gets one bounded wait to finish first.
       await this._waitOutChallenge(page);
-      const gracedMs = await this._waitOutEmptyDocument(page);
+      const emptyGraceMs = await this._waitOutEmptyDocument(page);
+      // Last: let whatever is still rendering finish. Without this the read
+      // below can land between "the page has content" and "the page has the
+      // content the caller came for", and a half-rendered page is returned as
+      // a successful scrape.
+      const renderMs = await this._settleRender(page);
+      const gracedMs = emptyGraceMs + renderMs;
 
       // A failure to read the document is a failure. With every read wrapped
       // in .catch(() => ''), a renderer that crashed or a page closed during
@@ -2166,8 +2433,13 @@ export class StealthBrowserManager {
       }
     });
 
-    // Add request headers
-    await page.setExtraHTTPHeaders(fingerprint.headers);
+    // Add request headers — Chromium only. These carry sec-ch-ua,
+    // sec-ch-ua-mobile and sec-ch-ua-platform, client hints Gecko does not
+    // implement, so setting them here would put them straight back onto a
+    // camoufox page after createStealthContext had taken them off the context.
+    if (isChromium(page)) {
+      await page.setExtraHTTPHeaders(fingerprint.headers);
+    }
 
     // Emulate realistic network conditions.
     //
@@ -2556,14 +2828,52 @@ export class CamoufoxAdapter extends BrowserEngine {
 
     await this._ensureMacOSLayout(camoufox);
 
+    // Before any Firefox page can run: camoufox reports an uncaught page error
+    // with no location, and playwright reads one anyway. See the guard.
+    guardFirefoxPageErrors();
+
     // camoufox's launcher is Camoufox(options) — the package has no launch()
     // export. It resolves the fetched Firefox binary (npx camoufox fetch) and
     // returns a Playwright-compatible Browser. Takes `headless` directly plus
     // passthrough Playwright Firefox launch options.
-    return camoufox.Camoufox({
+    //
+    // The snake_case names below are camoufox's own option names. They are the
+    // reason to run this engine at all: camoufox spoofs at the C++/Juggler
+    // level, where a page cannot see the seam, and every one of these was
+    // simply not being passed — camoufox ran with its own features off.
+    const options = {
       headless: config.headless !== false,
       ...config.launchOptions
-    });
+    };
+    // A bare `{ server }` is fine here; camoufox normalises both shapes.
+    if (config.proxy) options.proxy = config.proxy;
+    // geoip derives longitude, latitude, timezone, country and locale from the
+    // proxy's exit IP, which is the one thing that makes a proxied browser
+    // coherent. It costs a request through the proxy, and the first ever call
+    // downloads MaxMind's city database (~60 MB) into camoufox's install dir.
+    if (config.geoip) options.geoip = true;
+    // Blocking WebRTC is itself a signal. With geoip camoufox instead reports
+    // the proxy's exit IP through WebRTC, which is the coherent answer — so
+    // `blockWebRTC: false` is the stealthier setting behind a proxy.
+    if (config.blockWebRTC) options.block_webrtc = true;
+    // Native cursor humanization: camoufox moves the pointer along a plausible
+    // path rather than teleporting it.
+    if (config.humanize) options.humanize = true;
+
+    try {
+      return await camoufox.Camoufox(options);
+    } catch (err) {
+      if (options.geoip) {
+        // Do not quietly retry without geoip. A proxied camoufox whose
+        // geolocation says one country while its exit IP says another is a
+        // cleaner detection signal than an unproxied one.
+        throw new Error(
+          `camoufox failed to launch with geoip through the configured proxy: ${err.message}. ` +
+          'Check the proxy credentials and that the proxy can reach the internet.'
+        );
+      }
+      throw err;
+    }
   }
 
   /**

package/src/resources/ResourceRegistry.js

@@ -6,6 +6,23 @@
 
 import { createHash } from 'crypto';
 
+/**
+ * The stdio transport frames one JSON-RPC message at a time, and since SDK
+ * 1.30 its ReadBuffer CLOSES the transport when a message runs past
+ * STDIO_MESSAGE_CEILING_BYTES: the whole session dies, every tool with it, and
+ * the overflow is not recoverable (modelcontextprotocol/typescript-sdk#2793).
+ * A full-page PNG of a long article is 17.4 MB — en.wikipedia.org/wiki/World_War_II
+ * is 61,341px tall — and JPEG only brings it to 11.3 MB, so neither format saves
+ * a caller who passes full_page (R23, 2026-09-13).
+ *
+ * Hence a budget on the blob a read may emit: base64 costs four bytes per three,
+ * and a tenth of the ceiling is left for the JSON-RPC envelope around it.
+ */
+const STDIO_MESSAGE_CEILING_BYTES = 10 * 1024 * 1024;
+export const MAX_RESOURCE_BLOB_BYTES =
+  Number(process.env.CRAWLFORGE_MAX_RESOURCE_BLOB_BYTES) ||
+  Math.floor(STDIO_MESSAGE_CEILING_BYTES * 0.9 * 3 / 4);
+
 /**
  * Supported resource types and their MIME types.
  */
@@ -84,6 +101,12 @@ export class ResourceRegistry {
       createdAt: Date.now(),
       ttl: this.defaultTtl,
     });
+    // Handed back so the tool that took the shot can say, in the same result
+    // that carries the URI, whether that URI is readable over stdio at all.
+    return {
+      bytes: buf.length,
+      withinInlineBudget: buf.length <= MAX_RESOURCE_BLOB_BYTES,
+    };
   }
 
   /**
@@ -152,7 +175,7 @@ export class ResourceRegistry {
         resources.push({
           uri: `crawlforge://screenshot/${actionId}`,
           name: `Screenshot ${actionId}`,
-          description: 'Screenshot from scrape_with_actions',
+          description: 'Screenshot from a CrawlForge browser tool',
           mimeType: RESOURCE_MIME.screenshot,
         });
       }
@@ -265,6 +288,17 @@ export class ResourceRegistry {
     if (!entry || Date.now() - entry.createdAt >= entry.ttl) {
       throw new Error(`Screenshot not found or expired: ${actionId}`);
     }
+    // Refuse rather than emit: an oversized message costs the caller their
+    // whole session, and this error costs them one read they can act on.
+    if (entry.data.length > MAX_RESOURCE_BLOB_BYTES) {
+      throw new Error(
+        `Screenshot ${actionId} is ${entry.data.length} bytes, over the ` +
+        `${MAX_RESOURCE_BLOB_BYTES}-byte limit for one MCP message. Returning it would ` +
+        `close the connection instead of failing this read, so it is refused. Take the ` +
+        `shot again without full_page, or as format:"jpeg" with a lower quality, or ` +
+        `scoped to one element with selector.`
+      );
+    }
     return {
       contents: [{
         uri,

package/src/skills/agent-skills/crawlforge-stealth-browsing/SKILL.md

@@ -68,6 +68,30 @@ Operations: `configure`, `enable`, `disable`, `create_context`, `create_page`,
 
 Full decision table: [engine selection](references/engine-selection.md).
 
+### Proxies
+
+A block that survives both engines is usually the IP, not the fingerprint:
+Cloudflare scores the address and its ASN before it serves a challenge. Route
+through your own residential proxy — CrawlForge supplies none.
+
+```json
+{
+  "tool": "stealth_mode",
+  "params": {
+    "operation": "scrape", "url": "https://protected-site.com", "engine": "camoufox",
+    "stealthConfig": {
+      "proxyRotation": { "enabled": true, "proxies": ["http://user:p%40ss@gw.provider.net:8080"] }
+    }
+  }
+}
+```
+
+Credentials go in the URL, percent-encoded if the password contains `@`, `:` or
+`/`. `http`, `https`, `socks4` and `socks5` are accepted. With a proxy, camoufox
+derives its timezone, locale and geolocation from the exit IP, so the browser
+agrees with the address the site sees — that lookup happens at launch, so a
+camoufox browser keeps one proxy until `cleanup`.
+
 ### CLI
 
 ```bash

package/src/tools/advanced/BrowserSessionTool.js

@@ -170,6 +170,21 @@ function withJsResult(result) {
   return { ...result, jsResult: result.result.result };
 }
 
+/**
+ * The gate's warnings — a respect_robots override, a crawl-delay note — are
+ * what the shared parameter description promises the caller gets back
+ * ("returns a warning in the response"). `scrape` publishes them through
+ * unifiedScrape; the browser path dropped them on the floor, so a session that
+ * overrode robots.txt was told nothing at all (R23, 2026-09-13).
+ *
+ * Last navigation wins, the same rule `__crawlforgeNavigation` follows: the
+ * warnings describe the hop the caller just made, not every hop of the session.
+ */
+function gateWarningFields(page) {
+  const warnings = page?.__crawlforgeGateWarnings;
+  return warnings?.length ? { warnings } : {};
+}
+
 export class BrowserSessionTool {
   constructor(options = {}) {
     const {
@@ -278,7 +293,8 @@ export class BrowserSessionTool {
       viewportWidth: params.viewport?.width,
       viewportHeight: params.viewport?.height,
       timeout: params.timeout,
-      respectRobots: params.respect_robots
+      respectRobots: params.respect_robots,
+      tool: 'browser_session'
     };
     if (params.stealth) {
       browserOptions.stealthMode = { enabled: true };
@@ -325,6 +341,7 @@ export class BrowserSessionTool {
       operation: 'open',
       ...sessionInfo(session),
       ...verdictFields(verdict),
+      ...gateWarningFields(page),
       // The page is a wall, but the session behind it is real and holds a
       // browser context — say so, or a caller reading only `success` abandons
       // it to its TTL instead of closing it or acting through the challenge.
@@ -411,7 +428,7 @@ export class BrowserSessionTool {
     const result = await this.actionExecutor.executeActionsOnPage(session.page, params.actions, {
       continueOnError: params.continue_on_error,
       timeout: params.timeout,
-      browserOptions: { respectRobots: params.respect_robots }
+      browserOptions: { respectRobots: params.respect_robots, tool: 'browser_session' }
     });
 
     this.store.touch(session, result.finalUrl);
@@ -424,6 +441,7 @@ export class BrowserSessionTool {
       ...(Number.isInteger(session.page.__crawlforgeNavigation?.status)
         ? { httpStatus: session.page.__crawlforgeNavigation.status }
         : {}),
+      ...gateWarningFields(session.page),
       error: result.error,
       actionResults: result.results.map(withJsResult),
       screenshots: result.screenshots,

package/src/utils/firefoxPageErrorGuard.js

@@ -0,0 +1,93 @@
+/**
+ * Stops a Firefox page's own JavaScript error from taking the process down.
+ *
+ * Camoufox's Juggler reports an uncaught page error without a `location`:
+ *
+ *   ◀ RECV {"method":"Page.uncaughtError","params":{
+ *       "frameId":"mainframe-8",
+ *       "message":"TypeError: null has no properties",
+ *       "stack":"@http://127.0.0.1:55712/:3:13\n"}}
+ *
+ * playwright-core 1.62's FFPage._onUncaughtError forwards that missing
+ * `params.location` into Page.addPageError, and the BrowserContext dispatcher
+ * then reads it unconditionally:
+ *
+ *   location: { url: pageError.location.url, ... }
+ *
+ * The resulting "Cannot read properties of undefined (reading 'url')" is thrown
+ * inside the protocol dispatch loop, where nothing awaits it — it arrives as an
+ * uncaughtException. A plain `<script>null.boom;</script>` on any page is enough
+ * to trigger it, and bot.sannysoft.com does it in the ordinary course of
+ * running its checks.
+ *
+ * Under server.js the catch-all uncaughtException handler absorbs it and the
+ * scrape still returns, but every other consumer — the CLI, deep_research when
+ * it is embedded, any direct use of StealthBrowserManager — dies on the spot.
+ * Leaning on that handler is not a fix either: it logs a stack trace on a page
+ * that did nothing wrong, and Node treats the process state as undefined
+ * afterwards.
+ *
+ * So the missing value is filled in at its source. Firefox told us a page error
+ * happened but not where; the zeroed location says exactly that, and everything
+ * downstream — including the `pageerror` event a caller can listen for —
+ * behaves normally. Chromium and WebKit always send a location, so they never
+ * reach the fallback.
+ *
+ * Reaching into playwright's internals is deliberate and bounded: this patches
+ * one method, only when it exists and has not already been patched, and any
+ * failure leaves the original in place. A playwright that fixes this upstream,
+ * or renames the method, turns it into a no-op rather than a breakage. The
+ * `playwright-core/lib/coreBundle` specifier is one playwright-core publishes in
+ * its own `exports` map, and it resolves to the same module instance playwright
+ * itself loads. playwright-core is not declared as a dependency on purpose: it
+ * must stay the exact version playwright pins, so it is only ever reached
+ * through the copy playwright brought.
+ */
+
+import { createRequire } from 'module';
+
+const require = createRequire(import.meta.url);
+
+// Firefox reports that an error happened but not where. Zeros say "unknown"
+// without inventing a file or a line number.
+const UNKNOWN_LOCATION = Object.freeze({ url: '', lineNumber: 0, columnNumber: 0 });
+
+let applied = false;
+
+/**
+ * Idempotent, best-effort, and safe to call before every Firefox/Camoufox
+ * launch. Returns true when the guard is in place (including when a previous
+ * call installed it), false when this playwright build did not match.
+ */
+export function guardFirefoxPageErrors() {
+  if (applied) return true;
+
+  try {
+    const bundle = require('playwright-core/lib/coreBundle');
+    const prototype = bundle?.server?.Page?.prototype;
+    if (!prototype || typeof prototype.addPageError !== 'function') return false;
+    if (prototype.addPageError.__crawlforgeGuarded) {
+      applied = true;
+      return true;
+    }
+
+    const original = prototype.addPageError;
+    function addPageError(error, location) {
+      return original.call(this, error, location || UNKNOWN_LOCATION);
+    }
+    addPageError.__crawlforgeGuarded = true;
+    prototype.addPageError = addPageError;
+
+    applied = true;
+    return true;
+  } catch {
+    // A playwright that moved or sealed this is not a reason to fail a launch;
+    // the crash it prevents is rarer than the launch it would break.
+    return false;
+  }
+}
+
+/** Test seam: forget that the guard was installed. Does not un-patch. */
+export function _resetFirefoxPageErrorGuard() {
+  applied = false;
+}