crawlforge-mcp-server 6.7.0 → 6.8.0

package/src/core/creatorMode.js

@@ -29,7 +29,7 @@ for (const [key, value] of Object.entries(cwdEnv)) {
 // SECURITY: Clear any externally-set creator mode env var to prevent bypass
 delete process.env.CRAWLFORGE_CREATOR_MODE;
 
-const CREATOR_SECRET_HASH = 'cfef62e5068d48e7dd6a39c9e16f0be2615510c6b68274fc8abe3156feb5050b';
+const CREATOR_SECRET_HASH = '52645de07b540d6a42e21fffe20dc77961d144d8dc388d82aa22d20ac4ba31a9';
 
 // Module-scoped flag — cannot be set externally
 let _creatorModeVerified = false;

package/src/core/processing/BrowserProcessor.js

@@ -47,7 +47,11 @@ const BrowserProcessorSchema = z.object({
       simulateHumanBehavior: z.boolean().default(true),
       customUserAgent: z.string().optional(),
       hideWebDriver: z.boolean().default(true),
-      blockWebRTC: z.boolean().default(true)
+      blockWebRTC: z.boolean().default(true),
+      // Already resolved to a concrete engine by the caller (Phase 2): the
+      // tools call resolveStealthEngine() so they can report a camoufox→
+      // chromium fallback in their own result. Nothing 'auto' reaches here.
+      engine: z.enum(['chromium', 'camoufox']).optional()
     }).optional(),
     
     // Human behavior simulation options
@@ -307,7 +311,12 @@ export class BrowserProcessor {
     } else {
       // Standard browser initialization
       await this.initBrowser();
-      return await this.createPage(processedOptions);
+      const page = await this.createPage(processedOptions);
+      // Stamped for the same reason __crawlforgeGateWarnings is: the caller's
+      // result is assembled a layer up, and only this layer knows which engine
+      // actually ran. The standard pool is Chromium; camoufox is stealth-only.
+      page.__crawlforgeEngine = 'chromium';
+      return page;
     }
   }
 
@@ -356,8 +365,15 @@ export class BrowserProcessor {
       });
     }
 
+    // The engine the caller resolved (Phase 2). It has to reach BOTH calls:
+    // launchStealthBrowser parks the running browser on a mismatch, and
+    // createStealthContext re-validates the config it is given — so a context
+    // asked for without it would relaunch on the default engine underneath.
+    const engine = options.stealthMode.engine || 'chromium';
+
     // Launch stealth browser
     await this.stealthManager.launchStealthBrowser({
+      engine,
       level: options.stealthMode.level,
       randomizeFingerprint: options.stealthMode.randomizeFingerprint,
       hideWebDriver: options.stealthMode.hideWebDriver,
@@ -367,6 +383,7 @@ export class BrowserProcessor {
 
     // Create stealth context
     const { context, contextId } = await this.stealthManager.createStealthContext({
+      engine,
       level: options.stealthMode.level,
       customViewport: {
         width: options.viewportWidth || 1280,
@@ -376,7 +393,8 @@ export class BrowserProcessor {
 
     // Create stealth page
     const page = await this.stealthManager.createStealthPage(contextId);
-    
+    page.__crawlforgeEngine = engine;
+
     // Store context for cleanup
     this.activeContexts.set(contextId, { context, page });
 
@@ -458,7 +476,15 @@ export class BrowserProcessor {
       });
     }
 
-    // Add extra stealth protections
+    // Add extra stealth protections.
+    //
+    // Chromium only. This script hands the page a `window.chrome` object and a
+    // Chrome-shaped navigator; on camoufox's Firefox that is a tell no real
+    // Firefox has, and camoufox already spoofs all of it below the JS layer
+    // where a page cannot see the seam — the same reason StealthBrowserManager
+    // injects nothing into camoufox.
+    if (page.__crawlforgeEngine === 'camoufox') return;
+
     await page.addInitScript(() => {
       // Additional webdriver detection removal
       delete window.navigator.__proto__.webdriver;

package/src/skills/agent-skills/crawlforge-stealth-browsing/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: crawlforge-stealth-browsing
-description: "Bypasses bot detection and geo-restrictions with CrawlForge's stealth_mode and localization tools. Use when a site returns 403 or 429, CAPTCHAs, 'please enable JavaScript', or empty content, or is protected by Cloudflare, DataDome, or PerimeterX, or when the user needs region-specific pricing, geo-blocked content, or a specific locale, timezone, or currency. stealth_mode runs a stealth browser (playwright by default, camoufox for advanced fingerprinting) and can screenshot; localization emulates a country and language. Explains when to escalate from a normal scrape to stealth."
+description: "Bypasses bot detection and geo-restrictions with CrawlForge's stealth_mode and localization tools. Use when a site returns 403 or 429, CAPTCHAs, 'please enable JavaScript', or empty content, or is protected by Cloudflare, DataDome, or PerimeterX, or when the user needs region-specific pricing, geo-blocked content, or a specific locale, timezone, or currency. stealth_mode runs a stealth browser (engine auto by default: camoufox/Firefox for advanced fingerprinting, playwright/Chromium when pinned for speed) and can screenshot; localization emulates a country and language. Explains when to escalate from a normal scrape to stealth, and why a hard block needs a residential proxy."
 metadata:
   version: 5.6.6
   source: crawlforge-mcp-server
@@ -39,8 +39,7 @@ page that navigates to the target URL.
   "tool": "stealth_mode",
   "params": {
     "operation": "create_context",
-    "stealthConfig": { "level": "advanced", "simulateHumanBehavior": true },
-    "engine": "playwright"
+    "stealthConfig": { "level": "advanced", "simulateHumanBehavior": true }
   }
 }
 ```
@@ -58,21 +57,45 @@ Operations: `configure`, `enable`, `disable`, `create_context`, `create_page`,
 `get_stats`, `cleanup`. `stealthConfig.level` is `basic` / `medium` (default) /
 `advanced`. Always run `cleanup` when done to release the browser.
 
-### Engine: playwright vs camoufox
+### Engine: auto, playwright, camoufox
 
-- `engine:"playwright"` (default) — Chromium with stealth patches. Fast, good
-  for most basic bot detection.
+- `engine:"auto"` (default) — Camoufox when its binary is installed, Chromium
+  with a warning when it is not. Leave it alone unless you have a reason.
 - `engine:"camoufox"` — Firefox-based with native anti-detection (no patches).
-  Scores higher against DataDome / Cloudflare / PerimeterX and on CreepJS. Use
-  for heavily protected, financial, or e-commerce sites.
+  The only engine that cleared Cloudflare Turnstile (indeed.com) and Akamai
+  (harrods.com) in the 2026-09-21 benchmark. Pin it to require that engine: it
+  errors rather than falling back. Costs roughly +0.8 s and +400 MB per call
+  against Chromium.
+- `engine:"playwright"` — Chromium with stealth patches. Pin it for speed on
+  sites that need rendering rather than evasion.
+
+`stealth_mode` also accepts `"chromium"` as a synonym for `"playwright"`.
+`browser_session` takes the same `engine` on `operation:"open"` and
+`scrape_with_actions` takes it as `browserOptions.engine` — both only with
+`stealth: true`, and both default to `"auto"`. `scrape`'s `escalate_engine` is
+the one parameter that does not accept `"chromium"`; say `"playwright"` there.
+
+The result always names the engine that ran (`stealth.engine` on `scrape`,
+`engine` elsewhere), and an `auto` run that fell back to Chromium says so in
+`warnings[]` — read that rather than assuming you got Camoufox.
+
+Neither engine defeats DataDome or an interactive Turnstile. If both are
+blocked, the address is usually the problem, not the browser.
 
 Full decision table: [engine selection](references/engine-selection.md).
 
 ### Proxies
 
 A block that survives both engines is usually the IP, not the fingerprint:
-Cloudflare scores the address and its ASN before it serves a challenge. Route
-through your own residential proxy — CrawlForge supplies none.
+Cloudflare scores the address and its ASN before it serves a challenge, so a
+datacenter proxy changes the address without changing the class of address being
+scored. Route through your own residential proxy — CrawlForge supplies none.
+
+Server-wide, set `CRAWLFORGE_STEALTH_PROXIES` (comma-separated proxy URLs): the
+`scrape` escalation stage, `stealth_mode`, `browser_session`,
+`scrape_with_actions` and the `deep_research` retry use it when no proxy is
+passed on the call. (`agent` does not browse yet, so it is not a reader.) Per call, pass
+`stealthConfig.proxyRotation`, which always wins:
 
 ```json
 {
@@ -100,7 +123,8 @@ crawlforge stealth https://protected-site.com --engine camoufox --wait 3000 --sc
 ```
 
 The CLI exposes a one-shot form (`--engine`, `--wait <ms>`, `--screenshot`).
-Force the engine globally with `export CRAWLFORGE_STEALTH_ENGINE=camoufox`.
+`--engine` takes `chromium` (the default here — the CLI has no `auto`) or
+`camoufox`, so name `camoufox` on the command line when you want it.
 
 ## localization (cost: 2)
 

package/src/skills/agent-skills/crawlforge-stealth-browsing/references/engine-selection.md

@@ -1,11 +1,29 @@
 # Stealth Engine Selection
 
-`stealth_mode` supports two browser engines via the `engine` parameter.
-
-## playwright (default)
+`stealth_mode` picks a browser engine with an `engine` parameter;
+`browser_session` takes the same `engine` on `operation:"open"`,
+`scrape_with_actions` takes it as `browserOptions.engine` (both only with
+`stealth: true`), and `scrape` calls the same choice `escalate_engine`. All of
+them default to `auto`.
+
+## auto (default)
+
+- Camoufox when its binary is installed; Chromium otherwise, with the reason in
+  the result's `warnings[]`, so a machine that never ran `npx camoufox fetch`
+  still works. The result names the engine that ran — `stealth.engine` on
+  `scrape`, `engine` on the others — so check it instead of assuming.
+- Naming an engine explicitly turns the fallback off: `camoufox` fails with its
+  install instruction instead of degrading, and `playwright` / `chromium` go
+  straight to Chromium.
+- `chromium` and `playwright` are the same engine. Everything accepts both
+  spellings except `scrape`'s `escalate_engine`, which takes only `playwright`.
+
+## playwright
 
 - Chromium-based with stealth patches applied.
-- Masks `webdriver`, User-Agent, and navigator properties.
+- Reports `navigator.webdriver` as `false` (not deleted — a missing property is
+  itself a marker), and sets the User-Agent, platform, languages and core count
+  through CDP, so a Web Worker answers the same as the page.
 - Lower resource usage, faster startup.
 - Good for most sites with basic bot detection.
 
@@ -13,21 +31,27 @@
 
 - Firefox-based with **native** anti-detection (no runtime patches — uses
   Firefox's genuine properties).
-- Scores higher on CreepJS and against DataDome than patched Chromium.
-- Heavier; install with `npm install camoufox`.
-- Use for advanced fingerprinting: financial, trading, and e-commerce sites.
+- Scores higher on CreepJS and the other detector pages than patched Chromium,
+  and was the only engine to clear Cloudflare Turnstile (indeed.com) and Akamai
+  (harrods.com) in the 2026-09-21 benchmark.
+- Heavier: measured once each on an Apple Silicon Mac, 946 ms and 667 MB to
+  launch a browser, a context and a blank page, against Chromium's 148 ms and
+  253 MB.
+- Install with `npm install camoufox`, then `npx camoufox fetch` for the binary.
+- It does not defeat DataDome or an interactive Turnstile; in the same run
+  Chromium passed leboncoin.fr where Camoufox did not. One run per cell.
 
 ## Decision table
 
 | Scenario | Recommended engine |
 |----------|--------------------|
+| You do not know whether the site blocks | auto |
 | General JS-rendered sites | playwright |
-| Basic bot-detection bypass | playwright |
-| Speed-critical scraping | playwright |
-| Cloudflare-protected sites | camoufox |
-| Sites with DataDome | camoufox |
+| Speed-critical or high-volume scraping | playwright |
+| Cloudflare Turnstile pages, Akamai | camoufox |
+| Detector-score-sensitive work (CreepJS) | camoufox |
 | Sites with PerimeterX | camoufox |
-| Financial / trading sites | camoufox |
+| Sites with DataDome | either — neither cleared it; try a residential proxy |
 
 ## stealthConfig levels
 
@@ -42,22 +66,43 @@
 - `randomizeFingerprint`, `hideWebDriver`, `blockWebRTC`, `spoofTimezone`,
   `randomizeHeaders`, `useRandomUserAgent`, `simulateHumanBehavior`.
 - `customUserAgent`, `customViewport {width,height}`, `locale`, `timezone`.
-- `proxyRotation { enabled, proxies[], rotationInterval }`.
+- `proxyRotation { enabled, proxies[], rotationInterval }` — your own proxies,
+  per call. Server-wide, `CRAWLFORGE_STEALTH_PROXIES` (comma-separated URLs of
+  the same form) covers the paths with no caller to ask: the `scrape` escalation
+  stage, `stealth_mode`, `browser_session`, `scrape_with_actions` and the
+  `deep_research` retry — not `agent`, which has no browsing path yet. A call's
+  own `proxyRotation` always wins. CrawlForge supplies no proxies, and a datacenter
+  proxy does not help — Cloudflare scores the ASN before it serves a challenge.
+  Unrelated to the `PROXY_ROTATION_*` variables, which belong to `localization`.
 - `antiDetection { cloudflareBypass, recaptchaHandling, hideAutomation,
   spoofMediaDevices, spoofBatteryAPI }`.
 - `fingerprinting { canvasNoise, webglSpoofing, audioContextSpoofing,
   fontSpoofing, hardwareSpoofing }`.
 
-## Global override
+On `camoufox`, `customUserAgent` and `customViewport` are not applied: that engine
+brings its own Firefox identity, and overriding it from here put a Chrome identity
+on a Gecko engine. `locale` is applied, but at browser launch rather than per
+call — Camoufox sets language, Accept-Language and `Intl` below the JS layer,
+where a Worker matches the document — so the first locale a Camoufox browser is
+launched with holds until the browser is cleaned up, and behind a proxy the exit
+IP decides it instead. The persona's OS is the host's and is not configurable
+(on Camoufox that is a request its current client does not honour).
+
+## Environment
+
+There is no global engine override — the engine is a per-call parameter. Two
+things come close, and neither is global:
 
 ```bash
-export CRAWLFORGE_STEALTH_ENGINE=camoufox
+export RESEARCH_STEALTH_ENGINE=camoufox     # deep_research's blocked-source retry only
+crawlforge stealth <url> --engine camoufox  # the CLI, which defaults to chromium and has no auto
 ```
 
-Forces the engine for all stealth calls regardless of the `engine` parameter.
+The one server-level stealth variable is `CRAWLFORGE_STEALTH_PROXIES` (above).
 
 ## Sandboxing note
 
-Stealth Chromium runs with `--no-sandbox` + `--disable-web-security` (a
-deliberate fingerprint-spoofing trade-off). Camoufox (Firefox) is the
-alternative when that trade-off is unacceptable.
+Stealth Chromium runs with `--no-sandbox` (a deliberate fingerprint-spoofing
+trade-off). It no longer runs with `--disable-web-security`, which a page could
+read in one line; the non-stealth render browser still does. Camoufox (Firefox)
+is the alternative when the `--no-sandbox` trade-off is unacceptable.

package/src/skills/agent-skills/crawlforge-web-scraping/SKILL.md

@@ -74,8 +74,7 @@ shell). The escalated page goes through the same formats, so `markdown`,
   "params": {
     "url": "https://www.producthunt.com/",
     "formats": ["markdown"],
-    "escalate": true,
-    "escalate_engine": "playwright"
+    "escalate": true
   }
 }
 ```
@@ -83,10 +82,17 @@ shell). The escalated page goes through the same formats, so `markdown`,
 The result adds `escalated: true|false`, and `stealth: { engine,
 vendor_detected }` when the browser ran. The projection is `2 + 5`; the charge
 falls back to the base when the plain fetch succeeded and nothing escalated.
-`escalate_engine` is `"playwright"` (default) or `"camoufox"`. robots.txt is
-respected on the escalated path too, and a second call to a host that blocked
-within the last 24 hours skips the doomed plain fetch and says so in
-`warnings[]`.
+robots.txt is respected on the escalated path too, and a second call to a host
+that blocked within the last 24 hours skips the doomed plain fetch and says so
+in `warnings[]`.
+
+`escalate_engine` defaults to `"auto"`: Camoufox (Firefox, the engine that gets
+through Cloudflare Turnstile and Akamai) when its binary is installed, Chromium
+with a warning when it is not. Pin `"playwright"` (the only spelling for
+Chromium here — `"chromium"` is rejected) when the page just needs rendering and
+you want the faster, lighter browser; pin `"camoufox"` to require the stronger
+engine — it errors rather than falling back. `stealth.engine` in the result says
+which one ran, and a fallback is reported in `warnings[]`.
 
 Query-scoped formats return only the parts of the page that match, verbatim,
 with offsets into the `markdown` of the same call:

package/src/tools/advanced/BrowserSessionTool.js

@@ -41,6 +41,7 @@ import { internalOwnerToken, isInternalRequest } from '../../server/requestConte
 import { isRemoteTransport } from '../../utils/remoteMode.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
 import { stealthDocumentVerdict } from '../../utils/stealthVerdict.js';
+import { resolveStealthEngine } from '../../core/StealthBrowserManager.js';
 
 const SECOND = 1000;
 
@@ -95,6 +96,11 @@ const BrowserSessionSchema = z.object({
   // from their docs reads the same numbers; the store works in milliseconds.
   url: z.string().url().optional(),
   stealth: z.boolean().default(false),
+  // Which stealth engine the session runs on. Only meaningful with stealth:true
+  // — the standard pool is Chromium and camoufox exists only on the stealth
+  // path — so "camoufox" without stealth is refused rather than quietly
+  // downgraded (see openSession).
+  engine: z.enum(['auto', 'chromium', 'camoufox', 'playwright']).default('auto'),
   ttl: z.number().min(TTL_MIN_MS / SECOND).max(TTL_MAX_MS / SECOND).optional(),
   activity_ttl: z.number().min(ACTIVITY_TTL_MIN_MS / SECOND).max(ACTIVITY_TTL_MAX_MS / SECOND).optional(),
   viewport: z.object({
@@ -143,6 +149,11 @@ function sessionInfo(session) {
     sessionId: session.id,
     url: session.url,
     stealth: session.stealth,
+    // Which engine actually ran, read off the page BrowserProcessor stamped —
+    // not the engine that was asked for. With engine:"auto" those differ
+    // whenever camoufox is not installed, and the whole point of a resolved
+    // engine is that the caller can see which one it got.
+    ...(session.page?.__crawlforgeEngine ? { engine: session.page.__crawlforgeEngine } : {}),
     expiresAt: session.createdAt + session.ttlMs,
     idleExpiresAt: session.lastUsedAt + session.activityTtlMs
   };
@@ -181,7 +192,11 @@ function withJsResult(result) {
  * warnings describe the hop the caller just made, not every hop of the session.
  */
 function gateWarningFields(page) {
-  const warnings = page?.__crawlforgeGateWarnings;
+  return warningFields(page?.__crawlforgeGateWarnings);
+}
+
+/** The same `warnings` field, for a list that is not only the gate's. */
+function warningFields(warnings) {
   return warnings?.length ? { warnings } : {};
 }
 
@@ -288,6 +303,19 @@ export class BrowserSessionTool {
       throw new Error('operation "open" requires a url to load the session on.');
     }
 
+    // camoufox is the stealth browser's Firefox engine; the plain session runs
+    // on the shared Chromium pool, where there is nothing to switch. Refusing
+    // beats opening a session on the engine the caller did not ask for and
+    // saying so in a warning nobody reads. "auto"/"chromium"/"playwright" need
+    // no refusal: Chromium is what they would have got.
+    if (!params.stealth && params.engine === 'camoufox') {
+      throw refuse(
+        'ENGINE_NEEDS_STEALTH',
+        'engine:"camoufox" requires stealth:true — the Firefox anti-detect engine exists only on ' +
+        'the stealth path. Open with stealth:true, or drop engine to use the standard Chromium browser.'
+      );
+    }
+
     const browserOptions = {
       headless: true,
       viewportWidth: params.viewport?.width,
@@ -296,8 +324,13 @@ export class BrowserSessionTool {
       respectRobots: params.respect_robots,
       tool: 'browser_session'
     };
+    // Resolved here rather than downstream so an "auto" that lands on Chromium
+    // because camoufox is absent can be said out loud in this call's result.
+    let engineWarning = null;
     if (params.stealth) {
-      browserOptions.stealthMode = { enabled: true };
+      const resolved = await resolveStealthEngine(params.engine);
+      engineWarning = resolved.fallbackWarning;
+      browserOptions.stealthMode = { enabled: true, engine: resolved.engine };
     }
 
     // initializePage runs the SSRF guard, then the blocklist/robots gate, and
@@ -341,7 +374,9 @@ export class BrowserSessionTool {
       operation: 'open',
       ...sessionInfo(session),
       ...verdictFields(verdict),
-      ...gateWarningFields(page),
+      // One warnings array: the engine fallback rides with the gate's, or it
+      // would overwrite them (both spread a `warnings` key).
+      ...warningFields([...(engineWarning ? [engineWarning] : []), ...(page?.__crawlforgeGateWarnings || [])]),
       // The page is a wall, but the session behind it is real and holds a
       // browser context — say so, or a caller reading only `success` abandons
       // it to its TTL instead of closing it or acting through the challenge.

package/src/tools/advanced/ScrapeWithActionsTool.js

@@ -14,6 +14,7 @@ import { stealthDocumentVerdict } from '../../utils/stealthVerdict.js';
 import { elementText } from '../../utils/elementText.js';
 import { pageTitle } from '../../utils/pageTitle.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
+import { resolveStealthEngine } from '../../core/StealthBrowserManager.js';
 
 // Recording / replay helpers
 import {
@@ -192,7 +193,12 @@ const ScrapeWithActionsSchema = z.object({
     // Run the chain in the stealth browser (StealthBrowserManager) instead of
     // the standard pool. executeSession turns this into the stealthMode object
     // ActionExecutor/BrowserProcessor read.
-    stealth: z.boolean().default(false)
+    stealth: z.boolean().default(false),
+    // Which stealth engine the chain runs on. Only meaningful with stealth:true
+    // — the standard pool is Chromium and camoufox exists only on the stealth
+    // path — so "camoufox" without stealth is refused rather than quietly
+    // downgraded (see executeSession).
+    engine: z.enum(['auto', 'chromium', 'camoufox', 'playwright']).default('auto')
   }).optional(),
 
   // Content extraction options
@@ -394,8 +400,26 @@ export class ScrapeWithActionsTool extends EventEmitter {
 
     // `stealth: true` is the caller-facing switch; everything downstream reads
     // browserOptions.stealthMode.
+    //
+    // The engine is resolved here, not downstream, so an "auto" that lands on
+    // Chromium because camoufox is absent can be reported in this call's
+    // result instead of disappearing.
+    const warnings = [];
     if (browserOptions.stealth) {
-      browserOptions.stealthMode = { enabled: true };
+      const resolved = await resolveStealthEngine(browserOptions.engine);
+      browserOptions.stealthMode = { enabled: true, engine: resolved.engine };
+      if (resolved.fallbackWarning) warnings.push(resolved.fallbackWarning);
+    } else if (browserOptions.engine === 'camoufox') {
+      // camoufox is the stealth browser's Firefox engine; a non-stealth chain
+      // runs on the shared Chromium pool, where there is nothing to switch.
+      // Refusing beats running the chain on the engine the caller did not ask
+      // for. "auto"/"chromium"/"playwright" need no refusal: Chromium is what
+      // they would have got.
+      throw new Error(
+        'browserOptions.engine:"camoufox" requires browserOptions.stealth:true — the Firefox ' +
+        'anti-detect engine exists only on the stealth path. Set stealth:true, or drop engine ' +
+        'to use the standard Chromium browser.'
+      );
     }
 
     // The robots override rides with the browser options so ActionExecutor's
@@ -497,6 +521,10 @@ export class ScrapeWithActionsTool extends EventEmitter {
       error: chainResult.error || verdict.error || undefined,
       ...(verdict.blocked ? { blocked: verdict.blocked } : {}),
       ...(verdict.status !== null ? { httpStatus: verdict.status } : {}),
+      // The engine the chain actually ran on, and anything to say about how it
+      // was chosen (an "auto" that fell back to Chromium says so here).
+      engine: browserOptions.stealthMode?.engine || 'chromium',
+      ...(warnings.length ? { warnings } : {}),
 
       actionResults,
       totalActions: params.actions.length,

package/src/tools/scrape/escalation.js

@@ -21,7 +21,12 @@ export const SCRAPE_ESCALATION_CREDITS = 5;
 /** The two public fields, in tools/list order, with the text the client sees. */
 export const SCRAPE_ESCALATION_SHAPE = {
   escalate: z.boolean().optional().default(false).describe('When the plain fetch comes back blocked (403/429/challenge page/empty shell), retry once in the stealth browser and return its content instead of the block. Projected at 2+5; the actual charge stays at the base price when the plain fetch succeeded. Default: false'),
-  escalate_engine: z.enum(['playwright', 'camoufox']).optional().default('playwright').describe('Stealth engine for the escalated retry (default: "playwright")')
+  // "auto" is the default because the engine that gets past the walls this
+  // stage exists for is camoufox, and asking for it by name used to be the
+  // caller's problem. It resolves to camoufox when the package is installed
+  // and to playwright (Chromium) when it is not, saying so in `warnings`.
+  // Naming an engine still pins it exactly.
+  escalate_engine: z.enum(['auto', 'playwright', 'camoufox']).optional().default('auto').describe('Stealth engine for the escalated retry: "auto" (default — Camoufox when it is installed, Chromium otherwise, reported in warnings), "playwright" (Chromium) or "camoufox"')
 };
 
 /**

package/src/tools/scrape/unifiedScrape.js

@@ -15,7 +15,8 @@
 
 import { z } from 'zod';
 import { load } from 'cheerio';
-import { documentVerdict, segmentUnits, rankUnits } from 'crawlforge-extractors';
+import { segmentUnits, rankUnits } from 'crawlforge-extractors';
+import { stealthDocumentVerdict as documentVerdict } from '../../utils/stealthVerdict.js';
 import {
   SCRAPE_STRING_FORMATS, JsonFormatSchema, HighlightsFormatSchema, QuestionFormatSchema, FormatSchema,
   scrapeFormatSurcharge

package/src/utils/challengeDetection.js

@@ -1,10 +1,93 @@
 /**
- * challengeDetection.js — re-export of the challenge-vendor tables.
+ * challengeDetection.js — the challenge-vendor tables, refined for widgets.
  *
  * The tables moved to crawlforge-extractors 1.7.0 so the REST `scrape` route
  * reaches the same verdict as the MCP server (Phase 0, 0.1). Callers keep
  * importing from here.
+ *
+ * Upstream treats a vendor marker on a page under 4000 visible characters as
+ * definitive, which reports a block on a real page that merely embeds a
+ * Turnstile widget: quora.com rendered its real 397-character login page on
+ * both engines and the verdict layer still called it blocked (2026-09-21
+ * benchmark, finding 6). A widget embed and an interstitial are separable by
+ * what else is in the document, not by length — the interstitial carries
+ * Cloudflare's challenge bootstrap or the prose it shows while it holds the
+ * request, and a page that merely embeds the widget carries neither. So the
+ * refinement below suppresses that one case and leaves every other path
+ * upstream owns (a challenge title, Amazon's validateCaptcha form, DataDome,
+ * PerimeterX, Akamai) exactly as it is. Worth upstreaming.
+ */
+
+import { detectChallengePage as detectUpstream } from 'crawlforge-extractors';
+
+// The bootstrap a real interstitial ships and a page embedding a widget never
+// does: Cloudflare's challenge options blob and its cf-chl- element ids,
+// Vercel's checkpoint script and token. Two Cloudflare tokens are deliberately
+// NOT here, because a real page legitimately carries both:
+//   - challenges.cloudflare.com, the widget's own script host;
+//   - cf-chl-widget-*, the hidden response input Turnstile names itself.
+// quora.com's login page carries exactly one cf-chl- token and it is
+// "cf-chl-widget-gaztz_response" (measured 2026-09-21), so a bare cf-chl- test
+// matches every widget embed there is. Hence the lookahead — do not put
+// cf-chl- back.
+const INTERSTITIAL_MARKERS = [
+  { vendor: 'cloudflare', markers: /_cf_chl_opt|window\._cf_chl|cf-chl-(?!widget)|cf_chl_rc_/i, evidence: 'the Cloudflare challenge bootstrap' },
+  { vendor: 'vercel', markers: /_vercel\/challenge|vercel\.link\/security-checkpoint|x-vercel-challenge-token/i, evidence: 'the Vercel Security Checkpoint bootstrap' }
+];
+
+// What an interstitial says to the reader while it holds the request.
+const INTERSTITIAL_PROSE = /verifying you are human|checking your browser|needs to review the security of your connection|enable javascript and cookies to continue/i;
+
+// The floor for "this document has a real body". quora.com's real login page
+// is 397 visible characters, so the floor has to sit below that; Cloudflare's
+// own interstitial renders well under 200 characters of prose ("Verifying you
+// are human…", "Enable JavaScript and cookies to continue"), so a genuinely
+// empty wall carrying only a widget marker is still short enough to be caught.
+const REAL_BODY_MIN_CHARS = 200;
+
+/**
+ * Whether a document is a challenge wall rather than the page: a known
+ * challenge title OR a challenge bootstrap marker, regardless of the title and
+ * regardless of page length, so a custom-titled interstitial (nowsecure.nl,
+ * whose title is just "nowsecure.nl") is recognised too.
+ *
+ * @param {{ title?: string, html?: string, text?: string }} page
+ * @returns {{ vendor: string, evidence: string } | null}
+ */
+export function looksLikeInterstitial(page = {}) {
+  const { title = '', html = '' } = page;
+  // A title on its own reaches only upstream's title branch — its marker
+  // branch needs html and its definitive branch a form — so the challenge
+  // titles stay in one place rather than being copied here.
+  const byTitle = detectUpstream({ title });
+  if (byTitle) return byTitle;
+  for (const { vendor, markers, evidence } of INTERSTITIAL_MARKERS) {
+    if (markers.test(html)) return { vendor, evidence };
+  }
+  return null;
+}
+
+/** A title, a body a reader could read, and none of the interstitial's prose. */
+function looksLikeRealPage({ title = '', text = '' }) {
+  if (!String(title).trim()) return false;
+  const visible = String(text).replace(/\s+/g, ' ').trim();
+  if (visible.length < REAL_BODY_MIN_CHARS) return false;
+  return !INTERSTITIAL_PROSE.test(`${title} ${visible}`);
+}
+
+/**
+ * @param {{ title?: string, html?: string, text?: string }} page
+ * @returns {{ vendor: string, evidence: string } | null}
  */
+export function detectChallengePage(page = {}) {
+  const hit = detectUpstream(page);
+  // Cloudflare is the only vendor whose markers a real page can carry, so it is
+  // the only one refined. Reaching here with a cloudflare hit that is not an
+  // interstitial means the evidence was the widget's own script host or
+  // response input on a short page.
+  if (!hit || hit.vendor !== 'cloudflare') return hit;
+  if (looksLikeInterstitial(page)) return hit;
+  return looksLikeRealPage(page) ? null : hit;
+}
 
-export { detectChallengePage } from 'crawlforge-extractors';
-export { detectChallengePage as default } from 'crawlforge-extractors';
+export default detectChallengePage;

package/src/utils/stealthVerdict.js

@@ -1,10 +1,44 @@
 /**
- * stealthVerdict.js — re-export of the document verdict.
+ * stealthVerdict.js — the document verdict, with the widget refinement.
  *
  * `stealthDocumentVerdict` moved to crawlforge-extractors 1.7.0 as
  * `documentVerdict` — the same behaviour and messages for the stealth callers,
  * plus a `fetcher` option so the plain `scrape` path on both surfaces reaches
  * the same verdict (Phase 0, 0.1). Callers keep importing from here.
+ *
+ * Upstream decides the block before every other rule it owns, so the widget
+ * refinement in challengeDetection.js would never reach the verdict the tools
+ * and the benchmark actually print. The wrapper below re-runs upstream on the
+ * same document with the widget marker neutralised whenever the refinement
+ * clears the block, so an HTTP error page, an empty shell or a short
+ * error-titled placeholder that happens to embed a widget is still named by
+ * upstream and still decides (Phase 1, finding 6).
  */
 
-export { documentVerdict as stealthDocumentVerdict, SOFT_ERROR_MAX_CHARS } from 'crawlforge-extractors';
+import { documentVerdict, SOFT_ERROR_MAX_CHARS } from 'crawlforge-extractors';
+import { detectChallengePage } from './challengeDetection.js';
+
+export { SOFT_ERROR_MAX_CHARS };
+
+// The two Turnstile markers challengeDetection.js clears: the widget's script
+// host, and the hidden response input the widget names itself
+// (id="cf-chl-widget-<id>_response" — quora.com, measured 2026-09-21).
+// Upstream's Cloudflare pattern matches a bare cf-chl-, so neutralising the
+// host alone left the re-run blocking the same document a second time. Both
+// are swapped for inert text; the bootstrap markers are never touched, so a
+// document that carries one is still upstream's block to make.
+const TURNSTILE_WIDGET = /challenges\.cloudflare\.com|cf-chl-widget/gi;
+
+/**
+ * @param {{ url?: string, title?: string, text?: string, html?: string, status?: number|null }} scraped
+ * @param {{ waitedMs?: number, allowEmpty?: boolean, fetcher?: string, rendered?: boolean, contentReturned?: boolean }} [options]
+ * @returns {{ success: boolean, status: number|null, error?: string, blocked?: { vendor: string, evidence: string } }}
+ */
+export function stealthDocumentVerdict(scraped, options) {
+  const verdict = documentVerdict(scraped, options);
+  if (!verdict.blocked || detectChallengePage(scraped || {})) return verdict;
+  // The neutralised copy is read by upstream and discarded; the caller's own
+  // document is never rewritten.
+  const html = String(scraped?.html || '').replace(TURNSTILE_WIDGET, 'turnstile-widget');
+  return documentVerdict({ ...scraped, html }, options);
+}