crawlforge-mcp-server 6.10.0 → 6.12.0

package/CLAUDE.md

@@ -236,6 +236,11 @@ CRAWLFORGE_STEALTH_PROXIES=          # comma-separated proxy URLs used by the es
                                      # from the PROXY_ROTATION_* family, which belongs to
                                      # localization. The agent's automatic stealth retry
                                      # (stealth review Phase 3) reads it too.
+CRAWLFORGE_IMPIT=                    # `off` skips escalation's impit step (a Chrome TLS handshake with
+                                     # the honest UA, tried before the browser; stealth review Phase 7).
+                                     # Exit-IP dependent like the engine: it cleared indeed.com from a
+                                     # residential IP and nothing from the hosted datacenter IP, so
+                                     # render.yaml sets it off there. Measure with scripts/impit-probe.mjs.
 CRAWLFORGE_CLEARANCE_JAR=            # `off` disables the clearance jar (stealth review Phase 4):
                                      # cf_clearance/__cf_bm/datadome cookies a stealth render earned
                                      # are replayed to the next stealth context with the same engine,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "6.10.0",
+  "version": "6.12.0",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 31 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -121,7 +121,7 @@
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",
-    "crawlforge-extractors": "^1.9.0",
+    "crawlforge-extractors": "^1.10.0",
     "diff": "^9.0.0",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",
@@ -139,7 +139,8 @@
     "zod": "^4.5.4"
   },
   "optionalDependencies": {
-    "camoufox": "^0.1.19"
+    "camoufox": "^0.1.19",
+    "impit": "^0.14.5"
   },
   "devDependencies": {
     "@jest/globals": "^30.3.0",

package/server.js

@@ -59,7 +59,9 @@ import { READ_RESULT_INPUT_SHAPE, readResultHandler } from "./src/tools/result/r
 import { MAX_INLINE_CHARS_PARAM } from "./src/server/inlineThreshold.js"; // Phase 2
 import { REDACT_PII_PARAM } from "./src/server/redaction.js"; // Phase 5 (5.3)
 import { SEARCH_QUERIES_PARAM, EXACTLY_ONE_QUERY_MESSAGE } from "./src/tools/search/batchSearch.js"; // Phase 5 (5.1)
-import { markPreflightRefusal } from "./src/server/requestContext.js";
+import { markPreflightRefusal, internalOwnerToken } from "./src/server/requestContext.js";
+import { recordStealthEscalation } from "./src/utils/complianceAudit.js";
+import { loadImpit, impitEnabled, impitFetchPage, IMPIT_ENGINE } from "./src/utils/impitRung.js"; // stealth review Phase 7
 // D1.1 Resources + D1.2 Prompts + D1.4 Elicitation
 import { ResourceRegistry, MAX_RESOURCE_BLOB_BYTES } from "./src/resources/ResourceRegistry.js";
 import { PROMPTS, getPromptMessages } from "./src/prompts/PromptRegistry.js";
@@ -108,7 +110,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "6.10.0",
+  version: "6.12.0",
   description: "Production-ready MCP server with 31 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, stateful browser sessions with element refs, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -248,17 +250,30 @@ const scrapeTemplateTool = new ScrapeTemplateTool(); // D3.3
 // go through the same gate, engine resolver and server-level proxy list.
 // Injected so the tool modules never import StealthBrowserManager (that would pull a
 // browser dependency into every unit test that loads `scrape`). Same gate
-// and same browser the stealth_mode tool drives — no new evasion, and the
+// and same browser the stealth_mode tool drives, plus the impit TLS try, and the
 // engine name is resolved here, beside its sibling, by the one resolver
 // every stealth entry point shares.
-const stealthEscalation = async ({ url, engine, respectRobots }) => {
+const stealthEscalation = async ({ url, engine, respectRobots, tool }) => {
   const warnings = await stealthComplianceGate(url, respectRobots);
+  const auditIdentity = { apiKey: AuthManager.getConfig()?.apiKey, ownerToken: internalOwnerToken() };
+  // Stealth review Phase 7: under "auto", a Chrome TLS handshake with the
+  // honest User-Agent is tried before any browser launches. A caller who
+  // named an engine asked for that browser, so it is not tried for them.
+  // CRAWLFORGE_IMPIT=off skips it for a deployment whose exit IP it cannot help.
+  if (engine === 'auto' && impitEnabled() && await loadImpit()) {
+    recordStealthEscalation({ url, tool, engine: IMPIT_ENGINE, ...auditIdentity });
+    const page = await impitFetchPage(url);
+    if (page) return { ...page, warnings };
+  }
   // "auto" prefers camoufox and falls back to chromium when its binary is
   // missing. Mapping here instead ("camoufox" or else chromium) collapsed
   // auto to chromium and hid the fallback; the resolver says which engine it
   // landed on and why, and the caller is told.
   const resolved = await resolveStealthEngine(engine);
   if (resolved.fallbackWarning) warnings.push(resolved.fallbackWarning);
+  // Audit row: the request is about to go out under a browser identity. After
+  // the gate (a refusal presents none) and the resolver (the engine that runs).
+  recordStealthEscalation({ url, tool, engine: resolved.engine, ...auditIdentity });
   const scraped = await stealthBrowserManager.scrapeWithStealth({
     url,
     engine: resolved.engine
@@ -269,9 +284,9 @@ const stealthEscalation = async ({ url, engine, respectRobots }) => {
 };
 const unifiedScrapeTool = new UnifiedScrapeTool({
   actionExecutor: scrapeWithActionsTool.actionExecutor, // D4 D1 (+v4.8 screenshot reuses the shared browser pool)
-  escalateScrape: stealthEscalation
+  escalateScrape: (args) => stealthEscalation({ ...args, tool: 'scrape' })
 });
-const agentTool = new AgentTool({ escalateFetch: stealthEscalation }); // D4 D2 + stealth review Phase 3
+const agentTool = new AgentTool({ escalateFetch: (args) => stealthEscalation({ ...args, tool: 'agent' }) }); // D4 D2 + stealth review Phase 3
 const stealthBrowserManager = new StealthBrowserManager();
 const localizationManager = new LocalizationManager();
 
@@ -1419,6 +1434,11 @@ registerToolIfEnabled("stealth_mode", {
         const resolvedEngine = await resolveStealthEngine(engine);
         if (resolvedEngine.fallbackWarning) warnings.push(resolvedEngine.fallbackWarning);
 
+        recordStealthEscalation({
+          url, tool: 'stealth_mode', engine: resolvedEngine.engine,
+          apiKey: AuthManager.getConfig()?.apiKey, ownerToken: internalOwnerToken()
+        });
+
         const wantsScreenshot = formats.includes('screenshot');
         const scraped = await stealthBrowserManager.scrapeWithStealth({
           url,
@@ -1507,6 +1527,11 @@ registerToolIfEnabled("stealth_mode", {
         let navigation = null;
         try {
           if (urlToTest) {
+            recordStealthEscalation({
+              url: urlToTest, tool: 'stealth_mode',
+              engine: stealthBrowserManager.contexts.get(contextId)?.config?.engine ?? null,
+              apiKey: AuthManager.getConfig()?.apiKey, ownerToken: internalOwnerToken()
+            });
             // page.goto returns a Playwright Response handle, which is not
             // JSON-serializable — extract just the useful navigation details.
             // Explicit timeout keeps navigation inside every caller's window
@@ -1520,6 +1545,7 @@ registerToolIfEnabled("stealth_mode", {
               title: await page.title().catch(() => null)
             };
             const challenge = detectChallengePage({
+              status: navigation.status,
               title: navigation.title || '',
               html: await page.content().catch(() => ''),
               text: await page.innerText('body').catch(() => '')

package/src/core/StealthBrowserManager.js

@@ -26,6 +26,47 @@ import { ClearanceJar, sharedClearanceJar } from './ClearanceJar.js';
 // Grace given to a document that rendered no title and no text (see _waitOutEmptyDocument).
 export const EMPTY_DOCUMENT_GRACE_MS = 8000;
 
+// Where the Turnstile checkbox sits inside its frame: 28 px in from the left
+// edge (16 px padding + half the 24 px box) of the 300×65 normal-size widget,
+// vertically centred. A fixed offset into the frame's box, because the
+// checkbox itself lives in a closed shadow root inside a cross-origin frame
+// that vanilla Playwright cannot select into — but the frame element's box it
+// can read, from page.frames(), whatever shadow root the frame is mounted in.
+export const TURNSTILE_CHECKBOX_OFFSET_X = 28;
+const TURNSTILE_FRAME_URL = /^https:\/\/challenges\.cloudflare\.com\//;
+
+/**
+ * The first Turnstile frame box whose checkbox point is on screen. A page can
+ * mount a widget off screen (nowsecure.nl mounts one at y=-147), and a click
+ * there lands on nothing.
+ * @param {Array<{x:number,y:number,width:number,height:number}|null>} boxes
+ * @param {{width:number,height:number}|null} viewport
+ * @returns {{x:number,y:number,width:number,height:number}|null}
+ */
+export function pickTurnstileBox(boxes, viewport) {
+  for (const box of boxes) {
+    if (!box || !(box.width > 0) || !(box.height > 0)) continue;
+    const { x, y } = turnstileClickPoint(box);
+    if (x < 0 || y < 0) continue;
+    if (viewport && (x > viewport.width || y > viewport.height)) continue;
+    return box;
+  }
+  return null;
+}
+
+/**
+ * The checkbox point for a frame box, clamped inside the box so a compact
+ * widget narrower than the offset is still clicked inside itself.
+ * @param {{x:number,y:number,width:number,height:number}} box
+ * @returns {{x:number,y:number}}
+ */
+export function turnstileClickPoint(box) {
+  return {
+    x: box.x + Math.min(TURNSTILE_CHECKBOX_OFFSET_X, box.width / 2),
+    y: box.y + box.height / 2
+  };
+}
+
 // The Chrome major claimed when the installed binary cannot be read at all.
 // Only ever a floor: installedChromeVersion() prefers playwright-core's own
 // browsers.json, and a launched browser's real version overrides both.
@@ -2740,6 +2781,69 @@ export class StealthBrowserManager {
     await page.waitForLoadState('domcontentloaded', { timeout: 5000 }).catch(() => {});
   }
 
+  /**
+   * Click the Turnstile checkbox of a challenge that did not solve itself
+   * (review Phase 5). A fixed offset into the challenges.cloudflare.com
+   * frame's box, through page.mouse — plain Playwright, no shadow-root access.
+   * Then wait for either the widget's token or the title to change, and give a
+   * wall that is now navigating on its usual wait-out.
+   *
+   * Verified against Cloudflare's forced-interactive test sitekey on a local
+   * page, which proves the mechanism only: whether a real site's challenge
+   * accepts the click still depends on the IP and fingerprint it scores.
+   *
+   * @returns {Promise<boolean>} whether a click was made
+   */
+  async _clickTurnstile(page, { timeoutMs = 10000 } = {}) {
+    const frames = page.frames().filter((frame) => TURNSTILE_FRAME_URL.test(frame.url()));
+    if (frames.length === 0) return false;
+    const boxes = [];
+    for (const frame of frames) {
+      const element = await frame.frameElement().catch(() => null);
+      boxes.push(element ? await element.boundingBox().catch(() => null) : null);
+    }
+    const box = pickTurnstileBox(boxes, page.viewportSize());
+    if (!box) return false;
+    const { x, y } = turnstileClickPoint(box);
+    const title = await page.title().catch(() => '');
+    await page.mouse.move(x - 40, y - 12);
+    await page.mouse.move(x, y, { steps: 8 });
+    await page.mouse.click(x, y, { delay: 90 });
+    await page
+      .waitForFunction(
+        (t) => document.title !== t ||
+          [...document.querySelectorAll('input[name="cf-turnstile-response"]')].some((input) => input.value),
+        title,
+        { timeout: timeoutMs }
+      )
+      .catch(() => {});
+    await this._waitOutChallenge(page);
+    return true;
+  }
+
+  /**
+   * The click, only where it belongs: on Chromium, and only on a document the
+   * verdict still calls a Cloudflare wall. A real page that embeds a widget
+   * (quora.com's login, nowsecure.nl at 200) is never clicked.
+   * @returns {Promise<boolean>} whether a click was made
+   */
+  async _clickThroughChallenge(page, { status = null, engine = null } = {}) {
+    if (engine !== 'chromium') return false;
+    if (!page.frames().some((frame) => TURNSTILE_FRAME_URL.test(frame.url()))) return false;
+    let title; let html; let text;
+    try {
+      [title, html, text] = await Promise.all([
+        page.title(),
+        page.content(),
+        page.evaluate(() => (document.body ? document.body.innerText : ''))
+      ]);
+    } catch {
+      return false;
+    }
+    if (detectChallengePage({ title, html, text, status })?.vendor !== 'cloudflare') return false;
+    return this._clickTurnstile(page);
+  }
+
   async scrapeWithStealth({ url, engine, wait_for = 0, screenshot = false, stealthConfig = {} } = {}) {
     if (!url) throw new Error('scrapeWithStealth requires a url');
 
@@ -2775,6 +2879,8 @@ export class StealthBrowserManager {
       // up, so the interstitial came back as success:true (R17, 2026-09-04).
       // An auto-solving challenge gets one bounded wait to finish first.
       await this._waitOutChallenge(page);
+      // A Turnstile challenge still up after that wait needs its checkbox clicked.
+      await this._clickThroughChallenge(page, { status, engine: engineUsed });
       const emptyGraceMs = await this._waitOutEmptyDocument(page);
       // Last: let whatever is still rendering finish. Without this the read
       // below can land between "the page has content" and "the page has the
@@ -2811,7 +2917,7 @@ export class StealthBrowserManager {
       // Still a wall after everything above: whatever clearance this context
       // carried for the site did not work, so closeContext drops it instead of
       // keeping it for the next call.
-      if (detectChallengePage({ title, html, text })) {
+      if (detectChallengePage({ title, html, text, status })) {
         this._markBlocked(contextId, [url, page.url()]);
       }
 

package/src/schemas/toolOutputSchemas.js

@@ -138,7 +138,7 @@ const scrapeShape = {
   stealth: z.object({
     engine: z.string().optional(),
     vendor_detected: z.string().nullable().optional()
-  }).passthrough().optional().describe('Present when escalated is true: the stealth engine that ran, and the bot-defence vendor the plain fetch hit (null when the block named none)'),
+  }).passthrough().optional().describe('Present when escalated is true: the stealth engine that ran ("impit" when the Chrome TLS handshake got the page without a browser), and the bot-defence vendor the plain fetch hit (null when the block named none)'),
   content: z.object(scrapeFormatShapes).passthrough().optional().describe('One key per requested format'),
   warnings: z.array(z.string()).optional().describe('Per-format warnings; partial success never fails the whole call'),
   redaction: redactionShape,

package/src/skills/agent-skills/crawlforge-web-scraping/SKILL.md

@@ -80,11 +80,17 @@ shell). The escalated page goes through the same formats, so `markdown`,
 ```
 
 The result adds `escalated: true|false`, and `stealth: { engine,
-vendor_detected }` when the browser ran. The projection is `2 + 5`; the charge
+vendor_detected }` when the escalation ran. `engine` is `impit` when a Chrome
+TLS handshake with the honest User-Agent got the page before any browser
+launched (default `escalate_engine: "auto"` only). The projection is `2 + 5`,
+whichever step got the page; the charge
 falls back to the base when the plain fetch succeeded and nothing escalated.
 robots.txt is respected on the escalated path too, and a second call to a host
 that blocked within the last 24 hours skips the doomed plain fetch and says so
-in `warnings[]`.
+in `warnings[]`. That call is charged the full `2 + 5` once the browser runs,
+because no plain fetch ran that could have lowered it. The memory is kept for 24
+hours from the block, in the running server only, and a `scrape` without
+`escalate` that gets a clean page from that host clears it sooner.
 
 `escalate_engine` defaults to `"auto"`: Camoufox (Firefox, the engine that gets
 through Cloudflare Turnstile and Akamai) when its binary is installed, Chromium

package/src/tools/scrape/escalation.js

@@ -5,7 +5,9 @@
  * plain fetch comes back walled (Phase 3). The plain fetch always runs first
  * (G1: never start with stealth), so the escalation is a second stage, not a
  * different tool: it reuses the same compliance gate and the same
- * StealthBrowserManager `stealth_mode` uses, and adds no new evasion.
+ * StealthBrowserManager `stealth_mode` uses. Its one addition (stealth review
+ * Phase 7) is a Chrome TLS handshake tried before the browser under "auto";
+ * see src/utils/impitRung.js.
  *
  * The two fields and the price rule live here so `server.js`, the tool
  * module, `AuthManager.getToolCost` and the output schema read one
@@ -20,7 +22,7 @@ export const SCRAPE_ESCALATION_CREDITS = 5;
 
 /** The two public fields, in tools/list order, with the text the client sees. */
 export const SCRAPE_ESCALATION_SHAPE = {
-  escalate: z.boolean().optional().default(false).describe('When the plain fetch comes back blocked (403/429/challenge page/empty shell), retry once in the stealth browser and return its content instead of the block. Projected at 2+5; the actual charge stays at the base price when the plain fetch succeeded. Default: false'),
+  escalate: z.boolean().optional().default(false).describe('When the plain fetch comes back blocked (403/429/challenge page/empty shell), retry once in the stealth browser and return its content instead of the block. Under escalate_engine "auto" a Chrome TLS handshake with the honest CrawlForge User-Agent (impit) is tried first, and the browser runs only when that does not get the page. Projected at 2+5; the actual charge stays at the base price when the plain fetch succeeded. Default: false'),
   // "auto" is the default because the engine that gets past the walls this
   // stage exists for is camoufox, and asking for it by name used to be the
   // caller's problem. It resolves to camoufox when the package is installed

package/src/tools/scrape/unifiedScrape.js

@@ -380,7 +380,7 @@ export class UnifiedScrapeTool {
     // Opt-in, and second by construction: the plain fetch above has already
     // run and only a blocked verdict gets here (G1 — never start with
     // stealth). The stage reuses the compliance gate and the browser
-    // `stealth_mode` already drives, so it adds no evasion of its own.
+    // `stealth_mode` already drives, plus the impit TLS try (Phase 7) before it.
     // The vendor the plain fetch hit, or the one this host is remembered for
     // when that fetch was skipped.
     const vendorDetected = remembered ? (remembered.vendor ?? null) : (verdict?.blocked?.vendor ?? null);
@@ -428,7 +428,7 @@ export class UnifiedScrapeTool {
           // browser fixes — so the wording must not call every one a block.
           warnings.push(
             `escalate: the plain fetch ${vendorDetected ? `was blocked by ${vendorDetected}` : 'did not return the page'}; ` +
-            `the ${stealthEngine} stealth browser ${verdict.success ? 'returned it' : 'did not get it either'}`
+            `${stealthEngine === 'impit' ? 'a Chrome TLS handshake (impit, honest User-Agent)' : `the ${stealthEngine} stealth browser`} ${verdict.success ? 'returned it' : 'did not get it either'}`
           );
           // The host memory is deliberately left alone here. It records what
           // a PLAIN fetch met, which is what 3.3 reads before deciding to

package/src/utils/challengeDetection.js

@@ -76,7 +76,27 @@ function looksLikeRealPage({ title = '', text = '' }) {
 }
 
 /**
- * @param {{ title?: string, html?: string, text?: string }} page
+ * A page the server answered 200 that carries only a Turnstile widget: no
+ * bootstrap (the caller has already ruled that out) and none of the
+ * interstitial's prose. Cloudflare serves its walls as 403; a 200 with a
+ * widget is a page that embeds one. nowsecure.nl is exactly this — 200 to the
+ * honest CrawlForge UA, 43 visible characters and two widgets on Cloudflare's
+ * test sitekey — and every "Blocked" cell recorded for it came from this
+ * module, not from Cloudflare (2026-09-25). The status is what separates it
+ * from a widget-only wall, which the length floor cannot; without a status the
+ * floor still decides. It still has to be a document — a title and some text —
+ * so an empty 200 shell carrying only the widget script stays a wall, as
+ * Phase 1 decided.
+ */
+function looksLikeWidgetOnlyPage({ status, title = '', text = '' }) {
+  if (status !== 200) return false;
+  const visible = String(text).replace(/\s+/g, ' ').trim();
+  if (!String(title).trim() || !visible) return false;
+  return !INTERSTITIAL_PROSE.test(`${title} ${visible}`);
+}
+
+/**
+ * @param {{ title?: string, html?: string, text?: string, status?: number|null }} page
  * @returns {{ vendor: string, evidence: string } | null}
  */
 export function detectChallengePage(page = {}) {
@@ -87,6 +107,7 @@ export function detectChallengePage(page = {}) {
   // response input on a short page.
   if (!hit || hit.vendor !== 'cloudflare') return hit;
   if (looksLikeInterstitial(page)) return hit;
+  if (looksLikeWidgetOnlyPage(page)) return null;
   return looksLikeRealPage(page) ? null : hit;
 }
 

package/src/utils/complianceAudit.js

@@ -5,6 +5,8 @@
  * documented decision*. That means the override has to leave a trace: which key
  * asked, for which URL, from which tool, when. Without the row the override is
  * a silent product default again, which is the thing G5 exists to prevent.
+ * The same trail records each time a request goes out under a browser identity
+ * (a stealth render), so that choice is on record too.
  *
  * Rows go to `logs/compliance-audit.log` as JSONL (one row per line, appended)
  * and to a small in-memory ring the tools and tests can read back. Writing is
@@ -55,6 +57,24 @@ export function recordComplianceEvent(event = {}) {
   return row;
 }
 
+/**
+ * Record that a request is about to go out under a browser identity (a stealth
+ * render). Neither the raw key nor the raw owner token is stored: both are
+ * reduced to apiKeyId digests. `ownerId` names the website customer behind an
+ * internal-proxy request, where `apiKeyId` is always the service's own key.
+ * Never throws.
+ */
+export function recordStealthEscalation({ url, tool, engine, apiKey, ownerToken } = {}) {
+  return recordComplianceEvent({
+    event: 'stealth_escalation',
+    url,
+    tool,
+    engine: engine ?? null,
+    apiKeyId: apiKeyId(apiKey),
+    ...(ownerToken ? { ownerId: apiKeyId(ownerToken) } : {})
+  });
+}
+
 /** Most recent audit rows, newest last. Test/diagnostic hook. */
 export function getComplianceAuditRows() {
   return ring.slice();

package/src/utils/impitRung.js

@@ -0,0 +1,157 @@
+/**
+ * impitRung.js — the browser-TLS HTTP try inside the escalation stage
+ * (stealth review Phase 7).
+ *
+ * Some walls refuse Node's TLS handshake whatever the headers and accept a
+ * Chrome one: quora.com passed 0/4 over Node TLS and 4/4 over impit's Chrome
+ * profile (residential spike, 2026-09-26). impit makes that request without
+ * launching a browser, so the escalation stage tries it first and runs the
+ * browser only when it does not get the page.
+ *
+ * Owner decisions this module keeps (docs/STEALTH_REVIEW_2026-09.md, Phase 7):
+ * - It runs only inside escalation, whose callers already opted into a browser
+ *   identity. The plain fetch never uses it.
+ * - The User-Agent stays the honest CrawlForge one; only the TLS handshake and
+ *   impit's profile headers are Chrome's. indeed.com passed 3/4 that way and
+ *   0/4 with a Chrome UA.
+ * - The price is escalation's own (2 + 5), whichever step got the page.
+ *
+ * impit is an optional dependency with native bindings. When it is missing or
+ * fails to load, `loadImpit()` resolves to null and escalation runs the
+ * browser exactly as before.
+ *
+ * impit resolves DNS itself, outside the undici dispatcher ssrfGuard installs,
+ * so redirects are followed here, one hop at a time, and every hop is checked
+ * with DNS resolution — the same check safeGoto applies to a navigation.
+ */
+
+import { load } from 'cheerio';
+import { readBody } from 'crawlforge-extractors';
+import { resolveUserAgent } from './fetchIdentity.js';
+import { assertUrlAllowed } from './ssrfGuard.js';
+import { stealthDocumentVerdict } from './stealthVerdict.js';
+import { config, serverStealthProxies } from '../constants/config.js';
+
+/** The engine name escalation reports when this step got the page. */
+export const IMPIT_ENGINE = 'impit';
+
+const IMPIT_BROWSER = 'chrome151';
+const IMPIT_TIMEOUT_MS = 15000;
+const MAX_REDIRECTS = 5;
+
+// impit reads the HTML the server sent; it runs no JavaScript. quora.com
+// passed the verdict through it with a normal title over 59 characters of
+// "Something went wrong. Wait a moment and try again." — the client app's
+// fallback, not the page (measured 2026-09-26). The browser renders the
+// real page, so a document this thin goes to the browser instead.
+const MIN_UNRENDERED_TEXT_CHARS = 200;
+
+let impitClass; // undefined: not tried yet; null: unavailable
+
+/**
+ * Whether this deployment runs the step. `CRAWLFORGE_IMPIT=off` turns it off.
+ * Like the stealth engine, what it can clear depends on the exit IP: from a
+ * residential IP it cleared indeed.com, and from the hosted instance's
+ * datacenter IP (AS14618, Ashburn) it cleared nothing in 3 runs (2026-09-27),
+ * so render.yaml sets it off there. Read on every call, like the stealth
+ * proxy list, so an operator can change it without a restart.
+ * @returns {boolean}
+ */
+export function impitEnabled() {
+  return String(process.env.CRAWLFORGE_IMPIT || '').trim().toLowerCase() !== 'off';
+}
+
+/**
+ * The Impit class, or null when the optional dependency is absent or its
+ * native bindings do not load on this platform. Tried once per process.
+ * @returns {Promise<Function|null>}
+ */
+export async function loadImpit() {
+  if (impitClass === undefined) {
+    try {
+      const mod = await import('impit');
+      impitClass = mod.Impit ?? mod.default?.Impit ?? null;
+    } catch {
+      impitClass = null;
+    }
+  }
+  return impitClass;
+}
+
+/**
+ * Fetch `url` over a Chrome TLS handshake with the honest User-Agent and
+ * return the page when it passes the document verdict, or null when it does
+ * not (a wall, an error page, an empty shell) or the request failed. A null
+ * means "run the browser"; this function never throws for a failed fetch.
+ *
+ * An SSRF refusal on any hop DOES throw: the browser would be refused the same
+ * URL, so falling through to it would only fail slower.
+ *
+ * @param {string} url
+ * @param {object} [options]
+ * @param {Function} [options.Impit] the client class (tests inject a fake)
+ * @param {number} [options.timeoutMs]
+ * @returns {Promise<{ html: string, text: string, title: string, url: string, status: number, engine: string } | null>}
+ */
+export async function impitFetchPage(url, { Impit, timeoutMs = IMPIT_TIMEOUT_MS } = {}) {
+  const ImpitClass = Impit ?? await loadImpit();
+  if (!ImpitClass) return null;
+
+  // The operator's exit address, as the browser would use: a wall that has
+  // seen the server's own IP should not see it again from this step.
+  const proxyUrl = serverStealthProxies()[0];
+  const client = new ImpitClass({
+    browser: IMPIT_BROWSER,
+    headers: { 'user-agent': resolveUserAgent() },
+    followRedirects: false,
+    timeout: timeoutMs,
+    ...(proxyUrl ? { proxyUrl } : {})
+  });
+
+  const deadline = Date.now() + timeoutMs;
+  let current = url;
+  let response;
+  for (let hop = 0; ; hop++) {
+    await assertUrlAllowed(current, { resolveDns: true });
+    const remaining = deadline - Date.now();
+    if (remaining <= 0) return null;
+    try {
+      response = await client.fetch(current, { timeout: remaining });
+    } catch {
+      return null;
+    }
+    const location = response.status >= 300 && response.status < 400 ? response.headers?.get?.('location') : null;
+    if (!location) break;
+    if (hop >= MAX_REDIRECTS) return null;
+    try {
+      current = new URL(location, current).href;
+    } catch {
+      return null;
+    }
+    if (!/^https?:\/\//i.test(current)) return null;
+  }
+
+  let html;
+  try {
+    html = await readBody(response, { maxBytes: config.fetch.maxBodySize });
+  } catch {
+    return null;
+  }
+
+  const $ = load(html);
+  const title = $('title').first().text().trim();
+  $('script, style, noscript, template').remove();
+  const text = $('body').text().replace(/\s+/g, ' ').trim();
+  const verdict = stealthDocumentVerdict(
+    { url: current, title, text, html, status: response.status },
+    { fetcher: 'impit', rendered: false, contentReturned: false }
+  );
+  if (!verdict.success || text.length < MIN_UNRENDERED_TEXT_CHARS) return null;
+
+  return { html, text, title, url: current, status: response.status, engine: IMPIT_ENGINE };
+}
+
+/** Test hook: forget whether impit loaded. */
+export function _resetImpitLoader() {
+  impitClass = undefined;
+}

package/src/utils/robotsGate.js

@@ -24,7 +24,8 @@ import { identityHeaders, resolveUserAgent } from './fetchIdentity.js';
 import { throttleHost } from './hostRateLimiter.js';
 import { recordComplianceEvent, apiKeyId } from './complianceAudit.js';
 import { signRequestHeaders } from './webBotAuth.js';
-import { markPreflightRefusal } from '../server/requestContext.js';
+import { markPreflightRefusal, internalOwnerToken } from '../server/requestContext.js';
+import AuthManager from '../core/AuthManager.js';
 import { config } from '../constants/config.js';
 
 export class RobotsDisallowedError extends Error {
@@ -112,11 +113,16 @@ export async function robotsPreflight(url, options = {}) {
         ? 'respect_robots was disabled for this request. robots.txt did not disallow this URL, so the override changed nothing. The request is recorded against your API key.'
         : `respect_robots was disabled for this request and robots.txt on ${new URL(url).host} disallows this path. Fetching anyway is your decision and is recorded against your API key.`
     );
+    // No fetching tool passes its key down to here, so the row falls back to
+    // the key this server runs under, as the stealth_escalation row does.
+    // Without it every live override was logged as "anonymous".
+    const ownerToken = internalOwnerToken();
     recordComplianceEvent({
       event: 'robots_override',
       url,
       tool: options.tool || null,
-      apiKeyId: apiKeyId(options.apiKey),
+      apiKeyId: apiKeyId(options.apiKey ?? AuthManager.getConfig()?.apiKey),
+      ...(ownerToken ? { ownerId: apiKeyId(ownerToken) } : {}),
       userAgent,
       robotsAllowed: allowed
     });