crawlforge-mcp-server 6.11.0 → 6.12.1

package/CLAUDE.md

@@ -236,6 +236,11 @@ CRAWLFORGE_STEALTH_PROXIES=          # comma-separated proxy URLs used by the es
                                      # from the PROXY_ROTATION_* family, which belongs to
                                      # localization. The agent's automatic stealth retry
                                      # (stealth review Phase 3) reads it too.
+CRAWLFORGE_IMPIT=                    # `off` skips escalation's impit step (a Chrome TLS handshake with
+                                     # the honest UA, tried before the browser; stealth review Phase 7).
+                                     # Exit-IP dependent like the engine: it cleared indeed.com from a
+                                     # residential IP and nothing from the hosted datacenter IP, so
+                                     # render.yaml sets it off there. Measure with scripts/impit-probe.mjs.
 CRAWLFORGE_CLEARANCE_JAR=            # `off` disables the clearance jar (stealth review Phase 4):
                                      # cf_clearance/__cf_bm/datadome cookies a stealth render earned
                                      # are replayed to the next stealth context with the same engine,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "6.11.0",
+  "version": "6.12.1",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 31 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -121,7 +121,7 @@
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",
-    "crawlforge-extractors": "^1.9.0",
+    "crawlforge-extractors": "^1.10.0",
     "diff": "^9.0.0",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",

package/server.js

@@ -41,6 +41,7 @@ import { memoryMonitor } from "./src/utils/MemoryMonitor.js";
 import { config, validateConfig, getToolConfig } from "./src/constants/config.js";
 import AuthManager from "./src/core/AuthManager.js";
 import { makeWithAuth } from "./src/server/withAuth.js";
+import { stripScreenshotData } from "./src/server/screenshotResources.js";
 // Transport helpers
 import { connectStdio } from "./src/server/transports/stdio.js";
 import { connectStreamableHttp } from "./src/server/transports/streamableHttp.js";
@@ -61,7 +62,7 @@ import { REDACT_PII_PARAM } from "./src/server/redaction.js"; // Phase 5 (5.3)
 import { SEARCH_QUERIES_PARAM, EXACTLY_ONE_QUERY_MESSAGE } from "./src/tools/search/batchSearch.js"; // Phase 5 (5.1)
 import { markPreflightRefusal, internalOwnerToken } from "./src/server/requestContext.js";
 import { recordStealthEscalation } from "./src/utils/complianceAudit.js";
-import { loadImpit, impitFetchPage, IMPIT_ENGINE } from "./src/utils/impitRung.js"; // stealth review Phase 7
+import { loadImpit, impitEnabled, impitFetchPage, IMPIT_ENGINE } from "./src/utils/impitRung.js"; // stealth review Phase 7
 // D1.1 Resources + D1.2 Prompts + D1.4 Elicitation
 import { ResourceRegistry, MAX_RESOURCE_BLOB_BYTES } from "./src/resources/ResourceRegistry.js";
 import { PROMPTS, getPromptMessages } from "./src/prompts/PromptRegistry.js";
@@ -110,7 +111,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "6.11.0",
+  version: "6.12.1",
   description: "Production-ready MCP server with 31 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, stateful browser sessions with element refs, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -259,7 +260,8 @@ const stealthEscalation = async ({ url, engine, respectRobots, tool }) => {
   // Stealth review Phase 7: under "auto", a Chrome TLS handshake with the
   // honest User-Agent is tried before any browser launches. A caller who
   // named an engine asked for that browser, so it is not tried for them.
-  if (engine === 'auto' && await loadImpit()) {
+  // CRAWLFORGE_IMPIT=off skips it for a deployment whose exit IP it cannot help.
+  if (engine === 'auto' && impitEnabled() && await loadImpit()) {
     recordStealthEscalation({ url, tool, engine: IMPIT_ENGINE, ...auditIdentity });
     const page = await impitFetchPage(url);
     if (page) return { ...page, warnings };
@@ -939,12 +941,12 @@ registerToolIfEnabled("read_result", {
 
 // Tool: scrape_with_actions
 registerToolIfEnabled("scrape_with_actions", {
-  description: "Use this when you must interact with a page before scraping - login, click buttons, fill forms, scroll, or wait for dynamic content to load - for SPAs, login-gated content, or multi-step flows. Actions: snapshot, wait, click, type, press, scroll, screenshot, executeJavaScript, select (dropdowns), hover, navigate. Start a chain with {type:\"snapshot\"} to list the page's interactive elements with stable refs (@e1, @e2 ...), then target those refs in later actions instead of guessing CSS selectors; navigation invalidates refs, so snapshot again after one. Set browserOptions.stealth:true to run the chain in the stealth browser, and browserOptions.engine to pick its engine (\"auto\" by default - camoufox when it is installed, Chromium otherwise, and the result says which ran). robots.txt is respected on every navigation. Screenshots from this tool are stored as crawlforge://screenshot/{actionId} resources. Not for pages that render without interaction (scrape) and not as the first attempt on a blocked site (stealth_mode operation:\"scrape\"). Cost: 5 credits. Example: scrape_with_actions({url: \"https://app.com/dashboard\", actions: [{type:\"snapshot\"},{type:\"type\",selector:\"@e2\",text:\"user@a.com\"},{type:\"click\",selector:\"@e4\"}]})",
+  description: "Use this when you must interact with a page before scraping - login, click buttons, fill forms, scroll, or wait for dynamic content to load - for SPAs, login-gated content, or multi-step flows. Actions: snapshot, wait, click, type, press, scroll, screenshot, executeJavaScript (disabled unless the server runs with ALLOW_JAVASCRIPT_EXECUTION=true; refused on the hosted API), select (dropdowns), hover, navigate. Start a chain with {type:\"snapshot\"} to list the page's interactive elements with stable refs (@e1, @e2 ...), then target those refs in later actions instead of guessing CSS selectors; navigation invalidates refs, so snapshot again after one. Set browserOptions.stealth:true to run the chain in the stealth browser, and browserOptions.engine to pick its engine (\"auto\" by default - camoufox when it is installed, Chromium otherwise, and the result says which ran). robots.txt is respected on every navigation. Screenshots from this tool are stored as crawlforge://screenshot/{actionId} resources. Not for pages that render without interaction (scrape) and not as the first attempt on a blocked site (stealth_mode operation:\"scrape\"). Cost: 5 credits. Example: scrape_with_actions({url: \"https://app.com/dashboard\", actions: [{type:\"snapshot\"},{type:\"type\",selector:\"@e2\",text:\"user@a.com\"},{type:\"click\",selector:\"@e4\"}]})",
   annotations: { title: "Scrape with Browser Actions", readOnlyHint: true, destructiveHint: false, idempotentHint: false, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The URL to scrape"),
     actions: z.array(z.object({
-      type: z.enum(['snapshot', 'wait', 'click', 'type', 'press', 'scroll', 'screenshot', 'executeJavaScript', 'select', 'hover', 'navigate']),
+      type: z.enum(['snapshot', 'wait', 'click', 'type', 'press', 'scroll', 'screenshot', 'executeJavaScript', 'select', 'hover', 'navigate']).describe("executeJavaScript is disabled unless the server runs with ALLOW_JAVASCRIPT_EXECUTION=true; on the hosted API it is refused."),
       selector: z.string().optional().describe("A CSS selector, or a @e1 ref from an earlier snapshot action in this chain"),
       text: z.string().optional(),
       key: z.string().optional(),
@@ -1021,7 +1023,7 @@ registerToolIfEnabled("scrape_with_actions", {
       includeImages: z.boolean().default(true)
     }).optional().describe("Content extraction options. selectors results are returned as content.json.extracted, so include \"json\" in formats when passing selectors — without it the extraction is not part of the response."),
     continueOnActionError: z.boolean().default(false).describe("Continue executing actions if one fails"),
-    maxRetries: z.number().min(0).max(3).default(1).describe("Maximum retry attempts on failure"),
+    maxRetries: z.number().min(0).max(3).default(0).describe("Whole-chain retries on failure (0-3). A retry re-navigates to the starting URL and replays every action; each attempt is reported under attempts[]."),
     screenshotOnError: z.boolean().default(true).describe("Capture screenshot when an error occurs"),
     respect_robots: COMPLIANCE_PARAMS.respect_robots,
     ...MAX_INLINE_CHARS_PARAM,
@@ -1035,17 +1037,11 @@ registerToolIfEnabled("scrape_with_actions", {
     // resources (the documented contract) and annotate each with its URI.
     // The base64 `data` is dropped once stored, as the stealth_mode block
     // below already does: a failed chain's error screenshot came back inline
-    // as 1.7 MB of base64 beside a 40-char markdown (R21, 2026-09-09).
-    if (Array.isArray(result.screenshots)) {
-      result.screenshots = result.screenshots.map((shot) => {
-        if (shot?.actionId && shot?.data) {
-          resourceRegistry.storeScreenshot(shot.actionId, shot.data);
-          const { data, ...rest } = shot;
-          return { ...rest, resourceUri: `crawlforge://screenshot/${shot.actionId}` };
-        }
-        return shot;
-      });
-    }
+    // as 1.7 MB of base64 beside a 40-char markdown (R21, 2026-09-09). The
+    // same bytes also sit in actionResults[].result, content.screenshots and
+    // attempts[].results, so every copy is stripped, not just the top-level
+    // array (Phase 0, 0.4).
+    stripScreenshotData(result, (actionId, data) => resourceRegistry.storeScreenshot(actionId, data));
 
     return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
   } catch (error) {

package/src/cli/commands/actions.js

@@ -12,6 +12,7 @@ export function register(program) {
     .description('Run browser automation actions against a URL')
     .requiredOption('--script <file>', 'JSON file containing action script')
     .option('--screenshot', 'Capture screenshots during action execution')
+    .addHelpText('after', '\nexecuteJavaScript actions need ALLOW_JAVASCRIPT_EXECUTION=true in the server environment; otherwise they are refused.\n')
     .action(async (url, opts, cmd) => {
       const globals = cmd.parent.opts();
       const cliFlags = { json: globals.json, pretty: globals.pretty, quiet: globals.quiet };

package/src/core/ActionExecutor.js

@@ -22,11 +22,12 @@ const JS_EXECUTION_TIMEOUT_MS = parseInt(process.env.JS_EXECUTION_TIMEOUT_MS ||
 // for, the backstop can only say "timed out".
 const ACTION_TIMEOUT_GRACE_MS = 2000;
 
-// Ceiling for a single error-recovery strategy. By the time recovery runs the
-// action has already spent its whole timeout failing, so each strategy gets a
-// bounded slice — granting it another full deadline made a chain that was never
-// going to work cost several times its stated timeout.
-const RECOVERY_TIMEOUT_MS = 5000;
+// Total error-recovery budget per action, shared by every strategy it tries
+// (their delay() pauses included). By the time recovery runs the action has
+// already spent its whole timeout failing, so recovery gets one bounded slice —
+// giving each strategy another full deadline made a chain that was never going
+// to work cost several times its stated timeout.
+const RECOVERY_TIMEOUT_MS = 3000;
 
 // The only states locator.waitFor()/page.waitForSelector() accept. The rest of
 // the wait-action enum (enabled/disabled/stable) are ElementHandle states and
@@ -176,6 +177,8 @@ const ActionChainSchema = z.object({
   continueOnError: z.boolean().default(false),
   timeout: z.number().min(1000).max(300000).default(30000),
   retryChain: z.number().min(0).max(3).default(0),
+  // Per-chain override of the constructor's enableScreenshotOnError.
+  screenshotOnError: z.boolean().optional(),
   metadata: z.record(z.any()).prefault({})
 });
 
@@ -272,6 +275,10 @@ export class ActionExecutor extends EventEmitter {
         browserOptions,
         startTime,
         results: [],
+        // One entry per run of the chain (retryChain replays it); `attempt` is
+        // the 1-based run whose per-action results are in `results`.
+        attempt: 0,
+        attempts: [],
         errors: [],
         screenshots: [],
         metadata: {
@@ -318,7 +325,7 @@ export class ActionExecutor extends EventEmitter {
         executionContext.error = error.message;
 
         // Capture error screenshot if enabled
-        if (this.enableScreenshotOnError && page) {
+        if ((validatedChain.screenshotOnError ?? this.enableScreenshotOnError) && page) {
           try {
             const errorScreenshot = await this.captureScreenshot(page, {
               fullPage: true,
@@ -378,6 +385,7 @@ export class ActionExecutor extends EventEmitter {
           screenshotCount: executionContext.screenshots.length,
           capturedStates: undefined,
           capturedStateCount: (executionContext.capturedStates || []).length,
+          attempts: undefined, // each carries a copy of a results array
           results: executionContext.results.map(r => (
             r?.result?.data !== undefined
               ? { ...r, result: { ...r.result, data: undefined, dataBytes: typeof r.result.data === 'string' ? r.result.data.length : undefined } }
@@ -402,6 +410,8 @@ export class ActionExecutor extends EventEmitter {
         navigationStatus: executionContext.page?.__crawlforgeNavigation?.status ?? null,
         executionTime: Date.now() - startTime,
         results: executionContext.results,
+        attempt: executionContext.attempt,
+        attempts: executionContext.attempts,
         screenshots: executionContext.screenshots,
         capturedStates: executionContext.capturedStates || [],
         metadata: executionContext.metadata,
@@ -424,6 +434,8 @@ export class ActionExecutor extends EventEmitter {
         // results, the error screenshot, and any intermediate-state
         // captures) instead of discarding them.
         results: executionContext?.results || [],
+        attempt: executionContext?.attempt ?? 0,
+        attempts: executionContext?.attempts || [],
         screenshots: executionContext?.screenshots || [],
         capturedStates: executionContext?.capturedStates || []
       };
@@ -440,9 +452,11 @@ export class ActionExecutor extends EventEmitter {
     let lastError;
 
     for (let attempt = 0; attempt <= chain.retryChain; attempt++) {
+      executionContext.attempt = attempt + 1;
       try {
         if (attempt > 0) {
           this.log('info', 'Retrying chain execution, attempt ' + (attempt + 1));
+          // The failed run's results are already recorded in attempts[].
           executionContext.results = []; // Clear previous results on retry
           executionContext.capturedStates = []; // Clear previous captures on retry
           // Replaying the chain against whatever the failed attempt left behind
@@ -454,7 +468,14 @@ export class ActionExecutor extends EventEmitter {
 
         // Execute actions in sequence
         for (let i = 0; i < chain.actions.length; i++) {
-          const action = chain.actions[i];
+          // chain.timeout is the deadline for actions that name none and the
+          // ceiling for those that do — actionTimeout() reads it off the
+          // action, so without this the chain's timeout never reached a
+          // Playwright call (executeActionsOnPage applies the same default).
+          const action = {
+            ...chain.actions[i],
+            timeout: Math.min(chain.actions[i].timeout || chain.timeout, chain.timeout)
+          };
           const actionResult = await this.executeActionInternal(page, action, executionContext);
           
           executionContext.results.push(actionResult);
@@ -509,10 +530,16 @@ export class ActionExecutor extends EventEmitter {
           }
         }
 
+        executionContext.attempts.push({
+          attempt: attempt + 1, success: true, results: [...executionContext.results]
+        });
         return { success: true, attempt: attempt + 1 };
 
       } catch (error) {
         lastError = error;
+        executionContext.attempts.push({
+          attempt: attempt + 1, success: false, error: error.message, results: [...executionContext.results]
+        });
         this.log('warn', 'Chain execution attempt ' + (attempt + 1) + ' failed: ' + error.message);
         
         if (attempt < chain.retryChain) {
@@ -754,12 +781,18 @@ export class ActionExecutor extends EventEmitter {
   }
 
   /**
-   * Deadline for one recovery strategy — bounded, see RECOVERY_TIMEOUT_MS.
+   * Deadline for one Playwright call inside a recovery strategy: what is left
+   * of the shared budget (see RECOVERY_TIMEOUT_MS), never more than the
+   * action's own timeout. Strategies call it after their delay() so the pause
+   * is charged to the same budget. Floored at 1 because Playwright reads a
+   * timeout of 0 as "wait forever".
    * @param {Object} action - Action configuration
+   * @param {number} [deadline] - Epoch ms at which the recovery budget ends
    * @returns {number} Timeout in ms
    */
-  recoveryTimeout(action) {
-    return Math.min(this.actionTimeout(action), RECOVERY_TIMEOUT_MS);
+  recoveryTimeout(action, deadline) {
+    const left = deadline === undefined ? RECOVERY_TIMEOUT_MS : deadline - Date.now();
+    return Math.max(1, Math.min(this.actionTimeout(action), left));
   }
 
   /**
@@ -1500,16 +1533,35 @@ export class ActionExecutor extends EventEmitter {
       return { success: false };
     }
 
+    // An element that is not in the document cannot be force-clicked, scrolled
+    // into view or focused, so every strategy below would only spend the
+    // budget re-waiting for it. count() answers at once without waiting.
+    if (action.selector) {
+      let matches = 1;
+      try {
+        matches = await this.elementLocator(page, action.selector).count();
+      } catch (_) {
+        // A ref that cannot be resolved, or a page stub in tests: let the
+        // strategies decide.
+      }
+      if (matches === 0) return { success: false };
+    }
+
     const strategies = this.errorRecoveryStrategies.get(action.type) || [];
     // `retries` caps how many strategies get a turn. Walking all of them
     // unconditionally would add a second full round of timeouts to every action
     // that was never going to succeed.
     const budget = Math.max(0, action.retries ?? 1);
+    // One deadline shared by every strategy this action tries, so recovery
+    // costs at most RECOVERY_TIMEOUT_MS on top of the action's own timeout
+    // however many strategies are registered.
+    const deadline = Date.now() + RECOVERY_TIMEOUT_MS;
 
     for (const strategy of strategies.slice(0, budget)) {
+      if (Date.now() >= deadline) break;
       try {
         this.log('info', 'Attempting error recovery with strategy: ' + strategy.name);
-        const result = await strategy.recover(page, action, error, executionContext);
+        const result = await strategy.recover(page, action, error, executionContext, deadline);
         
         if (result.success) {
           return {
@@ -1534,21 +1586,20 @@ export class ActionExecutor extends EventEmitter {
     this.errorRecoveryStrategies.set('click', [
       {
         name: 'waitAndRetry',
-        recover: async (page, action) => {
+        recover: async (page, action, _error, _context, deadline) => {
           await this.delay(1000);
           await this.elementLocator(page, action.selector)
-            .click({ force: true, timeout: this.recoveryTimeout(action) });
+            .click({ force: true, timeout: this.recoveryTimeout(action, deadline) });
           return { success: true, data: { recovered: true, strategy: 'waitAndRetry' } };
         }
       },
       {
         name: 'scrollIntoView',
-        recover: async (page, action) => {
-          const timeout = this.recoveryTimeout(action);
+        recover: async (page, action, _error, _context, deadline) => {
           const locator = this.elementLocator(page, action.selector);
-          await locator.scrollIntoViewIfNeeded({ timeout });
+          await locator.scrollIntoViewIfNeeded({ timeout: this.recoveryTimeout(action, deadline) });
           await this.delay(500);
-          await locator.click({ timeout });
+          await locator.click({ timeout: this.recoveryTimeout(action, deadline) });
           return { success: true, data: { recovered: true, strategy: 'scrollIntoView' } };
         }
       }
@@ -1558,33 +1609,20 @@ export class ActionExecutor extends EventEmitter {
     this.errorRecoveryStrategies.set('type', [
       {
         name: 'focusAndRetry',
-        recover: async (page, action) => {
-          const timeout = this.recoveryTimeout(action);
+        recover: async (page, action, _error, _context, deadline) => {
           const locator = this.elementLocator(page, action.selector);
-          await locator.focus({ timeout });
+          await locator.focus({ timeout: this.recoveryTimeout(action, deadline) });
           await this.delay(500);
-          await locator.pressSequentially(action.text, { delay: action.delay, timeout });
+          await locator.pressSequentially(action.text, {
+            delay: action.delay, timeout: this.recoveryTimeout(action, deadline)
+          });
           return { success: true, data: { recovered: true, strategy: 'focusAndRetry' } };
         }
       }
     ]);
 
-    // Wait action recovery strategies
-    this.errorRecoveryStrategies.set('wait', [
-      {
-        name: 'extendTimeout',
-        recover: async (page, action) => {
-          if (!action.selector) return { success: false };
-          // One more bounded window, not a doubled one: the action has already
-          // waited its full timeout, so doubling made a wait that could never
-          // resolve cost 3x what the caller asked for.
-          await this.waitForCondition(
-            page, action.selector, action.condition, this.recoveryTimeout(action)
-          );
-          return { success: true, data: { recovered: true, strategy: 'extendTimeout' } };
-        }
-      }
-    ]);
+    // No strategies for `wait`: the wait already was the retry, so a failed
+    // one has nothing to recover and gets no second window.
   }
 
   /**

package/src/server/screenshotResources.js

@@ -0,0 +1,50 @@
+/**
+ * screenshotResources — move a scrape_with_actions result's screenshot bytes
+ * out of the inline result and into crawlforge://screenshot/{actionId}
+ * resources.
+ *
+ * The tool carries the same base64 in up to four places: `screenshots`,
+ * `actionResults[].result` (screenshot actions), `content.screenshots`
+ * (formats:["screenshots"] copies the same array) and `attempts[].results`
+ * (one per chain attempt). Stripping only the first left the others inline
+ * (Phase 0, 0.4). Each actionId is published once.
+ */
+
+/**
+ * @param {object} result - the tool result, mutated in place
+ * @param {(actionId: string, data: string) => void} publish - stores the base64
+ * @returns {object} the same result
+ */
+export function stripScreenshotData(result, publish) {
+  const published = new Set();
+
+  const strip = (shot, actionId) => {
+    if (!actionId || typeof shot?.data !== 'string') return shot;
+    if (!published.has(actionId)) {
+      publish(actionId, shot.data);
+      published.add(actionId);
+    }
+    const { data, ...rest } = shot;
+    return { ...rest, resourceUri: `crawlforge://screenshot/${actionId}` };
+  };
+
+  const stripResults = (results) => results.map((r) => (
+    r?.type === 'screenshot' ? { ...r, result: strip(r.result, r.id) } : r
+  ));
+
+  if (Array.isArray(result.screenshots)) {
+    result.screenshots = result.screenshots.map((shot) => strip(shot, shot?.actionId));
+  }
+  if (Array.isArray(result.content?.screenshots)) {
+    result.content.screenshots = result.content.screenshots.map((shot) => strip(shot, shot?.actionId));
+  }
+  if (Array.isArray(result.actionResults)) {
+    result.actionResults = stripResults(result.actionResults);
+  }
+  if (Array.isArray(result.attempts)) {
+    result.attempts = result.attempts.map((a) => (
+      Array.isArray(a?.results) ? { ...a, results: stripResults(a.results) } : a
+    ));
+  }
+  return result;
+}

package/src/skills/agent-skills/crawlforge-batch-automation/references/actions.md

@@ -157,7 +157,7 @@ main frame only — elements inside iframes and shadow DOM get no refs.
 | `formAutoFill` | — | Declarative form fill: `fields[]` + `submitSelector`. |
 | `browserOptions` | — | `headless`, `userAgent`, `viewportWidth/Height`, `timeout`. |
 | `continueOnActionError` | `false` | Keep going if one action fails. |
-| `maxRetries` | `1` | 0–3 retries on failure. |
+| `maxRetries` | `0` | 0–3 whole-chain retries on failure. A retry re-navigates to the starting URL and replays every action; each run is reported under `attempts[]`. |
 | `screenshotOnError` | `true` | Capture a screenshot when an error occurs. |
 
 ## CLI action-script format

package/src/skills/agent-skills/crawlforge-stealth-browsing/SKILL.md

@@ -91,6 +91,12 @@ Cloudflare scores the address and its ASN before it serves a challenge, so a
 datacenter proxy changes the address without changing the class of address being
 scored. Route through your own residential proxy — CrawlForge supplies none.
 
+A self-hosted server can instead hand every stealth session to Browserbase:
+set `CRAWLFORGE_BROWSER_BACKEND=browserbase` and `BROWSERBASE_API_KEY` (see
+`docs/cloud-browser.md`). That account supplies the residential exit and any
+CAPTCHA solving on its own terms; CrawlForge itself never solves a challenge —
+a challenge page is reported as blocked.
+
 Server-wide, set `CRAWLFORGE_STEALTH_PROXIES` (comma-separated proxy URLs): the
 `scrape` escalation stage, `stealth_mode`, `browser_session`,
 `scrape_with_actions`, the `deep_research` retry and the `agent` tool's automatic

package/src/tools/advanced/ScrapeWithActionsTool.js

@@ -215,7 +215,7 @@ const ScrapeWithActionsSchema = z.object({
 
   // Error handling
   continueOnActionError: z.boolean().default(false),
-  maxRetries: z.number().min(0).max(3).default(1),
+  maxRetries: z.number().min(0).max(3).default(0),
   screenshotOnError: z.boolean().default(true),
 
   // ── Recording / replay ──────────────────────────────────────────────────
@@ -446,6 +446,7 @@ export class ScrapeWithActionsTool extends EventEmitter {
         continueOnError: params.continueOnActionError,
         timeout: browserOptions.timeout || 30000,
         retryChain: params.maxRetries,
+        screenshotOnError: params.screenshotOnError,
         metadata: {
           sessionId: sessionContext.id,
           originalActionCount: params.actions.length,
@@ -483,6 +484,12 @@ export class ScrapeWithActionsTool extends EventEmitter {
 
     // Process action results
     const actionResults = this.processActionResults(chainResult.results);
+    // Every attempt the executor made (retryChain > 0 replays the whole
+    // chain), in the same shape as actionResults; `attempt` is the one whose
+    // results are reported above.
+    const attempts = (chainResult.attempts || [
+      { attempt: 1, success: chainResult.success, error: chainResult.error, results: chainResult.results }
+    ]).map((a) => ({ ...a, results: this.processActionResults(a.results || []) }));
     const intermediateStates = params.captureIntermediateStates ?
       await this.extractIntermediateStates(chainResult.capturedStates || [], params) : [];
 
@@ -527,6 +534,8 @@ export class ScrapeWithActionsTool extends EventEmitter {
       ...(warnings.length ? { warnings } : {}),
 
       actionResults,
+      attempt: chainResult.attempt ?? attempts.length,
+      attempts,
       totalActions: params.actions.length,
       successfulActions: actionResults.filter(r => r.success).length,
       failedActions: actionResults.filter(r => !r.success).length,

package/src/tools/extract/extractEmbeddedState.js

@@ -19,6 +19,9 @@ import { extractEmbeddedState, selectJsonPath } from 'crawlforge-extractors';
 // the caller an exact way to ask for less.
 const LARGE_RESULT_BYTES = 256_000;
 
+const BLOCKED_HINT =
+  'stealth_mode operation:"scrape" renders the page in a browser and returns the HTML; the embedded state is in the returned html. Do not repeat this call with a different path.';
+
 /**
  * @param {{ url: string, path?: string, user_agent?: string, respect_robots?: boolean }} params
  */
@@ -81,8 +84,15 @@ export async function extractEmbeddedStateHandler({ url, path, user_agent, respe
       }]
     };
   } catch (error) {
+    // A block or server error is not a path problem: the generic hint
+    // ("call again without `path`") would send the caller back into the same
+    // wall. Naming the next step here keeps withAuth from appending it.
+    const blocked = /^HTTP (403|429|444|5\d\d)\b/.test(error.message);
     return {
-      content: [{ type: 'text', text: `Failed to extract embedded state: ${error.message}` }],
+      content: [{
+        type: 'text',
+        text: `Failed to extract embedded state: ${error.message}` + (blocked ? `\nNext step: ${BLOCKED_HINT}` : '')
+      }],
       isError: true
     };
   }

package/src/utils/impitRung.js

@@ -48,6 +48,19 @@ const MIN_UNRENDERED_TEXT_CHARS = 200;
 
 let impitClass; // undefined: not tried yet; null: unavailable
 
+/**
+ * Whether this deployment runs the step. `CRAWLFORGE_IMPIT=off` turns it off.
+ * Like the stealth engine, what it can clear depends on the exit IP: from a
+ * residential IP it cleared indeed.com, and from the hosted instance's
+ * datacenter IP (AS14618, Ashburn) it cleared nothing in 3 runs (2026-09-27),
+ * so render.yaml sets it off there. Read on every call, like the stealth
+ * proxy list, so an operator can change it without a restart.
+ * @returns {boolean}
+ */
+export function impitEnabled() {
+  return String(process.env.CRAWLFORGE_IMPIT || '').trim().toLowerCase() !== 'off';
+}
+
 /**
  * The Impit class, or null when the optional dependency is absent or its
  * native bindings do not load on this platform. Tried once per process.