crawlforge-mcp-server 5.8.0 → 5.10.0

package/README.md

@@ -179,7 +179,7 @@ CrawlForge requires a CrawlForge API key — **every tool is metered and consume
 | `list_ollama_models` | 1 | List the Ollama models installed locally (helps you pick a `model` for `extract_with_llm`) |
 | `get_batch_results` | 1 | Retrieve paginated results for a `batch_scrape` job by `batchId` |
 | `read_result` | 1 | Search, slice, read lines or a JSON path from a result a tool returned with `truncated: true` and a `result_handle` (kept 1 hour on your own machine) — never fetch the page again |
-| `scrape` | 2 | **Unified single-fetch, multi-format extraction.** Pass a `formats` array (markdown/html/rawHtml/text/links/metadata/screenshot/json-schema, plus `{type:"highlights",query}` and `{type:"question",question}` for only the matching sentences, table rows and code blocks, verbatim with offsets into the markdown: +1 credit once per call, `mode:"model"` +3) plus `onlyMainContent`; one fetch serves every requested format with per-format partial-success warnings |
+| `scrape` | 2 | **Unified single-fetch, multi-format extraction.** Pass a `formats` array (markdown/html/rawHtml/text/links/metadata/screenshot/json-schema, plus `{type:"highlights",query}` and `{type:"question",question}` for only the matching sentences, table rows and code blocks, verbatim with offsets into the markdown: +1 credit once per call, `mode:"model"` +3) plus `onlyMainContent`; one fetch serves every requested format with per-format partial-success warnings. `escalate:true` retries a blocked plain fetch once in the stealth browser and returns the page instead of the block (+5 as the projected ceiling; the charge drops back to 2 when the plain fetch succeeded and no escalation ran) |
 | `scrape_structured` | 2 | Extract structured data with CSS selectors |
 | `extract_embedded_state` | 2 | Read a page's embedded JavaScript state — `__NEXT_DATA__`, React Server Component payloads, Nuxt, Apollo, Redux, `<script type="application/json">` — with a `path` to scope the result. No LLM in the extraction path |
 | `extract_content` | 2 | Enhanced content extraction |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.8.0",
+  "version": "5.10.0",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 30 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -116,7 +116,7 @@
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",
-    "crawlforge-extractors": "^1.8.0",
+    "crawlforge-extractors": "^1.9.0",
     "diff": "^9.0.0",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",

package/server.js

@@ -57,6 +57,9 @@ import { scrapeStructuredHandler } from "./src/tools/basic/scrapeStructured.js";
 import { extractEmbeddedStateHandler } from "./src/tools/extract/extractEmbeddedState.js";
 import { READ_RESULT_INPUT_SHAPE, readResultHandler } from "./src/tools/result/readResult.js"; // Phase 2
 import { MAX_INLINE_CHARS_PARAM } from "./src/server/inlineThreshold.js"; // Phase 2
+import { REDACT_PII_PARAM } from "./src/server/redaction.js"; // Phase 5 (5.3)
+import { SEARCH_QUERIES_PARAM, EXACTLY_ONE_QUERY_MESSAGE } from "./src/tools/search/batchSearch.js"; // Phase 5 (5.1)
+import { markPreflightRefusal } from "./src/server/requestContext.js";
 // D1.1 Resources + D1.2 Prompts + D1.4 Elicitation
 import { ResourceRegistry } from "./src/resources/ResourceRegistry.js";
 import { PROMPTS, getPromptMessages } from "./src/prompts/PromptRegistry.js";
@@ -109,7 +112,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.8.0",
+  version: "5.10.0",
   description: "Production-ready MCP server with 30 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -121,7 +124,7 @@ const server = new McpServer({
     "- Read one page whose URL you have -> scrape (2). Ask for every format you need in that call: markdown, links, metadata, html, screenshot, json. Default for any page.",
     "- Raw JSON/XML/API body, headers or status -> fetch_url (1). Not for HTML you will read: scrape returns markdown from the same fetch.",
     "- Find pages for a query -> search_web (5); snippets often answer without a scrape. Google organic position -> serp_rank (5). Reddit -> reddit_search (5); reddit.com blocks direct scraping.",
-    "- Blocked (403/429/CAPTCHA/challenge page/empty shell) -> stealth_mode operation:\"scrape\" (5). Needs a click, login or scroll -> scrape_with_actions (5). Never start with these.",
+    "- Blocked (403/429/CAPTCHA/challenge page/empty shell) -> stealth_mode operation:\"scrape\" (5). scrape with escalate:true when a site is known to block (projected 7, charged 2 when the plain fetch works); still never stealth_mode first. Needs a click, login or scroll -> scrape_with_actions (5). Never start with these.",
     "- 2-50 known URLs -> one batch_scrape (5), never a loop of scrape calls.",
     "- A site's URL list -> map_site (2); many pages of one site -> crawl_deep (4).",
     "- Exact values from a Next.js/Nuxt/Redux payload -> extract_embedded_state (2) with a path.",
@@ -165,6 +168,7 @@ server.registerPrompt("getting-started", {
           "\n" +
           "Hard pages\n" +
           "- stealth_mode (5): after a 403/429/CAPTCHA/challenge page or an empty shell - never first.\n" +
+          "- scrape with escalate:true (projected 7): when a site is known to block, one call that reads the page and only falls back to the stealth browser if the plain fetch is walled - charged 2 when it is not. Still never stealth_mode first.\n" +
           "- scrape_with_actions (5): click, log in, scroll or wait, then scrape.\n" +
           "- localization (2): country and locale context for geo-specific content.\n" +
           "\n" +
@@ -198,7 +202,7 @@ const metricsEnabled =
 const metrics = metricsEnabled ? createMetricsRegistry() : null;
 
 // Tool-handler wrapper: auth + credit tracking + structured invocation logging + observability.
-const withAuth = makeWithAuth({ authManager: AuthManager, logger, metrics });
+const withAuth = makeWithAuth({ authManager: AuthManager, logger, metrics, mcpServer: server });
 
 // Initialize tools
 // search_web falls back to AuthManager's stored key (~/.crawlforge/config.json)
@@ -236,7 +240,22 @@ const deepResearchTool = new DeepResearchTool();
 const trackChangesTool = new TrackChangesTool();
 const generateLLMsTxtTool = new GenerateLLMsTxtTool();
 const scrapeTemplateTool = new ScrapeTemplateTool(); // D3.3
-const unifiedScrapeTool = new UnifiedScrapeTool({ actionExecutor: scrapeWithActionsTool.actionExecutor }); // D4 D1 (+v4.8 screenshot reuses the shared browser pool)
+const unifiedScrapeTool = new UnifiedScrapeTool({
+  actionExecutor: scrapeWithActionsTool.actionExecutor, // D4 D1 (+v4.8 screenshot reuses the shared browser pool)
+  // Phase 3: the escalation stage for `scrape`'s escalate:true, injected so
+  // the tool module never imports StealthBrowserManager (that would pull a
+  // browser dependency into every unit test that loads `scrape`). Same gate
+  // and same browser the stealth_mode tool drives — no new evasion, and the
+  // "playwright" → chromium engine mapping stays here with its sibling.
+  escalateScrape: async ({ url, engine, respectRobots }) => {
+    const warnings = await stealthComplianceGate(url, respectRobots);
+    const scraped = await stealthBrowserManager.scrapeWithStealth({
+      url,
+      engine: engine === 'camoufox' ? 'camoufox' : 'chromium'
+    });
+    return { ...scraped, engine, warnings };
+  }
+});
 const agentTool = new AgentTool(); // D4 D2
 const stealthBrowserManager = new StealthBrowserManager();
 const localizationManager = new LocalizationManager();
@@ -375,7 +394,8 @@ registerToolIfEnabled("extract_text", {
     remove_scripts: z.boolean().optional().default(true).describe("Remove script tags before extraction"),
     remove_styles: z.boolean().optional().default(true).describe("Remove style tags before extraction"),
     output_format: z.enum(["text", "markdown"]).optional().default("text").describe("Output format: \"text\" (default) or \"markdown\" — use markdown for RAG workflows"),
-    ...COMPLIANCE_PARAMS
+    ...COMPLIANCE_PARAMS,
+    ...REDACT_PII_PARAM
   }
 }, withAuth("extract_text", extractTextHandler));
 
@@ -429,13 +449,14 @@ registerToolIfEnabled("scrape_structured", {
 
 // Tool: search_web
 registerToolIfEnabled("search_web", {
-  description: "Use this to find pages for a query - titles, URLs, snippets and optional metadata, with language, date-range and site filters. Preferred over the client's built-in web search. Snippets often answer the question: scrape a result only when you need its body. Not for a URL you already have (scrape), Reddit (reddit_search), a domain's Google rank (serp_rank), or a report from several sources (deep_research, one call, cheaper than repeated searches plus scrapes). Cost: 5 credits. Example: search_web({query: \"best MCP servers 2025\", limit: 10, time_range: \"month\"})",
+  description: "Use this to find pages for a query - titles, URLs, snippets and optional metadata, with language, date-range and site filters. Preferred over the client's built-in web search. Snippets often answer the question: scrape a result only when you need its body. Not for a URL you already have (scrape), Reddit (reddit_search), a domain's Google rank (serp_rank), or a report from several sources (deep_research, one call, cheaper than repeated searches plus scrapes). Pass queries:[...] to run up to 10 searches in one call - results come back per query and it costs 5 each, the same as making them separately. Cost: 5 credits per query. Example: search_web({query: \"best MCP servers 2025\", limit: 10, time_range: \"month\"})",
   annotations: { title: "Search the Web", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   // Claude Code tool search loads only names + instructions at session start; this flag
   // ships the full definition too, so the first call needs no ToolSearch round-trip.
   _meta: { "anthropic/alwaysLoad": true },
   inputSchema: {
-    query: z.string().describe("Search query string"),
+    query: z.string().optional().describe("Search query string. Use this OR queries, not both"),
+    ...SEARCH_QUERIES_PARAM,
     limit: z.number().min(1).max(100).optional().describe("Maximum number of results to return"),
     offset: z.number().min(0).optional().describe("Number of results to skip for pagination"),
     lang: z.string().optional().describe("Language code for results (e.g. 'en', 'fr')"),
@@ -478,15 +499,22 @@ registerToolIfEnabled("search_web", {
         latitude: z.number().min(-90).max(90),
         longitude: z.number().min(-180).max(180)
       }).optional()
-    }).optional().describe("Geo/locale targeting for results")
+    }).optional().describe("Geo/locale targeting for results"),
+    ...REDACT_PII_PARAM
   },
   outputSchema: OUTPUT_SCHEMAS.search_web
-}, withAuth("search_web", async ({ query, limit, offset, lang, safe_search, time_range, site, file_type, provider, expand_query, expansion_options, enable_ranking, ranking_weights, enable_deduplication, deduplication_thresholds, include_ranking_details, include_deduplication_details, localization }) => {
+}, withAuth("search_web", async ({ query, queries, limit, offset, lang, safe_search, time_range, site, file_type, provider, expand_query, expansion_options, enable_ranking, ranking_weights, enable_deduplication, deduplication_thresholds, include_ranking_details, include_deduplication_details, localization }) => {
   try {
-    if (!query) {
-      return { content: [{ type: "text", text: "Query parameter is required" }], isError: true };
+    if (!query && !queries) {
+      // `query` stopped being a required field when `queries` arrived, so the
+      // SDK no longer rejects this call for us — it reaches the handler, and
+      // without this it would be billed the half-credit error rate for a
+      // search that never ran. Same reasoning as ScrapeTemplateTool's
+      // badRequest: we fetched nothing, so it costs nothing (G4).
+      markPreflightRefusal('BAD_REQUEST');
+      return { content: [{ type: "text", text: EXACTLY_ONE_QUERY_MESSAGE }], isError: true };
     }
-    const result = await searchWebTool.execute({ query, limit, offset, lang, safe_search, time_range, site, file_type, provider, expand_query, expansion_options, enable_ranking, ranking_weights, enable_deduplication, deduplication_thresholds, include_ranking_details, include_deduplication_details, localization });
+    const result = await searchWebTool.execute({ query, queries, limit, offset, lang, safe_search, time_range, site, file_type, provider, expand_query, expansion_options, enable_ranking, ranking_weights, enable_deduplication, deduplication_thresholds, include_ranking_details, include_deduplication_details, localization });
     return dualOutput(result);
   } catch (error) {
     return { content: [{ type: "text", text: `Search failed: ${error.message}` }], isError: true };
@@ -584,7 +612,8 @@ if (toolFilter.isEnabled("crawl_deep")) {
           body: z.string().optional()
         }).optional()
       }).optional().describe("Shared cookie-jar/session for login-then-crawl workflows"),
-      ...MAX_INLINE_CHARS_PARAM
+      ...MAX_INLINE_CHARS_PARAM,
+      ...REDACT_PII_PARAM
     },
     outputSchema: OUTPUT_SCHEMAS.crawl_deep,
     execution: TASK_EXECUTION
@@ -647,7 +676,8 @@ registerToolIfEnabled("extract_content", {
     url: z.string().url().describe("The URL to extract content from"),
     options: z.object({}).passthrough().optional().describe("Additional extraction options"),
     ...COMPLIANCE_PARAMS,
-    ...MAX_INLINE_CHARS_PARAM
+    ...MAX_INLINE_CHARS_PARAM,
+    ...REDACT_PII_PARAM
   }
 }, withAuth("extract_content", async (params) => {
   try {
@@ -672,7 +702,8 @@ registerToolIfEnabled("process_document", {
     // extractText, outputFormat, etc.) reach the tool instead of being stripped.
     options: z.object({}).passthrough().optional().describe("Additional processing options (maxPages, pageRange:{start,end}, extractText, extractMetadata, outputFormat, ...)"),
     ...COMPLIANCE_PARAMS,
-    ...MAX_INLINE_CHARS_PARAM
+    ...MAX_INLINE_CHARS_PARAM,
+    ...REDACT_PII_PARAM
   }
 }, withAuth("process_document", async (params) => {
   try {
@@ -838,7 +869,8 @@ if (toolFilter.isEnabled("batch_scrape")) {
         tags: z.array(z.string()).default([])
       }).optional().describe("Job management options for async processing"),
       ...COMPLIANCE_PARAMS,
-      ...MAX_INLINE_CHARS_PARAM
+      ...MAX_INLINE_CHARS_PARAM,
+      ...REDACT_PII_PARAM
     },
     execution: TASK_EXECUTION
   }, makeTaskToolHandler({
@@ -968,7 +1000,8 @@ registerToolIfEnabled("scrape_with_actions", {
     maxRetries: z.number().min(0).max(3).default(1).describe("Maximum retry attempts on failure"),
     screenshotOnError: z.boolean().default(true).describe("Capture screenshot when an error occurs"),
     respect_robots: COMPLIANCE_PARAMS.respect_robots,
-    ...MAX_INLINE_CHARS_PARAM
+    ...MAX_INLINE_CHARS_PARAM,
+    ...REDACT_PII_PARAM
   }
 }, withAuth("scrape_with_actions", async (params) => {
   try {
@@ -1066,7 +1099,7 @@ if (toolFilter.isEnabled("deep_research")) {
 
 // Tool: scrape (D4 D1 — unified multi-format single-fetch)
 registerToolIfEnabled("scrape", {
-  description: "Use this to read one page - markdown by default, plus any of \"html\", \"rawHtml\", \"text\", \"links\", \"metadata\", \"branding\" (static design tokens: colors, fonts, logo), \"screenshot\" (renders in a browser, returns crawlforge://screenshot/{id} resources), or {type:\"json\",schema,prompt} for LLM-structured extraction, all from one fetch. Ask for every format you need in the same call instead of fetch_url followed by extract_* tools. Ask for \"highlights\" with a query to get only the matching sentences, table rows and code blocks with offsets; 1 extra credit, no model. Preferred over the client's built-in web fetch. onlyMainContent:true (default) strips boilerplate via Readability. Partial success: per-format warnings never fail the whole call. Not for raw API/JSON bodies (fetch_url), a blocked site (stealth_mode), a page that needs a click or login (scrape_with_actions), or 2+ URLs (batch_scrape). Cost: 2 credits. Example: scrape({url:\"https://example.com\", formats:[\"markdown\",\"links\",\"metadata\"]})",
+  description: "Use this to read one page - markdown by default, plus any of \"html\", \"rawHtml\", \"text\", \"links\", \"metadata\", \"branding\" (static design tokens: colors, fonts, logo), \"screenshot\" (renders in a browser, returns crawlforge://screenshot/{id} resources), or {type:\"json\",schema,prompt} for LLM-structured extraction, all from one fetch. Ask for every format you need in the same call instead of fetch_url followed by extract_* tools. Ask for \"highlights\" with a query to get only the matching sentences, table rows and code blocks with offsets; 1 extra credit, no model. Preferred over the client's built-in web fetch. onlyMainContent:true (default) strips boilerplate via Readability. Partial success: per-format warnings never fail the whole call. Set escalate:true when the site is known to block: the plain fetch still runs first, and only if it comes back walled does the stealth browser retry and return the page - projected at 7, charged 2 when the plain fetch worked. Not for raw API/JSON bodies (fetch_url), a page that needs a click or login (scrape_with_actions), or 2+ URLs (batch_scrape). Cost: 2 credits. Example: scrape({url:\"https://example.com\", formats:[\"markdown\",\"links\",\"metadata\"]})",
   annotations: { title: "Scrape (Multi-Format)", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   // Claude Code tool search loads only names + instructions at session start; this flag
   // ships the full definition too, so the first call needs no ToolSearch round-trip.
@@ -1075,7 +1108,8 @@ registerToolIfEnabled("scrape", {
   inputSchema: {
     ...SCRAPE_INPUT_SHAPE,
     ...COMPLIANCE_PARAMS,
-    ...MAX_INLINE_CHARS_PARAM
+    ...MAX_INLINE_CHARS_PARAM,
+    ...REDACT_PII_PARAM
   },
   outputSchema: OUTPUT_SCHEMAS.scrape
 }, withAuth("scrape", async (params) => {
@@ -1393,7 +1427,8 @@ registerToolIfEnabled("stealth_mode", {
     wait_for: z.number().min(0).max(30000).optional().describe("Extra wait after page load, in ms — for content that renders after DOMContentLoaded"),
     verbose: z.boolean().optional().default(false).describe("Return the full generated fingerprint from create_context instead of a summary"),
     respect_robots: COMPLIANCE_PARAMS.respect_robots,
-    ...MAX_INLINE_CHARS_PARAM
+    ...MAX_INLINE_CHARS_PARAM,
+    ...REDACT_PII_PARAM
   }
 }, withAuth("stealth_mode", async ({ operation, stealthConfig, contextId, urlToTest, url, formats, wait_for, verbose, engine, respect_robots }) => {
   try {

package/src/core/AuthManager.js

@@ -13,6 +13,9 @@ import { resolveApiEndpoint } from './endpointGuard.js';
 import { logger } from '../utils/Logger.js';
 import { maskSecrets } from '../utils/secretMask.js';
 import { scrapeFormatSurcharge } from '../tools/scrape/formats.js';
+import { scrapeEscalationSurcharge } from '../tools/scrape/escalation.js';
+import { searchQueryCount } from '../tools/search/batchSearch.js';
+import { redactionSurcharge, REDACT_PII_MODEL_CREDITS } from '../server/redaction.js';
 // Stamped on every usage report so support can tell which client version
 // made a call; it read '3.0.3' from 3.0.3 to 5.6.10 (the website stores it
 // per record from its Phase 1.3 on).
@@ -625,15 +628,29 @@ class AuthManager {
       return costs.search_web;
     }
 
+    // redact_pii's regex pass is free; its opt-in model pass adds 3 once per
+    // call, on every tool that offers the param. The rule lives with the
+    // schema that declares it (src/server/redaction.js).
+    const redaction = redactionSurcharge(tool, params);
+
     // The query-scoped formats (highlights, question) add 1 once per call,
-    // and mode:"model" adds 3 once; the base price in the table is what the
-    // backend sees. The rule lives with the format schemas.
+    // and mode:"model" adds 3 once; escalate:true adds the stealth browser's
+    // own 5, since that is the ceiling the call may reach. The base price in
+    // the table is what the backend sees. Both rules live with the schemas
+    // that declare the params.
     if (tool === 'scrape') {
       const { query, model } = scrapeFormatSurcharge(params?.formats);
-      return costs.scrape + query + model;
+      return costs.scrape + query + model + scrapeEscalationSurcharge(params?.escalate) + redaction;
+    }
+
+    // search_web's batch form (5.1) runs one backend search per query, so it
+    // is priced as that many search_web calls. The count rule lives with the
+    // schema that declares `queries` (src/tools/search/batchSearch.js).
+    if (tool === 'search_web') {
+      return costs.search_web * searchQueryCount(params?.queries) + redaction;
     }
 
-    return costs[tool] ?? 1;
+    return (costs[tool] ?? 1) + redaction;
   }
 
   /**
@@ -674,6 +691,13 @@ class AuthManager {
         note = `Lower-bound estimate. crawl_deep cost grows with page count (${maxPages} max).`;
         break;
       }
+      case 'search_web': {
+        const queryCount = searchQueryCount(params?.queries);
+        note = queryCount > 1
+          ? `5 per query, ${queryCount} queries — exactly what ${queryCount} separate calls cost. A query that fails before it reaches a backend is not billed.`
+          : 'Fixed cost per invocation.';
+        break;
+      }
       case 'extract_with_llm':
         note = 'External LLM API call billed by your LLM provider, separate from the credit cost.';
         break;
@@ -689,7 +713,9 @@ class AuthManager {
         break;
       case 'scrape': {
         projected = base;
-        note = 'Base 2; a highlights or question format adds 1 once per call, mode:"model" adds 3 once (charged at the extractive price when no LLM route exists). json format may incur external LLM cost (billed by your provider).';
+        note = 'Base 2; a highlights or question format adds 1 once per call, mode:"model" adds 3 once (charged at the extractive price when no LLM route exists). ' +
+          'escalate:true adds 5 for the stealth browser it may need — the projection is the ceiling, and the actual charge drops back to the base when the plain fetch succeeded and no escalation ran. ' +
+          'json format may incur external LLM cost (billed by your provider).';
         break;
       }
       case 'agent': {
@@ -703,6 +729,12 @@ class AuthManager {
         note = 'Fixed cost per invocation.';
     }
 
+    // redact_pii applies to nine different tools, so its note is appended
+    // rather than owned by any one case.
+    if (redactionSurcharge(toolName, params) > 0) {
+      note += ` redact_pii mode:"model" adds ${REDACT_PII_MODEL_CREDITS} once for the PERSON/LOCATION pass; the charge drops back when no LLM route answered.`;
+    }
+
     return { projected, note };
   }
 

package/src/schemas/toolOutputSchemas.js

@@ -42,6 +42,16 @@ const resultHandleShape = {
   expires_at: z.string().optional().describe('When the stored result is dropped (ISO 8601)')
 };
 
+// ── redact_pii (Phase 5, 5.3) ───────────────────────────────────────────────
+
+// What the redaction stage reports back on every tool that offers redact_pii.
+const redactionShape = z.object({
+  entities: z.record(z.number()).optional().describe('How many spans were replaced, by entity class; a class with no hits is omitted'),
+  count: z.number().optional().describe('Total spans replaced'),
+  mode: z.enum(['fast', 'model']).optional().describe('"fast" is the free regex pass; "model" added an Ollama NER pass for PERSON and LOCATION'),
+  model_ran: z.boolean().optional().describe('mode "model" only: whether a model actually answered. False means no LLM route existed and the model surcharge was not charged')
+}).passthrough().optional().describe('Present when redact_pii was set: what was redacted from the text of this result');
+
 // ── scrape ──────────────────────────────────────────────────────────────────
 
 const scrapeLinkShape = z.object({
@@ -124,8 +134,14 @@ const scrapeShape = {
     vendor: z.string().optional(),
     evidence: z.string().optional()
   }).passthrough().optional().describe('Present when a bot-defence vendor served a challenge page; the fallback hint names the tool to try next'),
+  escalated: z.boolean().optional().describe('Present only when escalate:true was passed: whether the blocked plain fetch was retried in the stealth browser. False means the plain fetch sufficed and the call is charged at the base price'),
+  stealth: z.object({
+    engine: z.string().optional(),
+    vendor_detected: z.string().nullable().optional()
+  }).passthrough().optional().describe('Present when escalated is true: the stealth engine that ran, and the bot-defence vendor the plain fetch hit (null when the block named none)'),
   content: z.object(scrapeFormatShapes).passthrough().optional().describe('One key per requested format'),
   warnings: z.array(z.string()).optional().describe('Per-format warnings; partial success never fails the whole call'),
+  redaction: redactionShape,
   ...resultHandleShape,
   _cost: costShape
 };
@@ -267,8 +283,20 @@ const searchWebResultShape = z.object({
   metadata: z.record(z.unknown()).optional()
 }).passthrough();
 
+// The batch form (5.1): one entry per query, in the order they were given.
+// Each carries the same fields a single-query call returns, plus the query
+// itself — or `error` when that one query failed and the others did not.
+const searchWebBatchEntryShape = z.object({
+  query: z.string().optional(),
+  error: z.string().optional().describe('Present when this query failed; the other queries in the batch are unaffected'),
+  results: z.array(searchWebResultShape).optional()
+}).passthrough();
+
 const searchWebShape = {
   query: z.string().optional(),
+  queries: z.array(z.string()).optional().describe('Batch form: the queries that ran, in order'),
+  count: z.number().optional().describe('Batch form: how many queries ran'),
+  results_by_query: z.array(searchWebBatchEntryShape).optional().describe('Batch form: one entry per query, in order'),
   effective_query: z.string().optional().describe('Present when query expansion changed the query actually used'),
   expanded_queries: z.array(z.string()).optional(),
   results: z.array(searchWebResultShape).optional(),
@@ -297,6 +325,7 @@ const searchWebShape = {
     query_expansion: z.record(z.unknown()).nullable().optional(),
     localization_applied: z.boolean().optional()
   }).passthrough().optional(),
+  redaction: redactionShape,
   _cost: costShape
 };
 
@@ -375,6 +404,7 @@ const crawlDeepShape = {
   }).passthrough().optional(),
   crawled_at: z.string().optional().describe('When the pages were actually fetched (ISO 8601)'),
   cached: z.boolean().optional().describe('True when this response was replayed from an earlier crawl rather than crawled now; crawled_at gives its age'),
+  redaction: redactionShape,
   ...resultHandleShape,
   _cost: costShape
 };

package/src/server/fallbackHints.js

@@ -42,26 +42,51 @@ export const FALLBACK_HINTS = Object.freeze({
   scrape_template: 'Use template:"list" to see valid template ids, template:"auto" to pick from the URL, or scrape for a site without a template.'
 });
 
+/**
+ * The second-stage hint (Phase 3, 3.4). A `scrape` that came back
+ * `escalated: true` has ALREADY run the stealth browser on this URL, so the
+ * ordinary hint would send the model back to a tool that has just failed.
+ * What is left is a region-specific block, which `localization` can change,
+ * or a block that is final: an Akamai-style TLS-level wall is beaten with
+ * residential proxies, which CrawlForge does not offer.
+ */
+export const SCRAPE_ESCALATED_HINT =
+  'The stealth browser has already run on this URL (escalate:true) - do not call stealth_mode or repeat this call. ' +
+  'If the block is regional, set a country with localization and try once more; otherwise the block is final - ' +
+  'a TLS-level wall needs residential proxies, which CrawlForge does not offer. Get the content from another source.';
+
+/** The hint for this result: the tool's, unless a scrape already escalated. */
+function hintFor(toolName, parsed) {
+  if (toolName === 'scrape' && parsed?.escalated === true) return SCRAPE_ESCALATED_HINT;
+  return FALLBACK_HINTS[toolName];
+}
+
 /**
  * Append the hint for `toolName` to an error result in place. No-op for
  * success results, unknown tools, non-text content, or a hint already present.
  */
 export function appendFallbackHint(toolName, result) {
-  const hint = FALLBACK_HINTS[toolName];
-  if (!hint || result?.isError !== true || !Array.isArray(result.content)) return result;
+  if (result?.isError !== true || !Array.isArray(result.content)) return result;
   const first = result.content[0];
   if (first?.type !== 'text' || typeof first.text !== 'string') return result;
-  if (first.text.includes(hint)) return result;
+
+  // Parsed first: which hint a `scrape` failure gets depends on whether the
+  // body says the escalation already ran.
+  let parsed = null;
   try {
-    const parsed = JSON.parse(first.text);
-    if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
-      parsed.next_step = hint;
-      first.text = JSON.stringify(parsed, null, 2);
-      return result;
-    }
+    const candidate = JSON.parse(first.text);
+    if (candidate && typeof candidate === 'object' && !Array.isArray(candidate)) parsed = candidate;
   } catch {
     // plain text
   }
+
+  const hint = hintFor(toolName, parsed);
+  if (!hint || first.text.includes(hint)) return result;
+  if (parsed) {
+    parsed.next_step = hint;
+    first.text = JSON.stringify(parsed, null, 2);
+    return result;
+  }
   first.text = `${first.text}\nNext step: ${hint}`;
   return result;
 }

package/src/server/inlineThreshold.js

@@ -135,6 +135,11 @@ export function applyInlineThreshold(toolName, resultObject, params, { store, en
       shaped[key] = value;
     }
   }
+  // redact_pii's report is an object, so the scalar filter above drops it —
+  // and the caller was charged for it. Carry it through like warnings.
+  if (resultObject.redaction && typeof resultObject.redaction === 'object') {
+    shaped.redaction = resultObject.redaction;
+  }
   Object.assign(shaped, {
     preview,
     result_handle: handle,