crawlforge-mcp-server 6.17.1 → 6.18.1

package/README.md

@@ -178,8 +178,8 @@ CrawlForge requires a CrawlForge API key — **every tool is metered and consume
 | Tool | Credits | What it does |
 |------|---------|--------------|
 | `fetch_url` | 1 | Fetch content from any URL |
-| `extract_text` | 1 | Extract clean text from web pages |
-| `extract_links` | 1 | Get all links from a page |
+| `extract_text` | 1 (6 projected with `escalate: true`, charged 1 when the plain fetch works) | Extract clean text from web pages |
+| `extract_links` | 1 (6 projected with `escalate: true`, charged 1 when the plain fetch works) | Get all links from a page |
 | `extract_metadata` | 1 | Extract page metadata (title, OG tags, schema.org) |
 | `scrape_template` | 1 | Structured data from well-known sites (Amazon, GitHub, LinkedIn, YouTube, Reddit, Hacker News, npm, and more) without writing selectors |
 | `list_ollama_models` | 1 | List the Ollama models installed locally (helps you pick a `model` for `extract_with_llm`) |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "6.17.1",
+  "version": "6.18.1",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 31 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -121,7 +121,7 @@
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",
-    "crawlforge-extractors": "^1.12.0",
+    "crawlforge-extractors": "^1.15.0",
     "diff": "^9.0.0",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",

package/server.js

@@ -52,8 +52,9 @@ import { createOAuthProvider } from "./src/server/auth/oauth.js";
 import { createMetricsRegistry } from "./src/observability/metrics.js";
 // Basic tool handlers (extracted from server.js)
 import { fetchUrlHandler } from "./src/tools/basic/fetchUrl.js";
-import { extractTextHandler } from "./src/tools/basic/extractText.js";
-import { extractLinksHandler } from "./src/tools/basic/extractLinks.js";
+import { createExtractTextHandler } from "./src/tools/basic/extractText.js";
+import { createExtractLinksHandler } from "./src/tools/basic/extractLinks.js";
+import { EXTRACT_ESCALATION_SHAPE } from "./src/utils/fetchLadder.js";
 import { extractMetadataHandler } from "./src/tools/basic/extractMetadata.js";
 import { scrapeStructuredHandler } from "./src/tools/basic/scrapeStructured.js";
 import { createExtractEmbeddedStateHandler, EMBEDDED_STATE_INPUT_SHAPE } from "./src/tools/extract/extractEmbeddedState.js";
@@ -112,7 +113,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "6.17.1",
+  version: "6.18.1",
   description: "Production-ready MCP server with 31 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, stateful browser sessions with element refs, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -439,29 +440,37 @@ registerToolIfEnabled("fetch_url", {
 
 // Tool: extract_text
 registerToolIfEnabled("extract_text", {
-  description: "Use this for a page's plain text or markdown with tags, scripts and styles removed - the cheapest read of a static HTML page. Use output_format:\"markdown\" for RAG. Not for article pages (extract_content strips nav and boilerplate), JS-rendered pages (scrape), or when you also want links or metadata (scrape with several formats, one fetch). Cost: 1 credit. Example: extract_text({url: \"https://example.com/article\", output_format:\"markdown\"})",
+  description: "Use this for a page's plain text or markdown with tags, scripts and styles removed - the cheapest read of a static HTML page. Use output_format:\"markdown\" for RAG; selector reads only the matched elements, max_length caps the returned text. Not for article pages (extract_content strips nav and boilerplate), JS-rendered pages (scrape), or when you also want links or metadata (scrape with several formats, one fetch). A PDF or other binary is refused (process_document reads documents); an empty client-rendered shell succeeds with rendered:false and a warning. Set escalate:true when the site is known to block: the plain fetch still runs first, and only if it is walled does the stealth browser re-read the page - projected at 6, charged 1 when the plain fetch worked. Cost: 1 credit. Example: extract_text({url: \"https://example.com/article\", output_format:\"markdown\"})",
   annotations: { title: "Extract Text", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The URL to extract text from"),
     remove_scripts: z.boolean().optional().default(true).describe("Remove script tags before extraction"),
     remove_styles: z.boolean().optional().default(true).describe("Remove style tags before extraction"),
     output_format: z.enum(["text", "markdown"]).optional().default("text").describe("Output format: \"text\" (default) or \"markdown\" — use markdown for RAG workflows"),
+    selector: z.string().optional().describe("CSS selector: read only the matched elements (nav/header/footer are then kept). No match is an error"),
+    max_length: z.number().int().min(1).max(1000000).optional().describe("Maximum characters of text or markdown to return; a longer result is cut and ends with \"...\""),
+    ...EXTRACT_ESCALATION_SHAPE,
     ...COMPLIANCE_PARAMS,
     ...REDACT_PII_PARAM
   }
-}, withAuth("extract_text", extractTextHandler));
+}, withAuth("extract_text", createExtractTextHandler({
+  escalateFetch: (args) => stealthEscalation({ ...args, tool: 'extract_text' })
+})));
 
 // Tool: extract_links
 registerToolIfEnabled("extract_links", {
-  description: "Use this to list the hyperlinks on one page - a crawl seed list, a broken-link audit, related resources. filter_external:true returns only outbound links. Not for a whole site (map_site), and not alongside a scrape of the same URL: scrape formats:[\"markdown\",\"links\"] returns both in one fetch. Cost: 1 credit. Example: extract_links({url: \"https://example.com\", filter_external: true})",
+  description: "Use this to list the hyperlinks on one page - a crawl seed list, a broken-link audit, related resources. filter_external:true drops internal links; mailto:, tel: and javascript: links are type \"other\". Not for a whole site (map_site), and not alongside a scrape of the same URL: scrape formats:[\"markdown\",\"links\"] returns both in one fetch. A PDF or other binary is refused (process_document reads documents); an empty client-rendered shell succeeds with rendered:false and a warning. Set escalate:true when the site is known to block: the plain fetch still runs first, and only if it is walled does the stealth browser re-read the page - projected at 6, charged 1 when the plain fetch worked. Cost: 1 credit. Example: extract_links({url: \"https://example.com\", filter_external: true})",
   annotations: { title: "Extract Links", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The URL to extract links from"),
-    filter_external: z.boolean().optional().default(false).describe("Only return external links"),
+    filter_external: z.boolean().optional().default(false).describe("Drop internal (same-host) links"),
     base_url: z.string().url().optional().describe("Base URL for resolving relative links"),
+    ...EXTRACT_ESCALATION_SHAPE,
     ...COMPLIANCE_PARAMS
   }
-}, withAuth("extract_links", extractLinksHandler));
+}, withAuth("extract_links", createExtractLinksHandler({
+  escalateFetch: (args) => stealthEscalation({ ...args, tool: 'extract_links' })
+})));
 
 // Tool: extract_metadata
 registerToolIfEnabled("extract_metadata", {
@@ -1022,7 +1031,7 @@ registerToolIfEnabled("scrape_with_actions", {
       fields: z.array(z.object({
         selector: z.string(),
         value: z.string(),
-        type: z.enum(['text', 'select', 'checkbox', 'radio', 'file']).default('text'),
+        type: z.enum(['text', 'select', 'checkbox', 'radio', 'file']).default('text').describe("How the field is filled: text types the value, select picks the option by value or label, checkbox/radio check the input (a selector matching a group checks the member whose value attribute equals value; a checkbox with value \"false\" is unchecked). file is not supported and fails that action"),
         waitAfter: z.number().min(0).max(5000).default(100)
       })),
       submitSelector: z.string().optional(),

package/src/core/ActionExecutor.js

@@ -129,6 +129,17 @@ const SelectActionSchema = BaseActionSchema.extend({
   message: 'Select action requires value or values'
 });
 
+// Not on any tool's public action list: scrape_with_actions' formAutoFill emits
+// it for a field declared `type: "checkbox"` or `type: "radio"`. `fieldType`
+// is a plain string so that a field type nothing here can fill ("file")
+// arrives as an action that fails, instead of being typed into as text.
+const CheckActionSchema = BaseActionSchema.extend({
+  type: z.literal('check'),
+  selector: z.string(),
+  value: z.string(),
+  fieldType: z.string()
+});
+
 const HoverActionSchema = BaseActionSchema.extend({
   type: z.literal('hover'),
   selector: z.string(),
@@ -175,6 +186,7 @@ const ActionSchema = z.union([
   PressActionSchema,
   ScrollActionSchema,
   SelectActionSchema,
+  CheckActionSchema,
   HoverActionSchema,
   NavigateActionSchema,
   ScreenshotActionSchema,
@@ -1029,6 +1041,8 @@ export class ActionExecutor extends EventEmitter {
         return await this.executeScrollAction(page, action);
       case 'select':
         return await this.executeSelectAction(page, action);
+      case 'check':
+        return await this.executeCheckAction(page, action);
       case 'hover':
         return await this.executeHoverAction(page, action);
       case 'navigate':
@@ -1309,6 +1323,45 @@ export class ActionExecutor extends EventEmitter {
     };
   }
 
+  /**
+   * Execute check action — a formAutoFill checkbox or radio field.
+   *
+   * A selector that names a whole group (`input[name="size"]`) is narrowed to
+   * the member whose `value` attribute is the field's value, the way the form
+   * itself pairs name and value; `.first()` would check whichever came first.
+   * A selector that names one input checks that input, except a checkbox with
+   * value "false", which is unchecked.
+   * @param {Page} page - Playwright page
+   * @param {Object} action - Check action
+   * @returns {Promise<Object>} Check result
+   */
+  async executeCheckAction(page, action) {
+    if (action.fieldType !== 'checkbox' && action.fieldType !== 'radio') {
+      throw new Error(
+        `formAutoFill field type "${action.fieldType}" is not supported ` +
+        '(supported: text, select, checkbox, radio)'
+      );
+    }
+
+    const timeout = this.actionTimeout(action);
+    const matches = page.locator(this.resolveSelector(page, action.selector));
+    await matches.first().waitFor({ state: 'attached', timeout });
+
+    const isGroup = (await matches.count()) > 1;
+    const target = isGroup
+      ? matches.and(page.locator(`[value=${JSON.stringify(action.value)}]`)).first()
+      : matches.first();
+    const checked = isGroup || action.fieldType === 'radio' || action.value !== 'false';
+
+    await target.setChecked(checked, { timeout });
+
+    return {
+      selector: action.selector,
+      value: action.value,
+      checked
+    };
+  }
+
   /**
    * Execute hover action
    * @param {Page} page - Playwright page
@@ -1581,9 +1634,12 @@ export class ActionExecutor extends EventEmitter {
     // hostname-based checks alone would miss DNS-rebinding/private-IP targets.
     await assertUrlAllowed(url, { resolveDns: true });
 
+    // The initial load and a retry's reload pass no timeout of their own, so
+    // browserOptions.timeout bounds them as it bounds the actions (0.2); a
+    // navigate action's own timeout comes first.
     const response = await page.goto(url, {
       waitUntil: options.waitUntil || 'domcontentloaded',
-      timeout: options.timeout || 30000
+      timeout: options.timeout || options.browserOptions?.timeout || 30000
     });
 
     // Re-validate where the navigation went: a redirect could have taken us

package/src/core/AgentOrchestrator.js

@@ -172,6 +172,40 @@ export function isCurrentStateTask(prompt) {
   return CURRENT_STATE_RE.test(prompt || '');
 }
 
+/**
+ * The names a prompt uses: words carrying a capital letter that is not just
+ * the capital opening a sentence ("TypeScript", "Hacker", "News", "NASA"; not
+ * "What"). These are what the task is about, so a planned query that drops
+ * one has lost its subject.
+ */
+export function namedTerms(prompt) {
+  const words = (prompt || '').split(/\s+/);
+  const terms = [];
+  words.forEach((raw, i) => {
+    if (raw.includes('://')) return;
+    const word = raw.replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, '').replace(/['’]s$/, '');
+    if (word.length < 2) return;
+    const opensSentence = i === 0 || /[.?!:]$/.test(words[i - 1]);
+    if (!/\p{Lu}/u.test(opensSentence ? word.slice(1) : word)) return;
+    if (!terms.some(t => t.toLowerCase() === word.toLowerCase())) terms.push(word);
+  });
+  return terms;
+}
+
+/**
+ * A task scoped to its seed URLs: the caller passed `urls` and the prompt
+ * points at them ("this page", "these URLs", "the given site"). The answer is
+ * then on those pages and nowhere else, so GATHER runs no web search — search
+ * results for the task's words are other people's pages. Asked how many books
+ * "this page" lists, the agent searched "count books on webpage" and merged
+ * Reddit threads about word counts into the evidence (R24, 2026-10-03).
+ */
+const SEED_SCOPE_RE = /\b(?:this|these|the (?:given|provided|above|following|seed))\s+(?:web\s?)?(?:pages?|urls?|sites?|websites?|links?|documents?|articles?)\b/i;
+
+export function isSeedScopedTask(prompt, seedUrls) {
+  return Array.isArray(seedUrls) && seedUrls.length > 0 && SEED_SCOPE_RE.test(prompt || '');
+}
+
 // ── Orchestrator ──────────────────────────────────────────────────────────────
 
 export class AgentOrchestrator {
@@ -358,6 +392,17 @@ export class AgentOrchestrator {
       );
     }
     // A current-state plan is told to make its first query the bare entity
+    // name, and for a prompt naming two things a small model can keep the
+    // wrong one: "latest stable version of the TypeScript npm package" was
+    // searched as the single word "npm" (R24, 2026-10-03). The first query
+    // carries every name the prompt uses; what the planner wrote stays as the
+    // qualifier ("TypeScript npm").
+    if (currentState) {
+      const first = searchQueries[0].toLowerCase();
+      const dropped = namedTerms(prompt).filter(t => !first.includes(t.toLowerCase()));
+      if (dropped.length > 0) searchQueries[0] = `${dropped.join(' ')} ${searchQueries[0]}`;
+    }
+    // A current-state plan is told to make its first query the bare entity
     // name, and a small model often stops there. The bare query surfaces the
     // live front page, which rarely states the fact asked for: "what does
     // Southwest charge for a first checked bag" fetched southwest.com's home,
@@ -397,7 +442,7 @@ export class AgentOrchestrator {
     /** url -> { title, text }: the engine's excerpt, evidence of last resort for a page that cannot be fetched. */
     const excerpts = new Map();
 
-    if (urlQueue.length < capUrls) {
+    if (urlQueue.length < capUrls && !isSeedScopedTask(prompt, seedUrls)) {
       try {
         const searchTool = await this._getSearchTool();
         for (const q of searchQueries) {
@@ -607,14 +652,24 @@ export class AgentOrchestrator {
           schema,
           provider: 'auto'
         });
+        // A required field left null is an incomplete answer, not a
+        // structured success (R24, 2026-10-03).
+        const missingRequired = result.success && Array.isArray(schema.required)
+          ? schema.required.filter(key => result.data?.[key] == null || result.data[key] === '')
+          : [];
+        const missingReason = missingRequired.length > 0
+          ? `Required ${missingRequired.length === 1 ? 'field' : 'fields'} ${missingRequired.map(k => `"${k}"`).join(', ')} ` +
+            `${missingRequired.length === 1 ? 'is' : 'are'} null: the fetched sources do not state ${missingRequired.length === 1 ? 'it' : 'them'}.`
+          : undefined;
+        if (missingReason) warnings.push(missingReason);
         return {
           success: result.success,
           answer: result.success ? result.data : null,
           structured: true,
           search_results: searchResults,
           evidence: evidence.map(e => (e.via ? { url: e.url, via: e.via } : { url: e.url })),
-          degraded: !result.success,
-          reason: result.success ? undefined : result.error,
+          degraded: !result.success || missingRequired.length > 0,
+          reason: result.success ? missingReason : result.error,
           steps: step,
           urls_fetched: urlsFetched,
           stealth_retries: usage.escalations,

package/src/core/AuthManager.js

@@ -687,6 +687,12 @@ class AuthManager {
       return costs.extract_embedded_state + scrapeEscalationSurcharge(params?.escalate) + redaction;
     }
 
+    // extract_text and extract_links (fix plan Phase E2) run the same ladder:
+    // projected 1+5 when escalating, charged 1 when the plain fetch sufficed.
+    if (tool === 'extract_text' || tool === 'extract_links') {
+      return costs[tool] + scrapeEscalationSurcharge(params?.escalate) + redaction;
+    }
+
     // search_web's batch form (5.1) runs one backend search per query, so it
     // is priced as that many search_web calls. The count rule lives with the
     // schema that declares `queries` (src/tools/search/batchSearch.js).
@@ -778,6 +784,12 @@ class AuthManager {
           ? 'Base 2; escalate:true adds 5 for the stealth browser it may need - the projection is the ceiling, and the actual charge drops back to 2 when the plain fetch succeeded and no escalation ran.'
           : 'Fixed cost per invocation.';
         break;
+      case 'extract_text':
+      case 'extract_links':
+        note = params?.escalate === true
+          ? 'Base 1; escalate:true adds 5 for the stealth browser it may need - the projection is the ceiling, and the actual charge drops back to 1 when the plain fetch succeeded and no escalation ran.'
+          : 'Fixed cost per invocation.';
+        break;
       case 'agent': {
         const agentUrls = params?.maxUrls || 10;
         const isPro = params?.model === 'pro';

package/src/core/crawlers/BFSCrawler.js

@@ -219,6 +219,12 @@ export class BFSCrawler {
       return;
     }
     this.visited.set(normalizedUrl, url);
+    // Budget spent: drop what is still waiting. Each of those tasks would only
+    // return at the cap check above, but the queue releases 10 a second, so a
+    // seed with 2,290 links held a max_pages:3 crawl open for 232 s (R24).
+    if (this.visited.size >= this.maxPages) {
+      this.queue.clear();
+    }
 
     try {
       // Check cache first
@@ -318,6 +324,7 @@ export class BFSCrawler {
         url,
         depth,
         error: error.message,
+        ...(error.code ? { code: error.code } : {}),
         timestamp: new Date().toISOString()
       });
     }

package/src/core/queue/QueueManager.js

@@ -77,6 +77,9 @@ export class QueueManager {
   getStats() {
     return {
       ...this.stats,
+      // Read live: the 'active' event fires when a task starts, never when the
+      // last one ends, so the stored value stayed at 8 on a finished crawl.
+      active: this.queue.pending,
       size: this.queue.size,
       pending: this.queue.pending,
       isPaused: this.queue.isPaused

package/src/schemas/toolOutputSchemas.js

@@ -181,6 +181,7 @@ const mapSiteShape = {
     url: z.string().optional(),
     score: z.number().optional()
   }).passthrough()).optional().describe('Present only when the `search` param was set'),
+  warnings: z.array(z.string()).optional().describe('Present when the map is partial, e.g. the start page could not be read and the URLs come from the sitemap only'),
   _cost: costShape
 };
 
@@ -378,16 +379,17 @@ const crawlDeepShape = {
   error: z.string().optional(),
   url: z.string().optional(),
   crawl_depth: z.number().optional(),
-  pages_crawled: z.number().optional(),
-  pages_found: z.number().optional(),
-  error_count: z.number().optional(),
+  pages_attempted: z.number().optional().describe('URLs the crawl tried: pages_crawled + error_count'),
+  pages_crawled: z.number().optional().describe('Pages fetched and returned in results'),
+  pages_found: z.number().optional().describe('Same as pages_crawled; kept for existing clients'),
+  error_count: z.number().optional().describe('URLs that failed; one entry each in errors[]'),
   duration_ms: z.number().optional(),
   pages_per_second: z.number().optional(),
   results: z.array(crawlDeepPageShape).optional(),
   errors: z.array(z.unknown()).optional(),
   stats: z.unknown().optional(),
   site_structure: z.object({
-    total_pages: z.number().optional(),
+    total_pages: z.number().optional().describe('Equals pages_crawled; failed URLs are not counted'),
     depth_distribution: z.record(z.number()).optional()
       .describe('Pages per crawl depth (links from the start URL)'),
     path_depth_distribution: z.record(z.number()).optional()

package/src/server/fallbackHints.js

@@ -11,8 +11,8 @@
 
 export const FALLBACK_HINTS = Object.freeze({
   fetch_url: 'For a JS-rendered page or an empty shell use scrape; after a 403/429/CAPTCHA/challenge page use stealth_mode operation:"scrape". Do not repeat the same fetch_url call.',
-  extract_text: 'Use scrape formats:["markdown"] (renders more pages); after a 403/429/challenge page use stealth_mode operation:"scrape".',
-  extract_links: 'Use scrape formats:["links"]; for a whole site use map_site.',
+  extract_text: 'After a 403/429/444/challenge page call extract_text again with escalate:true (projected 6, charged 1 if the plain fetch gets through); for a JS-rendered page use scrape formats:["markdown"]; for a PDF use process_document.',
+  extract_links: 'After a 403/429/444/challenge page call extract_links again with escalate:true (projected 6, charged 1 if the plain fetch gets through); for a JS-rendered page use scrape formats:["links"]; for a whole site use map_site.',
   extract_metadata: 'Use scrape formats:["metadata"]; after a 403/429/challenge page use stealth_mode operation:"scrape".',
   extract_content: 'Use scrape formats:["markdown"] (same clean output, renders more pages); after a 403/429/challenge page use stealth_mode operation:"scrape".',
   extract_embedded_state: 'Call again without `path` to see the top-level keys, or use scrape formats:["markdown"] if the page has no framework payload.',
@@ -56,8 +56,8 @@ export const SCRAPE_ESCALATED_HINT =
   'If the block is regional, set a country with localization and try once more; otherwise the block is final - ' +
   'a TLS-level wall needs residential proxies, which CrawlForge does not offer. Get the content from another source.';
 
-/** The tools whose escalate:true runs the stealth stage itself (Phase 3; extract_embedded_state since plan Phase 3.2). */
-const ESCALATING_TOOLS = new Set(['scrape', 'extract_embedded_state']);
+/** The tools whose escalate:true runs the stealth stage itself (Phase 3; extract_embedded_state since plan Phase 3.2; extract_text and extract_links since Phase E2). */
+const ESCALATING_TOOLS = new Set(['scrape', 'extract_embedded_state', 'extract_text', 'extract_links']);
 
 /** The hint for this result: the tool's, unless the call already escalated. */
 function hintFor(toolName, parsed) {

package/src/server/inlineThreshold.js

@@ -20,8 +20,9 @@ export const MAX_INLINE_CHARS_PARAM = {
 /**
  * Per-tool rule. `textPaths` are dotted paths tried in order; the first one
  * holding a string becomes the text view the preview and read_result work
- * on. An empty list means the pretty-printed JSON is the view. `when` gates
- * on params. extract_embedded_state has a shape of its own (shapeEmbeddedState
+ * on. An empty list means the pretty-printed JSON is the view. `keep` names
+ * top-level fields carried inline past the scalar filter (keepNamedFields).
+ * `when` gates on params. extract_embedded_state has a shape of its own (shapeEmbeddedState
  * below).
  */
 /** The browser_session operations that hand back content worth shaping. */
@@ -37,7 +38,14 @@ export const INLINE_THRESHOLD_TOOLS = Object.freeze({
   // an async job's page came back as 111 KB whole (R20, 2026-09-07).
   get_batch_results: { textPaths: [], truncate: true },
   stealth_mode: { textPaths: ['content.markdown', 'content.text', 'content.html'], truncate: true, when: (params) => params?.operation === 'scrape' },
-  scrape_with_actions: { textPaths: ['content.markdown', 'content.text', 'content.html'], truncate: true },
+  // `keep` names the top-level reports that stay inline past the scalar
+  // filter (keepNamedFields): a walled Guardian chain over 40,000 chars lost
+  // `blocked`, `navigations` and every action's outcome (2026-10-03).
+  scrape_with_actions: {
+    textPaths: ['content.markdown', 'content.text', 'content.html'],
+    truncate: true,
+    keep: ['error', 'blocked', 'consent', 'navigations', 'actionResults']
+  },
   // `read` hands back the same content shape scrape_with_actions does, and was
   // the one content-returning tool with no cap: a read of the World War II
   // article returned 541,308 characters inline where scrape returned 42,259
@@ -48,6 +56,7 @@ export const INLINE_THRESHOLD_TOOLS = Object.freeze({
   browser_session: {
     textPaths: ['content.markdown', 'content.text', 'content.html', 'snapshot.tree'],
     truncate: true,
+    keep: ['error', 'blocked', 'actionResults'],
     when: (params) => BROWSER_SESSION_CONTENT_OPERATIONS.has(params?.operation)
   },
   process_document: { textPaths: ['content.text'], truncate: true },
@@ -111,6 +120,42 @@ export function keepSmallContentFields(content, textPaths = [], maxInline = DEFA
   return Object.keys(kept).length > 0 ? kept : null;
 }
 
+/**
+ * The top-level fields a tool names in `keep`, each carried inline when its
+ * JSON fits a quarter of the inline budget. An actionResults list over that
+ * keeps every entry without its `result` (a snapshot tree, an extracted
+ * value), so which action ran and whether it worked stays inline.
+ * @returns {{ kept: object|null, partial: string[], dropped: string[] }}
+ */
+export function keepNamedFields(resultObject, keys = [], maxInline = DEFAULT_MAX_INLINE_CHARS) {
+  const cap = Math.max(1000, Math.floor(maxInline / 4));
+  const kept = {};
+  const partial = [];
+  const dropped = [];
+  for (const key of keys) {
+    const value = resultObject[key];
+    if (value === undefined) continue;
+    if (JSON.stringify(value).length <= cap) {
+      kept[key] = value;
+      continue;
+    }
+    if (key === 'actionResults' && Array.isArray(value)) {
+      const slim = value.map((entry) => {
+        if (!entry || typeof entry !== 'object') return entry;
+        const { result: _result, ...rest } = entry;
+        return rest;
+      });
+      if (JSON.stringify(slim).length <= cap) {
+        kept[key] = slim;
+        partial.push(key);
+        continue;
+      }
+    }
+    dropped.push(key);
+  }
+  return { kept: Object.keys(kept).length > 0 ? kept : null, partial, dropped };
+}
+
 function warningsOf(resultObject) {
   return Array.isArray(resultObject.warnings) ? resultObject.warnings.filter((w) => typeof w === 'string') : [];
 }
@@ -156,7 +201,16 @@ export function applyInlineThreshold(toolName, resultObject, params, { store, en
 
   const preview = text.slice(0, maxInline);
   const keptFields = keepSmallContentFields(resultObject.content, config.textPaths, maxInline);
-  const keptDesc = keptFields ? `; content.${Object.keys(keptFields).join(', content.')} kept inline` : '';
+  const named = keepNamedFields(resultObject, config.keep, maxInline);
+  const keptNames = [
+    ...(named.kept ? Object.keys(named.kept).filter((key) => !named.partial.includes(key)) : []),
+    ...(keptFields ? Object.keys(keptFields).map((key) => `content.${key}`) : [])
+  ];
+  const keptDesc = [
+    keptNames.length > 0 ? `; ${keptNames.join(', ')} kept inline` : '',
+    named.partial.includes('actionResults') ? '; actionResults kept without each action\'s result (json_path "actionResults[<i>].result" reads one)' : '',
+    named.dropped.length > 0 ? `; ${named.dropped.join(', ')} too large to keep inline (json_path "${named.dropped[0]}" reads it)` : ''
+  ].join('');
   const hint = `Result is ${json.length} chars as JSON, over the inline limit of ${maxInline}; preview holds the first ${preview.length} chars of ${viewDesc} (${text.length} chars in total)${keptDesc} and the full result is kept for 1 hour under result_handle ${handle}: ${readWith}.`;
 
   const shaped = {};
@@ -171,6 +225,7 @@ export function applyInlineThreshold(toolName, resultObject, params, { store, en
   if (resultObject.redaction && typeof resultObject.redaction === 'object') {
     shaped.redaction = resultObject.redaction;
   }
+  if (named.kept) Object.assign(shaped, named.kept);
   // The query-scoped formats live beside the page text under `content`
   // (highlights, answer, json, metadata, links). They are the small, exact
   // answer the caller paid for, and truncating the markdown must not drop
@@ -191,6 +246,14 @@ export function applyInlineThreshold(toolName, resultObject, params, { store, en
   return { result: shaped, stored: true };
 }
 
+/**
+ * Room for the `_cost` block withAuth adds to the text after shaping:
+ * pretty-printed, projected/actual/remaining_credits and the projection
+ * note, whose longest form for extract_embedded_state (escalate:true) is
+ * about 200 chars, about 340 in all.
+ */
+export const COST_BLOCK_RESERVE = 400;
+
 /** Object.keys(data) for a plain object, "array(<n>)" for an array, null otherwise. */
 export function embeddedDataKeys(data) {
   if (Array.isArray(data)) return `array(${data.length})`;
@@ -227,10 +290,12 @@ export function wholeLinePreview(text, budget) {
  * everything that says what the page carries — url, path, bytes, the whole
  * `found` list, data_keys, the escalation report and window_state's list —
  * then a preview of the pretty-printed JSON cut at a line boundary. The
- * preview gets whatever max_inline_chars leaves after the rest, measured as
- * compact JSON, so the whole inline result stays within the limit whenever
- * the rest fits. The full result is under the handle for read_result
- * json_path. The REST route shapes it the same way; keep the two in step.
+ * preview gets whatever max_inline_chars leaves after the rest, measured the
+ * way withAuth sends it — pretty-printed, with COST_BLOCK_RESERVE held back
+ * for the `_cost` block it adds after shaping — so the text the client
+ * receives stays within the limit whenever the rest fits. The full result is
+ * under the handle for read_result json_path. The REST route shapes it the
+ * same way, measured as the compact JSON it sends; keep the two in step.
  */
 function shapeEmbeddedState(resultObject, { json, text, maxInline, handle, expires_at, readWith }) {
   const dataKeys = embeddedDataKeys(resultObject.data);
@@ -257,8 +322,9 @@ function shapeEmbeddedState(resultObject, { json, text, maxInline, handle, expir
     expires_at,
     warnings: [...warningsOf(resultObject), hint]
   };
-  // `"preview":"",` is what the key itself adds once the value is in.
-  const envelope = JSON.stringify({ ...shaped, preview: '', ...tail }).length;
+  // The preview's value is one line of the pretty-printed text and escapes
+  // the same way there as in compact JSON; `""` is its empty value's quotes.
+  const envelope = JSON.stringify({ ...shaped, preview: '', ...tail }, null, 2).length + COST_BLOCK_RESERVE;
   const preview = wholeLinePreview(text, Math.max(0, maxInline - envelope + 2));
   return { ...shaped, preview, ...tail };
 }

package/src/skills/agent-skills/crawlforge-batch-automation/references/actions.md

@@ -154,7 +154,7 @@ main frame only — elements inside iframes and shadow DOM get no refs.
 | `formats` | `["json"]` | `markdown`, `html`, `json`, `text`, `screenshots`. |
 | `captureIntermediateStates` | `false` | Snapshot after each action. |
 | `captureScreenshots` | `true` | Screenshot during execution. |
-| `formAutoFill` | — | Declarative form fill: `fields[]` + `submitSelector`. |
+| `formAutoFill` | — | Declarative form fill: `fields[]` + `submitSelector`. Each field's `type` picks how it is filled: `text` (default) types, `select` picks the option by value or label, `checkbox`/`radio` check the input (a group selector such as `input[name=size]` checks the member whose value attribute equals `value`). `file` is not supported and fails that action. |
 | `browserOptions` | — | `headless`, `userAgent`, `viewportWidth/Height`, `timeout`. |
 | `continueOnActionError` | `false` | Keep going if one action fails. |
 | `maxRetries` | `0` | 0–3 whole-chain retries on failure. A retry re-navigates to the starting URL and replays every action; each run is reported under `attempts[]`. |

package/src/skills/agent-skills/crawlforge-getting-started/references/credits.md

@@ -8,8 +8,8 @@ metered; there is no free tier. Tools marked "scales" cost more as work grows.
 | Tool | Notes |
 |------|-------|
 | `fetch_url` | Raw HTTP fetch. |
-| `extract_text` | Tag-stripped text/markdown. |
-| `extract_links` | All links on a page. |
+| `extract_text` | Tag-stripped text/markdown. `escalate:true` adds 5 as the projected ceiling for the stealth re-read; the actual charge drops back to 1 when the plain fetch succeeded and no escalation ran. |
+| `extract_links` | All links on a page. `escalate:true` adds 5 as the projected ceiling for the stealth re-read; the actual charge drops back to 1 when the plain fetch succeeded and no escalation ran. |
 | `extract_metadata` | Title / meta / OG / schema.org. |
 | `scrape_template` | Pre-built site extractor. |
 | `list_ollama_models` | List local LLMs. |

package/src/skills/agent-skills/crawlforge-web-scraping/SKILL.md

@@ -158,6 +158,10 @@ Returns headers + body. Good for JSON/XML APIs or as a first step before
 `extract_text` (cost 1) = faster, strips tags; pass `output_format:"markdown"`
 for RAG-friendly output.
 
+`extract_text` and `extract_links` take `escalate:true` for a site known to
+block: the plain fetch runs first and the stealth browser re-reads the page
+only if it is walled (projected 6, charged 1 when the plain fetch worked).
+
 ## extract_links / extract_metadata (cost: 1 each)
 
 ```json

package/src/skills/agent-skills/crawlforge-web-scraping/references/tool-reference.md

@@ -34,14 +34,28 @@ your provider).
 | `remove_scripts` | boolean | `true` | Strip `script` tags. |
 | `remove_styles` | boolean | `true` | Strip `style` tags. |
 | `output_format` | enum | `text` | `text` or `markdown` (use markdown for RAG). |
+| `selector` | string | — | CSS selector: read only the matched elements (nav/header/footer inside them are kept). No match is an error: `No elements found for selector: <sel>`. |
+| `max_length` | integer | — | 1–1,000,000. Text or markdown longer than this is cut and ends with `...`; `word_count`/`char_count` count the cut result. |
+| `escalate` | boolean | `false` | When the plain fetch is blocked (403/429/444/challenge page/empty shell), re-read the page once in the stealth browser (impit first under `auto`). A 404 or 5xx never escalates. Projected 6, charged 1 when the plain fetch worked. |
+| `escalate_engine` | enum | `auto` | `auto`, `playwright` (Chromium) or `camoufox`. |
+
+Text mode returns one line per block element (`<h1>Hi</h1><p>there</p>` reads `Hi\nthere`). The REST route (`/api/v1/tools/extract_text`) takes the same parameters plus `preserve_links` and `preserve_formatting`, which the MCP tool does not.
+
+A PDF or other binary body fails with `UNSUPPORTED_CONTENT_TYPE` (use `process_document`). An empty client-rendered shell succeeds with `rendered:false` and the warning `page renders client-side; use scrape`.
 
 ## extract_links (cost: 1)
 
 | Param | Type | Default | Notes |
 |-------|------|---------|-------|
 | `url` | string (URL) | — | Required. |
-| `filter_external` | boolean | `false` | Only return outbound links. |
-| `base_url` | string (URL) | — | Resolve relative links against this. |
+| `filter_external` | boolean | `false` | Drop internal (same-host) links; external and `other` links stay. |
+| `base_url` | string (URL) | — | Resolve relative links against this. Without it, the page's `<base href>`, else the page URL. |
+| `escalate` | boolean | `false` | When the plain fetch is blocked (403/429/444/challenge page/empty shell), re-read the page once in the stealth browser (impit first under `auto`). A 404 or 5xx never escalates. Projected 6, charged 1 when the plain fetch worked. |
+| `escalate_engine` | enum | `auto` | `auto`, `playwright` (Chromium) or `camoufox`. |
+
+Each link is `{ href, text, type, domain, rel, original_href }` (plus `title` when the link has one). `type` is `internal` (same host as the page), `external` or `other` (`mailto:`, `tel:`, `javascript:` and other non-web schemes; only `mailto:`/`tel:` keep an `href`, the rest are `null`). `#fragment`-only links are skipped, and links are deduplicated on the URL without its fragment or trailing slash. Counts: `total_count`, `internal_count`, `external_count`, `other_count`. The REST route returns the same records.
+
+A PDF or other binary body fails with `UNSUPPORTED_CONTENT_TYPE` (use `process_document`). An empty client-rendered shell succeeds with `rendered:false` and the warning `page renders client-side; use scrape`.
 
 ## extract_metadata (cost: 1)