crawlforge-mcp-server 6.6.1 → 6.6.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "6.6.1",
+  "version": "6.6.2",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 30 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",

package/server.js

@@ -61,7 +61,7 @@ import { REDACT_PII_PARAM } from "./src/server/redaction.js"; // Phase 5 (5.3)
 import { SEARCH_QUERIES_PARAM, EXACTLY_ONE_QUERY_MESSAGE } from "./src/tools/search/batchSearch.js"; // Phase 5 (5.1)
 import { markPreflightRefusal } from "./src/server/requestContext.js";
 // D1.1 Resources + D1.2 Prompts + D1.4 Elicitation
-import { ResourceRegistry } from "./src/resources/ResourceRegistry.js";
+import { ResourceRegistry, MAX_RESOURCE_BLOB_BYTES } from "./src/resources/ResourceRegistry.js";
 import { PROMPTS, getPromptMessages } from "./src/prompts/PromptRegistry.js";
 import { ElicitationHelper } from "./src/core/ElicitationHelper.js";
 // Phase 6: MCP-spec adoption — structured output, tool filtering, spec hygiene
@@ -108,7 +108,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "6.6.1",
+  version: "6.6.2",
   description: "Production-ready MCP server with 31 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, stateful browser sessions with element refs, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -1063,9 +1063,24 @@ registerToolIfEnabled("browser_session", {
     // is megabytes beside a few lines of JSON (R21, 2026-09-09).
     const publish = (shot) => {
       if (!shot?.actionId || !shot?.data) return shot;
-      resourceRegistry.storeScreenshot(shot.actionId, shot.data);
+      const { bytes, withinInlineBudget } = resourceRegistry.storeScreenshot(shot.actionId, shot.data);
       const { data, ...rest } = shot;
-      return { ...rest, resourceUri: `crawlforge://screenshot/${shot.actionId}` };
+      return {
+        ...rest,
+        resourceUri: `crawlforge://screenshot/${shot.actionId}`,
+        bytes,
+        // full_page is a knob this tool hands the caller, and on a long page it
+        // produces an image no MCP message can carry. Saying so here costs one
+        // field; learning it from the read costs a wasted call (R23).
+        ...(withinInlineBudget
+          ? {}
+          : {
+              warning: `This image is ${bytes} bytes, too large to read back over MCP ` +
+                `(limit ${MAX_RESOURCE_BLOB_BYTES} bytes) — reading the resource will be refused. ` +
+                `Take it again without full_page, or as format:"jpeg" with a lower quality, or ` +
+                `scoped to one element with selector.`
+            })
+      };
     };
     if (result.screenshot) result.screenshot = publish(result.screenshot);
     if (Array.isArray(result.screenshots)) result.screenshots = result.screenshots.map(publish);

package/src/core/ActionExecutor.js

@@ -1184,12 +1184,13 @@ export class ActionExecutor extends EventEmitter {
     const timeout = this.actionTimeout(action);
 
     await assertUrlAllowed(action.url, { resolveDns: true });
-    await this.assertRobotsAllowed(action.url, executionContext?.browserOptions);
+    const gateWarnings = await this.assertRobotsAllowed(action.url, executionContext?.browserOptions);
 
     await this.navigateToUrl(page, action.url, {
       waitUntil: action.waitUntil,
       timeout
     });
+    page.__crawlforgeGateWarnings = gateWarnings;
 
     return {
       url: action.url,
@@ -1380,9 +1381,13 @@ export class ActionExecutor extends EventEmitter {
    * @throws {BlockedHostError|RobotsDisallowedError}
    */
   async assertRobotsAllowed(url, browserOptions = {}) {
-    await browserPreflight(url, {
+    return await browserPreflight(url, {
       respectRobots: browserOptions?.respectRobots,
-      tool: 'scrape_with_actions'
+      // The audit row is the record of the CUSTOMER's decision (G5), so it has
+      // to name the tool that actually made it. browser_session borrows this
+      // executor, and until R23 every session's override was filed against
+      // scrape_with_actions.
+      tool: browserOptions?.tool || 'scrape_with_actions'
     });
   }
 
@@ -1402,13 +1407,17 @@ export class ActionExecutor extends EventEmitter {
     // or a disallowed path never costs a Chromium process. preflightFetch is
     // deliberately not used here: its identity/signature headers belong on an
     // HTTP fetch, not on a browser context.
-    await this.assertRobotsAllowed(url, browserOptions);
+    const gateWarnings = await this.assertRobotsAllowed(url, browserOptions);
 
     const isStealth = !!browserOptions.stealthMode?.enabled;
 
     // Use the enhanced BrowserProcessor initialization that supports stealth mode
     const page = await this.browserProcessor.initializePage(browserOptions);
 
+    // Stamped on the page for the same reason __crawlforgeNavigation is: the
+    // caller's result is assembled a layer up, and the gate ran a layer down.
+    page.__crawlforgeGateWarnings = gateWarnings;
+
     try {
       // Apply CloudFlare and reCAPTCHA detection if stealth mode is enabled
       if (isStealth && this.browserProcessor.stealthManager) {

package/src/resources/ResourceRegistry.js

@@ -6,6 +6,23 @@
 
 import { createHash } from 'crypto';
 
+/**
+ * The stdio transport frames one JSON-RPC message at a time, and since SDK
+ * 1.30 its ReadBuffer CLOSES the transport when a message runs past
+ * STDIO_MESSAGE_CEILING_BYTES: the whole session dies, every tool with it, and
+ * the overflow is not recoverable (modelcontextprotocol/typescript-sdk#2793).
+ * A full-page PNG of a long article is 17.4 MB — en.wikipedia.org/wiki/World_War_II
+ * is 61,341px tall — and JPEG only brings it to 11.3 MB, so neither format saves
+ * a caller who passes full_page (R23, 2026-09-13).
+ *
+ * Hence a budget on the blob a read may emit: base64 costs four bytes per three,
+ * and a tenth of the ceiling is left for the JSON-RPC envelope around it.
+ */
+const STDIO_MESSAGE_CEILING_BYTES = 10 * 1024 * 1024;
+export const MAX_RESOURCE_BLOB_BYTES =
+  Number(process.env.CRAWLFORGE_MAX_RESOURCE_BLOB_BYTES) ||
+  Math.floor(STDIO_MESSAGE_CEILING_BYTES * 0.9 * 3 / 4);
+
 /**
  * Supported resource types and their MIME types.
  */
@@ -84,6 +101,12 @@ export class ResourceRegistry {
       createdAt: Date.now(),
       ttl: this.defaultTtl,
     });
+    // Handed back so the tool that took the shot can say, in the same result
+    // that carries the URI, whether that URI is readable over stdio at all.
+    return {
+      bytes: buf.length,
+      withinInlineBudget: buf.length <= MAX_RESOURCE_BLOB_BYTES,
+    };
   }
 
   /**
@@ -152,7 +175,7 @@ export class ResourceRegistry {
         resources.push({
           uri: `crawlforge://screenshot/${actionId}`,
           name: `Screenshot ${actionId}`,
-          description: 'Screenshot from scrape_with_actions',
+          description: 'Screenshot from a CrawlForge browser tool',
           mimeType: RESOURCE_MIME.screenshot,
         });
       }
@@ -265,6 +288,17 @@ export class ResourceRegistry {
     if (!entry || Date.now() - entry.createdAt >= entry.ttl) {
       throw new Error(`Screenshot not found or expired: ${actionId}`);
     }
+    // Refuse rather than emit: an oversized message costs the caller their
+    // whole session, and this error costs them one read they can act on.
+    if (entry.data.length > MAX_RESOURCE_BLOB_BYTES) {
+      throw new Error(
+        `Screenshot ${actionId} is ${entry.data.length} bytes, over the ` +
+        `${MAX_RESOURCE_BLOB_BYTES}-byte limit for one MCP message. Returning it would ` +
+        `close the connection instead of failing this read, so it is refused. Take the ` +
+        `shot again without full_page, or as format:"jpeg" with a lower quality, or ` +
+        `scoped to one element with selector.`
+      );
+    }
     return {
       contents: [{
         uri,

package/src/tools/advanced/BrowserSessionTool.js

@@ -170,6 +170,21 @@ function withJsResult(result) {
   return { ...result, jsResult: result.result.result };
 }
 
+/**
+ * The gate's warnings — a respect_robots override, a crawl-delay note — are
+ * what the shared parameter description promises the caller gets back
+ * ("returns a warning in the response"). `scrape` publishes them through
+ * unifiedScrape; the browser path dropped them on the floor, so a session that
+ * overrode robots.txt was told nothing at all (R23, 2026-09-13).
+ *
+ * Last navigation wins, the same rule `__crawlforgeNavigation` follows: the
+ * warnings describe the hop the caller just made, not every hop of the session.
+ */
+function gateWarningFields(page) {
+  const warnings = page?.__crawlforgeGateWarnings;
+  return warnings?.length ? { warnings } : {};
+}
+
 export class BrowserSessionTool {
   constructor(options = {}) {
     const {
@@ -278,7 +293,8 @@ export class BrowserSessionTool {
       viewportWidth: params.viewport?.width,
       viewportHeight: params.viewport?.height,
       timeout: params.timeout,
-      respectRobots: params.respect_robots
+      respectRobots: params.respect_robots,
+      tool: 'browser_session'
     };
     if (params.stealth) {
       browserOptions.stealthMode = { enabled: true };
@@ -325,6 +341,7 @@ export class BrowserSessionTool {
       operation: 'open',
       ...sessionInfo(session),
       ...verdictFields(verdict),
+      ...gateWarningFields(page),
       // The page is a wall, but the session behind it is real and holds a
       // browser context — say so, or a caller reading only `success` abandons
       // it to its TTL instead of closing it or acting through the challenge.
@@ -411,7 +428,7 @@ export class BrowserSessionTool {
     const result = await this.actionExecutor.executeActionsOnPage(session.page, params.actions, {
       continueOnError: params.continue_on_error,
       timeout: params.timeout,
-      browserOptions: { respectRobots: params.respect_robots }
+      browserOptions: { respectRobots: params.respect_robots, tool: 'browser_session' }
     });
 
     this.store.touch(session, result.finalUrl);
@@ -424,6 +441,7 @@ export class BrowserSessionTool {
       ...(Number.isInteger(session.page.__crawlforgeNavigation?.status)
         ? { httpStatus: session.page.__crawlforgeNavigation.status }
         : {}),
+      ...gateWarningFields(session.page),
       error: result.error,
       actionResults: result.results.map(withJsResult),
       screenshots: result.screenshots,