crawlforge-mcp-server 6.6.0 → 6.6.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "6.6.0",
+  "version": "6.6.2",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 30 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",

package/server.js

@@ -61,13 +61,13 @@ import { REDACT_PII_PARAM } from "./src/server/redaction.js"; // Phase 5 (5.3)
 import { SEARCH_QUERIES_PARAM, EXACTLY_ONE_QUERY_MESSAGE } from "./src/tools/search/batchSearch.js"; // Phase 5 (5.1)
 import { markPreflightRefusal } from "./src/server/requestContext.js";
 // D1.1 Resources + D1.2 Prompts + D1.4 Elicitation
-import { ResourceRegistry } from "./src/resources/ResourceRegistry.js";
+import { ResourceRegistry, MAX_RESOURCE_BLOB_BYTES } from "./src/resources/ResourceRegistry.js";
 import { PROMPTS, getPromptMessages } from "./src/prompts/PromptRegistry.js";
 import { ElicitationHelper } from "./src/core/ElicitationHelper.js";
 // Phase 6: MCP-spec adoption — structured output, tool filtering, spec hygiene
 import { OUTPUT_SCHEMAS } from "./src/schemas/toolOutputSchemas.js";
 import { dualOutput } from "./src/server/registerTool.js";
-import { createToolFilter } from "./src/server/toolFilter.js";
+import { createToolFilter, TOOL_GROUPS } from "./src/server/toolFilter.js";
 import { applySpecHygiene } from "./src/server/specHygiene.js";
 
 // Initialize Authentication Manager
@@ -108,7 +108,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "6.6.0",
+  version: "6.6.2",
   description: "Production-ready MCP server with 31 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, stateful browser sessions with element refs, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -1051,7 +1051,8 @@ registerToolIfEnabled("browser_session", {
     full_page: z.boolean().default(false).describe("screenshot: capture the full scrollable page"),
     format: z.enum(["png", "jpeg"]).default("png").describe("screenshot: image format"),
     quality: z.number().min(0).max(100).default(80).describe("screenshot: JPEG quality"),
-    selector: z.string().optional().describe("screenshot: capture just this element (a ref like \"@e2\" works)")
+    selector: z.string().optional().describe("screenshot: capture just this element (a ref like \"@e2\" works)"),
+    ...MAX_INLINE_CHARS_PARAM
   }
 }, withAuth("browser_session", async (params) => {
   try {
@@ -1062,9 +1063,24 @@ registerToolIfEnabled("browser_session", {
     // is megabytes beside a few lines of JSON (R21, 2026-09-09).
     const publish = (shot) => {
       if (!shot?.actionId || !shot?.data) return shot;
-      resourceRegistry.storeScreenshot(shot.actionId, shot.data);
+      const { bytes, withinInlineBudget } = resourceRegistry.storeScreenshot(shot.actionId, shot.data);
       const { data, ...rest } = shot;
-      return { ...rest, resourceUri: `crawlforge://screenshot/${shot.actionId}` };
+      return {
+        ...rest,
+        resourceUri: `crawlforge://screenshot/${shot.actionId}`,
+        bytes,
+        // full_page is a knob this tool hands the caller, and on a long page it
+        // produces an image no MCP message can carry. Saying so here costs one
+        // field; learning it from the read costs a wasted call (R23).
+        ...(withinInlineBudget
+          ? {}
+          : {
+              warning: `This image is ${bytes} bytes, too large to read back over MCP ` +
+                `(limit ${MAX_RESOURCE_BLOB_BYTES} bytes) — reading the resource will be refused. ` +
+                `Take it again without full_page, or as format:"jpeg" with a lower quality, or ` +
+                `scoped to one element with selector.`
+            })
+      };
     };
     if (result.screenshot) result.screenshot = publish(result.screenshot);
     if (Array.isArray(result.screenshots)) result.screenshots = result.screenshots.map(publish);
@@ -1712,16 +1728,11 @@ async function runServer() {
   console.error(`Environment: ${config.server.nodeEnv}`);
   console.error("Search enabled: true (via CrawlForge proxy)");
 
-  const allTools = [
-    "fetch_url", "extract_text", "extract_links", "extract_metadata", "scrape_structured", "extract_embedded_state",
-    "search_web", "serp_rank", "reddit_search", "crawl_deep", "map_site",
-    "extract_content", "process_document", "summarize_content", "analyze_content",
-    "batch_scrape", "get_batch_results", "read_result", "scrape_with_actions",
-    "deep_research", "track_changes", "generate_llms_txt",
-    "stealth_mode", "localization", "extract_structured", "extract_with_llm",
-    "list_ollama_models", "scrape_template", // D3.3
-    "scrape", "agent"  // D4
-  ];
+  // Derived from TOOL_GROUPS, the same list the filter itself is built on, so a
+  // new tool is announced the moment it is grouped. The hand-written copy that
+  // used to live here silently under-reported: it never learned browser_session
+  // and so claimed "30/30" while 31 tools were registered and advertised.
+  const allTools = Object.values(TOOL_GROUPS).flat();
   const enabledTools = allTools.filter((name) => toolFilter.isEnabled(name));
   console.error(`Tools available (${enabledTools.length}/${allTools.length}): ${enabledTools.join(", ")}`);
 

package/src/core/ActionExecutor.js

@@ -1184,12 +1184,13 @@ export class ActionExecutor extends EventEmitter {
     const timeout = this.actionTimeout(action);
 
     await assertUrlAllowed(action.url, { resolveDns: true });
-    await this.assertRobotsAllowed(action.url, executionContext?.browserOptions);
+    const gateWarnings = await this.assertRobotsAllowed(action.url, executionContext?.browserOptions);
 
     await this.navigateToUrl(page, action.url, {
       waitUntil: action.waitUntil,
       timeout
     });
+    page.__crawlforgeGateWarnings = gateWarnings;
 
     return {
       url: action.url,
@@ -1380,9 +1381,13 @@ export class ActionExecutor extends EventEmitter {
    * @throws {BlockedHostError|RobotsDisallowedError}
    */
   async assertRobotsAllowed(url, browserOptions = {}) {
-    await browserPreflight(url, {
+    return await browserPreflight(url, {
       respectRobots: browserOptions?.respectRobots,
-      tool: 'scrape_with_actions'
+      // The audit row is the record of the CUSTOMER's decision (G5), so it has
+      // to name the tool that actually made it. browser_session borrows this
+      // executor, and until R23 every session's override was filed against
+      // scrape_with_actions.
+      tool: browserOptions?.tool || 'scrape_with_actions'
     });
   }
 
@@ -1402,13 +1407,17 @@ export class ActionExecutor extends EventEmitter {
     // or a disallowed path never costs a Chromium process. preflightFetch is
     // deliberately not used here: its identity/signature headers belong on an
     // HTTP fetch, not on a browser context.
-    await this.assertRobotsAllowed(url, browserOptions);
+    const gateWarnings = await this.assertRobotsAllowed(url, browserOptions);
 
     const isStealth = !!browserOptions.stealthMode?.enabled;
 
     // Use the enhanced BrowserProcessor initialization that supports stealth mode
     const page = await this.browserProcessor.initializePage(browserOptions);
 
+    // Stamped on the page for the same reason __crawlforgeNavigation is: the
+    // caller's result is assembled a layer up, and the gate ran a layer down.
+    page.__crawlforgeGateWarnings = gateWarnings;
+
     try {
       // Apply CloudFlare and reCAPTCHA detection if stealth mode is enabled
       if (isStealth && this.browserProcessor.stealthManager) {

package/src/resources/ResourceRegistry.js

@@ -6,6 +6,23 @@
 
 import { createHash } from 'crypto';
 
+/**
+ * The stdio transport frames one JSON-RPC message at a time, and since SDK
+ * 1.30 its ReadBuffer CLOSES the transport when a message runs past
+ * STDIO_MESSAGE_CEILING_BYTES: the whole session dies, every tool with it, and
+ * the overflow is not recoverable (modelcontextprotocol/typescript-sdk#2793).
+ * A full-page PNG of a long article is 17.4 MB — en.wikipedia.org/wiki/World_War_II
+ * is 61,341px tall — and JPEG only brings it to 11.3 MB, so neither format saves
+ * a caller who passes full_page (R23, 2026-09-13).
+ *
+ * Hence a budget on the blob a read may emit: base64 costs four bytes per three,
+ * and a tenth of the ceiling is left for the JSON-RPC envelope around it.
+ */
+const STDIO_MESSAGE_CEILING_BYTES = 10 * 1024 * 1024;
+export const MAX_RESOURCE_BLOB_BYTES =
+  Number(process.env.CRAWLFORGE_MAX_RESOURCE_BLOB_BYTES) ||
+  Math.floor(STDIO_MESSAGE_CEILING_BYTES * 0.9 * 3 / 4);
+
 /**
  * Supported resource types and their MIME types.
  */
@@ -84,6 +101,12 @@ export class ResourceRegistry {
       createdAt: Date.now(),
       ttl: this.defaultTtl,
     });
+    // Handed back so the tool that took the shot can say, in the same result
+    // that carries the URI, whether that URI is readable over stdio at all.
+    return {
+      bytes: buf.length,
+      withinInlineBudget: buf.length <= MAX_RESOURCE_BLOB_BYTES,
+    };
   }
 
   /**
@@ -152,7 +175,7 @@ export class ResourceRegistry {
         resources.push({
           uri: `crawlforge://screenshot/${actionId}`,
           name: `Screenshot ${actionId}`,
-          description: 'Screenshot from scrape_with_actions',
+          description: 'Screenshot from a CrawlForge browser tool',
           mimeType: RESOURCE_MIME.screenshot,
         });
       }
@@ -265,6 +288,17 @@ export class ResourceRegistry {
     if (!entry || Date.now() - entry.createdAt >= entry.ttl) {
       throw new Error(`Screenshot not found or expired: ${actionId}`);
     }
+    // Refuse rather than emit: an oversized message costs the caller their
+    // whole session, and this error costs them one read they can act on.
+    if (entry.data.length > MAX_RESOURCE_BLOB_BYTES) {
+      throw new Error(
+        `Screenshot ${actionId} is ${entry.data.length} bytes, over the ` +
+        `${MAX_RESOURCE_BLOB_BYTES}-byte limit for one MCP message. Returning it would ` +
+        `close the connection instead of failing this read, so it is refused. Take the ` +
+        `shot again without full_page, or as format:"jpeg" with a lower quality, or ` +
+        `scoped to one element with selector.`
+      );
+    }
     return {
       contents: [{
         uri,

package/src/server/inlineThreshold.js

@@ -24,6 +24,9 @@ export const MAX_INLINE_CHARS_PARAM = {
  * false` keeps the whole result inline and only adds the handle
  * (extract_embedded_state's never-truncate rule). `when` gates on params.
  */
+/** The browser_session operations that hand back content worth shaping. */
+const BROWSER_SESSION_CONTENT_OPERATIONS = new Set(['snapshot', 'act', 'read']);
+
 export const INLINE_THRESHOLD_TOOLS = Object.freeze({
   scrape: { textPaths: ['content.markdown', 'content.text', 'content.html', 'content.rawHtml'], truncate: true },
   fetch_url: { textPaths: ['body'], truncate: true },
@@ -35,6 +38,18 @@ export const INLINE_THRESHOLD_TOOLS = Object.freeze({
   get_batch_results: { textPaths: [], truncate: true },
   stealth_mode: { textPaths: ['content.markdown', 'content.text', 'content.html'], truncate: true, when: (params) => params?.operation === 'scrape' },
   scrape_with_actions: { textPaths: ['content.markdown', 'content.text', 'content.html'], truncate: true },
+  // `read` hands back the same content shape scrape_with_actions does, and was
+  // the one content-returning tool with no cap: a read of the World War II
+  // article returned 541,308 characters inline where scrape returned 42,259
+  // (2026-09-12). The operations, the paths and their order are the REST
+  // route's (src/app/api/v1/tools/browser_session/route.ts, CONTENT_OPERATIONS)
+  // so the same call is shaped the same way whichever surface serves it; the
+  // other four return a session id and an expiry and must not be shaped.
+  browser_session: {
+    textPaths: ['content.markdown', 'content.text', 'content.html', 'snapshot.tree'],
+    truncate: true,
+    when: (params) => BROWSER_SESSION_CONTENT_OPERATIONS.has(params?.operation)
+  },
   process_document: { textPaths: ['content.text'], truncate: true },
   deep_research: { textPaths: [], truncate: true },
   extract_embedded_state: { textPaths: [], truncate: false }

package/src/tools/advanced/BrowserSessionTool.js

@@ -40,6 +40,7 @@ import { isCreatorModeVerified } from '../../core/creatorMode.js';
 import { internalOwnerToken, isInternalRequest } from '../../server/requestContext.js';
 import { isRemoteTransport } from '../../utils/remoteMode.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
+import { stealthDocumentVerdict } from '../../utils/stealthVerdict.js';
 
 const SECOND = 1000;
 
@@ -76,7 +77,14 @@ const REST_MAX_SESSIONS_PER_OWNER = 1;
 const SessionActionSchema = z.object({
   type: z.string(),
   continueOnError: z.boolean().default(false),
-  retries: z.number().min(0).max(5).default(1)
+  retries: z.number().min(0).max(5).default(1),
+  // The third default ActionExecutor declares and then throws away. It is read
+  // only by executeJavaScript, where `action.returnResult ? result : undefined`
+  // decides whether the script's return value survives at all — so without it
+  // stamped here every executeJavaScript in a session succeeded and handed back
+  // nothing, while the same action through scrape_with_actions (which keeps its
+  // parsed value) returned the data. Harmless on the action types that ignore it.
+  returnResult: z.boolean().default(true)
 }).passthrough();
 
 const BrowserSessionSchema = z.object({
@@ -140,6 +148,43 @@ function sessionInfo(session) {
   };
 }
 
+/**
+ * The verdict fields a result carries when there is something to say about the
+ * document — the same names `scrape` and `scrape_with_actions` publish.
+ */
+function verdictFields(verdict) {
+  return {
+    ...(Number.isInteger(verdict.status) ? { httpStatus: verdict.status } : {}),
+    ...(verdict.blocked ? { blocked: verdict.blocked } : {})
+  };
+}
+
+/**
+ * `scrape_with_actions` publishes an executeJavaScript action's return value as
+ * a flat `jsResult` beside the nested one (processActionResults); a session's
+ * `act` returned the nested shape alone, so the same action read differently
+ * depending on which tool ran it. Same hoist, same field name.
+ */
+function withJsResult(result) {
+  if (result?.type !== 'executeJavaScript' || !result.result) return result;
+  return { ...result, jsResult: result.result.result };
+}
+
+/**
+ * The gate's warnings — a respect_robots override, a crawl-delay note — are
+ * what the shared parameter description promises the caller gets back
+ * ("returns a warning in the response"). `scrape` publishes them through
+ * unifiedScrape; the browser path dropped them on the floor, so a session that
+ * overrode robots.txt was told nothing at all (R23, 2026-09-13).
+ *
+ * Last navigation wins, the same rule `__crawlforgeNavigation` follows: the
+ * warnings describe the hop the caller just made, not every hop of the session.
+ */
+function gateWarningFields(page) {
+  const warnings = page?.__crawlforgeGateWarnings;
+  return warnings?.length ? { warnings } : {};
+}
+
 export class BrowserSessionTool {
   constructor(options = {}) {
     const {
@@ -248,7 +293,8 @@ export class BrowserSessionTool {
       viewportWidth: params.viewport?.width,
       viewportHeight: params.viewport?.height,
       timeout: params.timeout,
-      respectRobots: params.respect_robots
+      respectRobots: params.respect_robots,
+      tool: 'browser_session'
     };
     if (params.stealth) {
       browserOptions.stealthMode = { enabled: true };
@@ -282,7 +328,63 @@ export class BrowserSessionTool {
       throw error;
     }
 
-    return { success: true, operation: 'open', ...sessionInfo(session) };
+    // The session opened; whether the document it landed on is the page is a
+    // separate question. g2.com answered `open` with a DataDome 403 whose body
+    // was empty, and this returned success:true with no status at all, while
+    // `scrape` on the same URL named the vendor and the 403 — the same fault
+    // R18 found in scrape_with_actions (2026-09-04), in the one browser tool
+    // that never learned the lesson.
+    const verdict = await this.pageVerdict(page, { stealth: params.stealth });
+
+    return {
+      success: verdict.success,
+      operation: 'open',
+      ...sessionInfo(session),
+      ...verdictFields(verdict),
+      ...gateWarningFields(page),
+      // The page is a wall, but the session behind it is real and holds a
+      // browser context — say so, or a caller reading only `success` abandons
+      // it to its TTL instead of closing it or acting through the challenge.
+      ...(verdict.error
+        ? { error: `${verdict.error} The session is open as ${session.id}: act on it, or close it.` }
+        : {})
+    };
+  }
+
+  /**
+   * What the session's page currently is: the page, a bot wall, an HTTP error
+   * page, or an error placeholder. One helper, shared with `scrape` and
+   * `scrape_with_actions`, so all three name a block identically instead of
+   * this tool staying silent about it.
+   *
+   * `allowEmpty` because a session is routinely opened on an app shell that
+   * only paints after the actions the caller is about to send — an empty
+   * document is a normal starting state here, not a failure. A real wall still
+   * fails on its challenge signature or its HTTP status, which is what the
+   * empty-document rule would have caught anyway.
+   *
+   * Never throws. A verdict is a diagnosis; a page that cannot be read for one
+   * (closed, mid-navigation) must not fail the operation being diagnosed.
+   */
+  async pageVerdict(page, { stealth = false, ...known } = {}) {
+    try {
+      const title = known.title !== undefined ? known.title : await page.title();
+      const html = known.html !== undefined ? known.html : await page.content();
+      const text = known.text !== undefined
+        ? known.text
+        : await page.evaluate(() => document.body?.innerText || '');
+
+      return stealthDocumentVerdict(
+        { url: page.url(), title, text, html, status: page.__crawlforgeNavigation?.status ?? null },
+        // The verdict's messages name whatever fetched the document, and its
+        // default is the stealth browser. A plain session is not that, and
+        // telling someone "the stealth browser did not pass it" when they never
+        // asked for stealth hides the one retry that might work.
+        { allowEmpty: true, fetcher: stealth ? 'the stealth browser session' : 'the browser session' }
+      );
+    } catch {
+      return { success: true, status: null };
+    }
   }
 
   async snapshotSession(params, ownerId) {
@@ -326,7 +428,7 @@ export class BrowserSessionTool {
     const result = await this.actionExecutor.executeActionsOnPage(session.page, params.actions, {
       continueOnError: params.continue_on_error,
       timeout: params.timeout,
-      browserOptions: { respectRobots: params.respect_robots }
+      browserOptions: { respectRobots: params.respect_robots, tool: 'browser_session' }
     });
 
     this.store.touch(session, result.finalUrl);
@@ -334,8 +436,14 @@ export class BrowserSessionTool {
       success: result.success,
       operation: 'act',
       ...sessionInfo(session),
+      // The status of the last navigation this chain made, if it made one —
+      // a `navigate` action onto a 404 or a wall is otherwise invisible.
+      ...(Number.isInteger(session.page.__crawlforgeNavigation?.status)
+        ? { httpStatus: session.page.__crawlforgeNavigation.status }
+        : {}),
+      ...gateWarningFields(session.page),
       error: result.error,
-      actionResults: result.results,
+      actionResults: result.results.map(withJsResult),
       screenshots: result.screenshots,
       ...(result.capturedStates.length > 0 ? { capturedStates: result.capturedStates } : {}),
       stats: result.stats
@@ -378,11 +486,24 @@ export class BrowserSessionTool {
       };
     }
 
+    // Read is where the content is actually handed over, so it is the last
+    // place a wall can be named before a caller treats it as the page: g2.com
+    // came back here as the single word "g2.com" with success:true. The
+    // document is already in hand, so this costs no extra page work.
+    const verdict = await this.pageVerdict(session.page, {
+      stealth: session.stealth,
+      title: extracted.title ?? '',
+      text: extracted.content?.text || '',
+      html
+    });
+
     this.store.touch(session, url);
     return {
-      success: true,
+      success: verdict.success,
       operation: 'read',
       ...sessionInfo(session),
+      ...verdictFields(verdict),
+      ...(verdict.error ? { error: verdict.error } : {}),
       title: extracted.title ?? null,
       extractionMethod: extracted.extractionMethod,
       content