crawlforge-mcp-server 6.6.0 → 6.6.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "6.6.0",
+  "version": "6.6.1",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 30 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",

package/server.js

@@ -67,7 +67,7 @@ import { ElicitationHelper } from "./src/core/ElicitationHelper.js";
 // Phase 6: MCP-spec adoption — structured output, tool filtering, spec hygiene
 import { OUTPUT_SCHEMAS } from "./src/schemas/toolOutputSchemas.js";
 import { dualOutput } from "./src/server/registerTool.js";
-import { createToolFilter } from "./src/server/toolFilter.js";
+import { createToolFilter, TOOL_GROUPS } from "./src/server/toolFilter.js";
 import { applySpecHygiene } from "./src/server/specHygiene.js";
 
 // Initialize Authentication Manager
@@ -108,7 +108,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "6.6.0",
+  version: "6.6.1",
   description: "Production-ready MCP server with 31 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, stateful browser sessions with element refs, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -1051,7 +1051,8 @@ registerToolIfEnabled("browser_session", {
     full_page: z.boolean().default(false).describe("screenshot: capture the full scrollable page"),
     format: z.enum(["png", "jpeg"]).default("png").describe("screenshot: image format"),
     quality: z.number().min(0).max(100).default(80).describe("screenshot: JPEG quality"),
-    selector: z.string().optional().describe("screenshot: capture just this element (a ref like \"@e2\" works)")
+    selector: z.string().optional().describe("screenshot: capture just this element (a ref like \"@e2\" works)"),
+    ...MAX_INLINE_CHARS_PARAM
   }
 }, withAuth("browser_session", async (params) => {
   try {
@@ -1712,16 +1713,11 @@ async function runServer() {
   console.error(`Environment: ${config.server.nodeEnv}`);
   console.error("Search enabled: true (via CrawlForge proxy)");
 
-  const allTools = [
-    "fetch_url", "extract_text", "extract_links", "extract_metadata", "scrape_structured", "extract_embedded_state",
-    "search_web", "serp_rank", "reddit_search", "crawl_deep", "map_site",
-    "extract_content", "process_document", "summarize_content", "analyze_content",
-    "batch_scrape", "get_batch_results", "read_result", "scrape_with_actions",
-    "deep_research", "track_changes", "generate_llms_txt",
-    "stealth_mode", "localization", "extract_structured", "extract_with_llm",
-    "list_ollama_models", "scrape_template", // D3.3
-    "scrape", "agent"  // D4
-  ];
+  // Derived from TOOL_GROUPS, the same list the filter itself is built on, so a
+  // new tool is announced the moment it is grouped. The hand-written copy that
+  // used to live here silently under-reported: it never learned browser_session
+  // and so claimed "30/30" while 31 tools were registered and advertised.
+  const allTools = Object.values(TOOL_GROUPS).flat();
   const enabledTools = allTools.filter((name) => toolFilter.isEnabled(name));
   console.error(`Tools available (${enabledTools.length}/${allTools.length}): ${enabledTools.join(", ")}`);
 

package/src/server/inlineThreshold.js

@@ -24,6 +24,9 @@ export const MAX_INLINE_CHARS_PARAM = {
  * false` keeps the whole result inline and only adds the handle
  * (extract_embedded_state's never-truncate rule). `when` gates on params.
  */
+/** The browser_session operations that hand back content worth shaping. */
+const BROWSER_SESSION_CONTENT_OPERATIONS = new Set(['snapshot', 'act', 'read']);
+
 export const INLINE_THRESHOLD_TOOLS = Object.freeze({
   scrape: { textPaths: ['content.markdown', 'content.text', 'content.html', 'content.rawHtml'], truncate: true },
   fetch_url: { textPaths: ['body'], truncate: true },
@@ -35,6 +38,18 @@ export const INLINE_THRESHOLD_TOOLS = Object.freeze({
   get_batch_results: { textPaths: [], truncate: true },
   stealth_mode: { textPaths: ['content.markdown', 'content.text', 'content.html'], truncate: true, when: (params) => params?.operation === 'scrape' },
   scrape_with_actions: { textPaths: ['content.markdown', 'content.text', 'content.html'], truncate: true },
+  // `read` hands back the same content shape scrape_with_actions does, and was
+  // the one content-returning tool with no cap: a read of the World War II
+  // article returned 541,308 characters inline where scrape returned 42,259
+  // (2026-09-12). The operations, the paths and their order are the REST
+  // route's (src/app/api/v1/tools/browser_session/route.ts, CONTENT_OPERATIONS)
+  // so the same call is shaped the same way whichever surface serves it; the
+  // other four return a session id and an expiry and must not be shaped.
+  browser_session: {
+    textPaths: ['content.markdown', 'content.text', 'content.html', 'snapshot.tree'],
+    truncate: true,
+    when: (params) => BROWSER_SESSION_CONTENT_OPERATIONS.has(params?.operation)
+  },
   process_document: { textPaths: ['content.text'], truncate: true },
   deep_research: { textPaths: [], truncate: true },
   extract_embedded_state: { textPaths: [], truncate: false }

package/src/tools/advanced/BrowserSessionTool.js

@@ -40,6 +40,7 @@ import { isCreatorModeVerified } from '../../core/creatorMode.js';
 import { internalOwnerToken, isInternalRequest } from '../../server/requestContext.js';
 import { isRemoteTransport } from '../../utils/remoteMode.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
+import { stealthDocumentVerdict } from '../../utils/stealthVerdict.js';
 
 const SECOND = 1000;
 
@@ -76,7 +77,14 @@ const REST_MAX_SESSIONS_PER_OWNER = 1;
 const SessionActionSchema = z.object({
   type: z.string(),
   continueOnError: z.boolean().default(false),
-  retries: z.number().min(0).max(5).default(1)
+  retries: z.number().min(0).max(5).default(1),
+  // The third default ActionExecutor declares and then throws away. It is read
+  // only by executeJavaScript, where `action.returnResult ? result : undefined`
+  // decides whether the script's return value survives at all — so without it
+  // stamped here every executeJavaScript in a session succeeded and handed back
+  // nothing, while the same action through scrape_with_actions (which keeps its
+  // parsed value) returned the data. Harmless on the action types that ignore it.
+  returnResult: z.boolean().default(true)
 }).passthrough();
 
 const BrowserSessionSchema = z.object({
@@ -140,6 +148,28 @@ function sessionInfo(session) {
   };
 }
 
+/**
+ * The verdict fields a result carries when there is something to say about the
+ * document — the same names `scrape` and `scrape_with_actions` publish.
+ */
+function verdictFields(verdict) {
+  return {
+    ...(Number.isInteger(verdict.status) ? { httpStatus: verdict.status } : {}),
+    ...(verdict.blocked ? { blocked: verdict.blocked } : {})
+  };
+}
+
+/**
+ * `scrape_with_actions` publishes an executeJavaScript action's return value as
+ * a flat `jsResult` beside the nested one (processActionResults); a session's
+ * `act` returned the nested shape alone, so the same action read differently
+ * depending on which tool ran it. Same hoist, same field name.
+ */
+function withJsResult(result) {
+  if (result?.type !== 'executeJavaScript' || !result.result) return result;
+  return { ...result, jsResult: result.result.result };
+}
+
 export class BrowserSessionTool {
   constructor(options = {}) {
     const {
@@ -282,7 +312,62 @@ export class BrowserSessionTool {
       throw error;
     }
 
-    return { success: true, operation: 'open', ...sessionInfo(session) };
+    // The session opened; whether the document it landed on is the page is a
+    // separate question. g2.com answered `open` with a DataDome 403 whose body
+    // was empty, and this returned success:true with no status at all, while
+    // `scrape` on the same URL named the vendor and the 403 — the same fault
+    // R18 found in scrape_with_actions (2026-09-04), in the one browser tool
+    // that never learned the lesson.
+    const verdict = await this.pageVerdict(page, { stealth: params.stealth });
+
+    return {
+      success: verdict.success,
+      operation: 'open',
+      ...sessionInfo(session),
+      ...verdictFields(verdict),
+      // The page is a wall, but the session behind it is real and holds a
+      // browser context — say so, or a caller reading only `success` abandons
+      // it to its TTL instead of closing it or acting through the challenge.
+      ...(verdict.error
+        ? { error: `${verdict.error} The session is open as ${session.id}: act on it, or close it.` }
+        : {})
+    };
+  }
+
+  /**
+   * What the session's page currently is: the page, a bot wall, an HTTP error
+   * page, or an error placeholder. One helper, shared with `scrape` and
+   * `scrape_with_actions`, so all three name a block identically instead of
+   * this tool staying silent about it.
+   *
+   * `allowEmpty` because a session is routinely opened on an app shell that
+   * only paints after the actions the caller is about to send — an empty
+   * document is a normal starting state here, not a failure. A real wall still
+   * fails on its challenge signature or its HTTP status, which is what the
+   * empty-document rule would have caught anyway.
+   *
+   * Never throws. A verdict is a diagnosis; a page that cannot be read for one
+   * (closed, mid-navigation) must not fail the operation being diagnosed.
+   */
+  async pageVerdict(page, { stealth = false, ...known } = {}) {
+    try {
+      const title = known.title !== undefined ? known.title : await page.title();
+      const html = known.html !== undefined ? known.html : await page.content();
+      const text = known.text !== undefined
+        ? known.text
+        : await page.evaluate(() => document.body?.innerText || '');
+
+      return stealthDocumentVerdict(
+        { url: page.url(), title, text, html, status: page.__crawlforgeNavigation?.status ?? null },
+        // The verdict's messages name whatever fetched the document, and its
+        // default is the stealth browser. A plain session is not that, and
+        // telling someone "the stealth browser did not pass it" when they never
+        // asked for stealth hides the one retry that might work.
+        { allowEmpty: true, fetcher: stealth ? 'the stealth browser session' : 'the browser session' }
+      );
+    } catch {
+      return { success: true, status: null };
+    }
   }
 
   async snapshotSession(params, ownerId) {
@@ -334,8 +419,13 @@ export class BrowserSessionTool {
       success: result.success,
       operation: 'act',
       ...sessionInfo(session),
+      // The status of the last navigation this chain made, if it made one —
+      // a `navigate` action onto a 404 or a wall is otherwise invisible.
+      ...(Number.isInteger(session.page.__crawlforgeNavigation?.status)
+        ? { httpStatus: session.page.__crawlforgeNavigation.status }
+        : {}),
       error: result.error,
-      actionResults: result.results,
+      actionResults: result.results.map(withJsResult),
       screenshots: result.screenshots,
       ...(result.capturedStates.length > 0 ? { capturedStates: result.capturedStates } : {}),
       stats: result.stats
@@ -378,11 +468,24 @@ export class BrowserSessionTool {
       };
     }
 
+    // Read is where the content is actually handed over, so it is the last
+    // place a wall can be named before a caller treats it as the page: g2.com
+    // came back here as the single word "g2.com" with success:true. The
+    // document is already in hand, so this costs no extra page work.
+    const verdict = await this.pageVerdict(session.page, {
+      stealth: session.stealth,
+      title: extracted.title ?? '',
+      text: extracted.content?.text || '',
+      html
+    });
+
     this.store.touch(session, url);
     return {
-      success: true,
+      success: verdict.success,
       operation: 'read',
       ...sessionInfo(session),
+      ...verdictFields(verdict),
+      ...(verdict.error ? { error: verdict.error } : {}),
       title: extracted.title ?? null,
       extractionMethod: extracted.extractionMethod,
       content