crawlforge-mcp-server 5.0.3 → 5.0.5

package/CLAUDE.md

@@ -62,7 +62,7 @@ These guidelines are working if: fewer unnecessary changes in diffs, fewer rewri
 
 CrawlForge MCP Server - A professional MCP (Model Context Protocol) server providing 27 web scraping, crawling, and content processing tools (5 inline + 22 advanced).
 
-**Current Version:** 5.0.3
+**Current Version:** 5.0.5
 
 ## Development Commands
 
@@ -181,7 +181,7 @@ fetch_url, extract_text, extract_links, extract_metadata, scrape_structured
 search_web, serp_rank, crawl_deep, map_site, extract_content, process_document, summarize_content, analyze_content, extract_structured, extract_with_llm, list_ollama_models, batch_scrape, scrape_with_actions, deep_research, track_changes, generate_llms_txt, stealth_mode, localization, scrape_template, scrape, agent
 
 **serp_rank (DataForSEO):**
-- `serp_rank` — reports where a target domain ranks in Google's REAL organic results for a keyword (the SERP position Google Custom Search / `search_web` cannot give). Backed by the DataForSEO Google Organic SERP API (Live Advanced, `POST /v3/serp/google/organic/live/advanced`, HTTP Basic auth). Credentials via `DATAFORSEO_LOGIN` / `DATAFORSEO_PASSWORD`, billed to the user's own DataForSEO account (~US$0.002/call), separate from CrawlForge credits. When unconfigured it returns `{ configured:false }` and charges **0** credits; when configured, **Cost: 5**. Never fabricates a rank.
+- `serp_rank` — reports where a target domain ranks in Google's REAL organic results for a keyword (the SERP position Google Custom Search / `search_web` cannot give). Backed by the DataForSEO Google Organic SERP API (Live Advanced, `POST /v3/serp/google/organic/live/advanced`, HTTP Basic auth). Credentials via `DATAFORSEO_LOGIN` / `DATAFORSEO_PASSWORD`, billed to the user's own DataForSEO account (~US$0.002 per 10 results of `depth`, so $0.004 at the default `depth:20` and $0.02 at `depth:100`), separate from CrawlForge credits. When unconfigured it returns `{ configured:false }` and charges **0** credits; when configured, **Cost: 5**. Never fabricates a rank.
 
 **v4.6.0 additions (Phase D):**
 - `scrape` — single fetch + one cheerio load dispatching a `formats` array (markdown/html/rawHtml/text/links/metadata/screenshot/json-schema) + `onlyMainContent`; partial-success via per-format `warnings[]`. Cost: 2.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.0.3",
+  "version": "5.0.5",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 27 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -127,7 +127,8 @@
     "turndown-plugin-gfm": "^1.0.2",
     "undici": "^7.24.0",
     "winston": "^3.11.0",
-    "zod": "^3.23.8"
+    "zod": "^3.23.8",
+    "zod-to-json-schema": "^3.25.1"
   },
   "optionalDependencies": {
     "camoufox": "^0.1.19"

package/server.js

@@ -99,7 +99,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.0.2",
+  version: "5.0.5",
   description: "Production-ready MCP server with 27 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, real Google SERP rank tracking, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -427,7 +427,7 @@ registerToolIfEnabled("serp_rank", {
     location_code: z.number().optional().describe("Numeric DataForSEO location code (overrides location_name)"),
     language_code: z.string().optional().describe("Language code (e.g. 'en')"),
     device: z.enum(["desktop", "mobile"]).optional().describe("Device to emulate"),
-    depth: z.number().min(10).max(200).optional().describe("How many results to scan, 10-200 (100 = 1 page of cost)")
+    depth: z.number().min(10).max(200).optional().describe("How many results to scan, 10-200 (default 20; DataForSEO bills ~$0.002 per 10 and gets slower the deeper it goes)")
   },
   outputSchema: OUTPUT_SCHEMAS.serp_rank
 }, withAuth("serp_rank", async ({ keyword, target, location_name, location_code, language_code, device, depth }) => {

package/src/constants/config.js

@@ -303,7 +303,11 @@ export function validateConfig() {
 export function getToolConfig(toolName) {
   const toolConfigs = {
     search_web: {
-      apiKey: config.crawlforge.apiKey,
+      // Read env at call time, not import time: the CLI resolves the key
+      // (--api-key flag / env / ~/.crawlforge/config.json) in a preAction
+      // hook that runs AFTER this module was imported, so the frozen
+      // config.crawlforge.apiKey snapshot misses stored keys.
+      apiKey: process.env.CRAWLFORGE_API_KEY || config.crawlforge.apiKey,
       apiBaseUrl: config.crawlforge.apiBaseUrl,
       
       // Common configuration

package/src/core/AgentOrchestrator.js

@@ -200,9 +200,13 @@ export class AgentOrchestrator {
     // "news.ycombinator.com") are the most authoritative sources for the task —
     // queue them ahead of search results.
     const namedSites = prompt.match(/https?:\/\/[^\s"'<>]+|(?<![\w.@/-])[a-z0-9][\w-]*(?:\.[a-z0-9][\w-]*)*\.[a-z]{2,}(?![\w-])/gi) || [];
+    // Seeds and prompt-named sites also outrank search results at SHAPE time
+    // (see priorityUrls below) — generic term overlap must not bury them.
+    const priorityUrls = [...seedUrls];
     for (const site of namedSites) {
       const url = (/^https?:\/\//i.test(site) ? site : `https://${site}`).replace(/[.,;:!?)]+$/, '');
       if (!urlQueue.includes(url)) urlQueue.push(url);
+      if (!priorityUrls.includes(url)) priorityUrls.push(url);
     }
     const searchResults = [];
 
@@ -261,10 +265,16 @@ export class AgentOrchestrator {
     // url+title+text) instead of raw queue order, then give every source a
     // per-source slice of the synthesis budget so no source is silently cut off.
     const promptTerms = prompt.toLowerCase().split(/\s+/).filter(t => t.length > 3);
+    // Seeds/prompt-named sites get a fixed boost above any term-overlap score:
+    // a generic article mentioning the task's words ("title", "story") must
+    // never outrank the source the user explicitly pointed at (live retest
+    // 2026-08-20: an NFL article buried the named news.ycombinator.com page).
+    const isPriority = url => priorityUrls.some(p => url === p || url.startsWith(`${p}/`) || `${url}/` === p);
     const orderedEvidence = evidence
       .map(e => ({
         ...e,
-        _score: promptTerms.filter(t => `${e.url} ${e.title || ''} ${e.text}`.toLowerCase().includes(t)).length
+        _score: (isPriority(e.url) ? 1000 : 0) +
+          promptTerms.filter(t => `${e.url} ${e.title || ''} ${e.text}`.toLowerCase().includes(t)).length
       }))
       .sort((a, b) => b._score - a._score);
     const perSourceCap = Math.max(1500, Math.floor(12000 / Math.max(evidence.length, 1)));
@@ -317,14 +327,19 @@ export class AgentOrchestrator {
     let degradedReason;
 
     try {
+      // Wording matters for small local models (llama3.2-class): without the
+      // "already fetched / do not refuse" framing they answer "I cannot access
+      // the internet" or "the sources do not contain it" even when the answer
+      // sits in the first source (live retest 2026-08-20).
       const synthesisPrompt =
-        `You are a research assistant. Answer this task using ONLY the sources below:\n\n` +
+        `You are a data-extraction assistant. The web sources below have ALREADY been fetched for you and their text is included — no internet access is needed; do not refuse for lack of browsing ability. They are ordered most-relevant first. Answer the task using ONLY this source text:\n\n` +
         `Task: ${prompt}\n\n` +
         `${combinedText}\n\n` +
         `Rules:\n` +
         `- Answer ONLY from the provided sources; do not use outside knowledge.\n` +
+        `- Read the sources carefully before concluding anything is missing from them.\n` +
         `- Cite the exact source URL(s) you used.\n` +
-        `- If the sources do not contain the answer, say so explicitly.\n` +
+        `- If, after careful reading, the sources genuinely do not contain the answer, say so explicitly.\n` +
         `- NEVER invent or guess a URL; cite only URLs that appear in the sources above.\n\n` +
         `Provide a clear, concise answer.`;
 

package/src/core/AuthManager.js

@@ -644,7 +644,7 @@ class AuthManager {
       case 'serp_rank':
         note = projected === 0
           ? 'DataForSEO not configured — no-op, no credits charged. Set DATAFORSEO_LOGIN/PASSWORD to enable.'
-          : 'DataForSEO SERP API (~US$0.002/call) billed to your own DataForSEO account, separate from the credit cost.';
+          : 'DataForSEO SERP API (~US$0.002 per 10 results of depth — $0.004 at the default depth 20) billed to your own DataForSEO account, separate from the credit cost.';
         break;
       case 'scrape': {
         projected = base;

package/src/server/transports/streamableHttp.js

@@ -30,10 +30,50 @@ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { createServer } from 'node:http';
 import { randomUUID } from 'node:crypto';
 import { readFileSync } from 'node:fs';
+import { z } from 'zod';
+import { zodToJsonSchema } from 'zod-to-json-schema';
 
 const pkg = JSON.parse(readFileSync(new URL('../../../package.json', import.meta.url), 'utf8'));
 const SERVER_VERSION = pkg.version;
 
+/**
+ * Build the `tools` array for the Smithery static server card, straight from
+ * the live tool registry.
+ *
+ * Smithery scans a published server to populate its listing, but our /mcp
+ * endpoint 401s without a key, so the scan cannot enumerate anything. Their
+ * documented fallback is a static server card carrying the metadata — and a
+ * card with no `tools` leaves the listing showing whatever was typed in by
+ * hand at publish time, which is how it goes stale.
+ *
+ * Deriving it here means the card tracks the registry on every release instead
+ * of drifting. Tool `inputSchema`s are registered as ZodRawShapes, so wrap
+ * before converting; a tool whose schema will not convert is still listed,
+ * with an open object schema, rather than dropped.
+ */
+function buildToolCards(server) {
+  const registered = server?._registeredTools ?? {};
+  return Object.entries(registered)
+    .filter(([, tool]) => tool?.enabled !== false)
+    .map(([name, tool]) => {
+      let inputSchema = { type: 'object', properties: {} };
+      try {
+        const shape = tool?.inputSchema;
+        if (shape) {
+          const zodObject = typeof shape?.safeParse === 'function' ? shape : z.object(shape);
+          const converted = zodToJsonSchema(zodObject, { $refStrategy: 'none' });
+          delete converted.$schema;
+          inputSchema = converted;
+        }
+      } catch {
+        // Keep the tool visible with an open schema rather than hiding it.
+      }
+      const card = { name, description: tool?.description ?? tool?.annotations?.title ?? '', inputSchema };
+      if (tool?.annotations) card.annotations = tool.annotations;
+      return card;
+    });
+}
+
 /**
  * The MCP SDK's Protocol.connect() allows at most one active transport per
  * Server/McpServer instance (it throws 'Already connected to a transport'
@@ -163,6 +203,10 @@ export async function connectStreamableHttp(server, authManager, logger, options
           icon: 'https://www.crawlforge.dev/icon.png'
         },
         transport: { type: 'streamable-http', url: '/mcp' },
+        authentication: { required: true, schemes: ['apiKey'] },
+        tools: buildToolCards(server),
+        resources: [],
+        prompts: [],
         configSchema: {
           type: 'object',
           properties: {

package/src/tools/extract/_fetchAndParse.js

@@ -96,6 +96,37 @@ function classifyContentType(contentType) {
   return 'binary';
 }
 
+/**
+ * Flatten a parsed document's <body> to text while preserving line structure.
+ *
+ * Cheerio's .text() joins adjacent elements with no separator, which welds
+ * table rows / list items together ("1.Story title329 points") and starves
+ * downstream LLM extraction (AgentOrchestrator, extractStructured,
+ * extractWithLlm) of any structure to parse. Works on a detached clone so the
+ * caller's $ tree is untouched — unifiedScrape reuses $ for other formats.
+ *
+ * @param {import('cheerio').CheerioAPI} $
+ * @returns {string}
+ */
+export function flattenBodyText($) {
+  // Block boundaries are marked with a U+E000 private-use sentinel so the
+  // HTML source's own insignificant newlines can be collapsed to spaces
+  // first, and only the sentinels become line breaks. (NUL won't survive:
+  // .after()/.replaceWith() parse their argument as HTML and the parser
+  // strips NUL; U+E000 passes through and never occurs in real page text.)
+  const $body = $('body').clone();
+  $body.find('br').replaceWith('\uE000');
+  $body.find('td, th').after(' ');
+  $body
+    .find('p, div, li, tr, h1, h2, h3, h4, h5, h6, blockquote, pre, table, ul, ol, dl, section, article, header, footer')
+    .after('\uE000');
+  return $body
+    .text()
+    .replace(/\s+/g, ' ')
+    .replace(/ ?(?:\uE000 ?)+/g, '\n')
+    .trim();
+}
+
 /**
  * Fetch a URL and return parsed HTML via Cheerio.
  *
@@ -149,7 +180,7 @@ export async function fetchAndParse(url, options = {}) {
     $(stripTags.join(', ')).remove();
   }
 
-  const textContent = $('body').text().replace(/\s+/g, ' ').trim();
+  const textContent = flattenBodyText($);
 
   return { html, $, textContent, finalUrl: response.url };
 }

package/src/tools/search/adapters/dataforseoSearch.js

@@ -11,9 +11,13 @@
  * Endpoint (Live Advanced, synchronous — one request, one response):
  *   POST https://api.dataforseo.com/v3/serp/google/organic/live/advanced
  *
- * Cost: ~US$0.002 per 100 results (depth) on Live Advanced. For high-volume
- * scheduled tracking, DataForSEO's task-based "Standard" queue (task_post →
- * tasks_ready → task_get) is cheaper; swap the endpoint + poll if cost matters.
+ * Cost: US$0.002 per 10 results of `depth` on Live Advanced — measured live,
+ * not estimated: depth 10 bills $0.002, the depth 20 default bills $0.004, and
+ * depth 100 bills $0.02. Deeper scans are also slower (see the timeout note
+ * below), so raise `depth` only when a rank below the default is worth paying
+ * for. For high-volume scheduled tracking, DataForSEO's task-based "Standard"
+ * queue (task_post → tasks_ready → task_get) is cheaper; swap the endpoint +
+ * poll if cost matters.
  */
 
 export class DataForSEOSearchAdapter {
@@ -25,9 +29,16 @@ export class DataForSEOSearchAdapter {
     this.login = login;
     this.password = password;
     this.apiBaseUrl = options.apiBaseUrl || 'https://api.dataforseo.com';
-    // Live Advanced is synchronous and usually answers in a few seconds; cap it
-    // so a hung connection can't wedge the tool. Overridable for tests/self-host.
-    this.timeoutMs = options.timeoutMs ?? 30000;
+    // Live Advanced is synchronous and runs a real-time Google scrape, so its
+    // latency swings widely with their capacity AND with `depth`. Measured on
+    // one account in a single session: ~13-15s at depth 10, and 30s / 45s /
+    // over 60s (twice) for the SAME depth-100 request. A killed request is
+    // still billed — DataForSEO has already run the scrape — so the cap exists
+    // only to stop a hung connection wedging the tool, and sits well above the
+    // slow end rather than through the middle of it.
+    // Overridable via DATAFORSEO_TIMEOUT_MS, or directly for tests/self-host.
+    this.timeoutMs =
+      options.timeoutMs ?? (Number(process.env.DATAFORSEO_TIMEOUT_MS) || 120000);
     // HTTP Basic auth header, computed once.
     this.authHeader = 'Basic ' + Buffer.from(`${login}:${password}`).toString('base64');
   }
@@ -40,7 +51,7 @@ export class DataForSEOSearchAdapter {
    * @param {number} [params.locationCode] - Numeric DataForSEO location code (overrides locationName)
    * @param {string} [params.languageCode='en'] - Language code
    * @param {('desktop'|'mobile')} [params.device='desktop'] - Device to emulate
-   * @param {number} [params.depth=100] - How many results to scan (100 = one page of cost)
+   * @param {number} [params.depth=20] - How many results to scan (billed per 10)
    * @returns {Promise<{items: Array<Object>, meta: Object}>} Normalized organic results + metadata
    */
   async search(params) {
@@ -50,7 +61,7 @@ export class DataForSEOSearchAdapter {
       locationCode,
       languageCode = 'en',
       device = 'desktop',
-      depth = 100,
+      depth = 20,
     } = params;
 
     if (!keyword) {

package/src/tools/search/serpRank.js

@@ -24,7 +24,10 @@ const SerpRankSchema = z.object({
   location_code: z.number().int().optional(),
   language_code: z.string().optional().default('en'),
   device: z.enum(['desktop', 'mobile']).optional().default('desktop'),
-  depth: z.number().int().min(10).max(200).optional().default(100), // DataForSEO caps depth at 200
+  // DataForSEO bills per 10 results scanned and gets slower the deeper it goes,
+  // so the default stays shallow: 20 covers Google's first two pages for $0.004.
+  // Raise it (max 200, their cap) when a deeper position is worth the spend.
+  depth: z.number().int().min(10).max(200).optional().default(20),
 });
 
 /** Reduce a domain or URL to a bare, comparable host: "https://www.Example.com/x" → "example.com". */