crawlforge-mcp-server 5.0.4 → 5.0.5

package/CLAUDE.md

@@ -62,7 +62,7 @@ These guidelines are working if: fewer unnecessary changes in diffs, fewer rewri
 
 CrawlForge MCP Server - A professional MCP (Model Context Protocol) server providing 27 web scraping, crawling, and content processing tools (5 inline + 22 advanced).
 
-**Current Version:** 5.0.4
+**Current Version:** 5.0.5
 
 ## Development Commands
 
@@ -181,7 +181,7 @@ fetch_url, extract_text, extract_links, extract_metadata, scrape_structured
 search_web, serp_rank, crawl_deep, map_site, extract_content, process_document, summarize_content, analyze_content, extract_structured, extract_with_llm, list_ollama_models, batch_scrape, scrape_with_actions, deep_research, track_changes, generate_llms_txt, stealth_mode, localization, scrape_template, scrape, agent
 
 **serp_rank (DataForSEO):**
-- `serp_rank` — reports where a target domain ranks in Google's REAL organic results for a keyword (the SERP position Google Custom Search / `search_web` cannot give). Backed by the DataForSEO Google Organic SERP API (Live Advanced, `POST /v3/serp/google/organic/live/advanced`, HTTP Basic auth). Credentials via `DATAFORSEO_LOGIN` / `DATAFORSEO_PASSWORD`, billed to the user's own DataForSEO account (~US$0.002/call), separate from CrawlForge credits. When unconfigured it returns `{ configured:false }` and charges **0** credits; when configured, **Cost: 5**. Never fabricates a rank.
+- `serp_rank` — reports where a target domain ranks in Google's REAL organic results for a keyword (the SERP position Google Custom Search / `search_web` cannot give). Backed by the DataForSEO Google Organic SERP API (Live Advanced, `POST /v3/serp/google/organic/live/advanced`, HTTP Basic auth). Credentials via `DATAFORSEO_LOGIN` / `DATAFORSEO_PASSWORD`, billed to the user's own DataForSEO account (~US$0.002 per 10 results of `depth`, so $0.004 at the default `depth:20` and $0.02 at `depth:100`), separate from CrawlForge credits. When unconfigured it returns `{ configured:false }` and charges **0** credits; when configured, **Cost: 5**. Never fabricates a rank.
 
 **v4.6.0 additions (Phase D):**
 - `scrape` — single fetch + one cheerio load dispatching a `formats` array (markdown/html/rawHtml/text/links/metadata/screenshot/json-schema) + `onlyMainContent`; partial-success via per-format `warnings[]`. Cost: 2.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.0.4",
+  "version": "5.0.5",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 27 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -127,7 +127,8 @@
     "turndown-plugin-gfm": "^1.0.2",
     "undici": "^7.24.0",
     "winston": "^3.11.0",
-    "zod": "^3.23.8"
+    "zod": "^3.23.8",
+    "zod-to-json-schema": "^3.25.1"
   },
   "optionalDependencies": {
     "camoufox": "^0.1.19"

package/server.js

@@ -99,7 +99,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.0.2",
+  version: "5.0.5",
   description: "Production-ready MCP server with 27 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, real Google SERP rank tracking, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -427,7 +427,7 @@ registerToolIfEnabled("serp_rank", {
     location_code: z.number().optional().describe("Numeric DataForSEO location code (overrides location_name)"),
     language_code: z.string().optional().describe("Language code (e.g. 'en')"),
     device: z.enum(["desktop", "mobile"]).optional().describe("Device to emulate"),
-    depth: z.number().min(10).max(200).optional().describe("How many results to scan, 10-200 (100 = 1 page of cost)")
+    depth: z.number().min(10).max(200).optional().describe("How many results to scan, 10-200 (default 20; DataForSEO bills ~$0.002 per 10 and gets slower the deeper it goes)")
   },
   outputSchema: OUTPUT_SCHEMAS.serp_rank
 }, withAuth("serp_rank", async ({ keyword, target, location_name, location_code, language_code, device, depth }) => {

package/src/core/AuthManager.js

@@ -644,7 +644,7 @@ class AuthManager {
       case 'serp_rank':
         note = projected === 0
           ? 'DataForSEO not configured — no-op, no credits charged. Set DATAFORSEO_LOGIN/PASSWORD to enable.'
-          : 'DataForSEO SERP API (~US$0.002/call) billed to your own DataForSEO account, separate from the credit cost.';
+          : 'DataForSEO SERP API (~US$0.002 per 10 results of depth — $0.004 at the default depth 20) billed to your own DataForSEO account, separate from the credit cost.';
         break;
       case 'scrape': {
         projected = base;

package/src/server/transports/streamableHttp.js

@@ -30,10 +30,50 @@ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { createServer } from 'node:http';
 import { randomUUID } from 'node:crypto';
 import { readFileSync } from 'node:fs';
+import { z } from 'zod';
+import { zodToJsonSchema } from 'zod-to-json-schema';
 
 const pkg = JSON.parse(readFileSync(new URL('../../../package.json', import.meta.url), 'utf8'));
 const SERVER_VERSION = pkg.version;
 
+/**
+ * Build the `tools` array for the Smithery static server card, straight from
+ * the live tool registry.
+ *
+ * Smithery scans a published server to populate its listing, but our /mcp
+ * endpoint 401s without a key, so the scan cannot enumerate anything. Their
+ * documented fallback is a static server card carrying the metadata — and a
+ * card with no `tools` leaves the listing showing whatever was typed in by
+ * hand at publish time, which is how it goes stale.
+ *
+ * Deriving it here means the card tracks the registry on every release instead
+ * of drifting. Tool `inputSchema`s are registered as ZodRawShapes, so wrap
+ * before converting; a tool whose schema will not convert is still listed,
+ * with an open object schema, rather than dropped.
+ */
+function buildToolCards(server) {
+  const registered = server?._registeredTools ?? {};
+  return Object.entries(registered)
+    .filter(([, tool]) => tool?.enabled !== false)
+    .map(([name, tool]) => {
+      let inputSchema = { type: 'object', properties: {} };
+      try {
+        const shape = tool?.inputSchema;
+        if (shape) {
+          const zodObject = typeof shape?.safeParse === 'function' ? shape : z.object(shape);
+          const converted = zodToJsonSchema(zodObject, { $refStrategy: 'none' });
+          delete converted.$schema;
+          inputSchema = converted;
+        }
+      } catch {
+        // Keep the tool visible with an open schema rather than hiding it.
+      }
+      const card = { name, description: tool?.description ?? tool?.annotations?.title ?? '', inputSchema };
+      if (tool?.annotations) card.annotations = tool.annotations;
+      return card;
+    });
+}
+
 /**
  * The MCP SDK's Protocol.connect() allows at most one active transport per
  * Server/McpServer instance (it throws 'Already connected to a transport'
@@ -163,6 +203,10 @@ export async function connectStreamableHttp(server, authManager, logger, options
           icon: 'https://www.crawlforge.dev/icon.png'
         },
         transport: { type: 'streamable-http', url: '/mcp' },
+        authentication: { required: true, schemes: ['apiKey'] },
+        tools: buildToolCards(server),
+        resources: [],
+        prompts: [],
         configSchema: {
           type: 'object',
           properties: {

package/src/tools/search/adapters/dataforseoSearch.js

@@ -11,9 +11,13 @@
  * Endpoint (Live Advanced, synchronous — one request, one response):
  *   POST https://api.dataforseo.com/v3/serp/google/organic/live/advanced
  *
- * Cost: ~US$0.002 per 100 results (depth) on Live Advanced. For high-volume
- * scheduled tracking, DataForSEO's task-based "Standard" queue (task_post →
- * tasks_ready → task_get) is cheaper; swap the endpoint + poll if cost matters.
+ * Cost: US$0.002 per 10 results of `depth` on Live Advanced — measured live,
+ * not estimated: depth 10 bills $0.002, the depth 20 default bills $0.004, and
+ * depth 100 bills $0.02. Deeper scans are also slower (see the timeout note
+ * below), so raise `depth` only when a rank below the default is worth paying
+ * for. For high-volume scheduled tracking, DataForSEO's task-based "Standard"
+ * queue (task_post → tasks_ready → task_get) is cheaper; swap the endpoint +
+ * poll if cost matters.
  */
 
 export class DataForSEOSearchAdapter {
@@ -25,9 +29,16 @@ export class DataForSEOSearchAdapter {
     this.login = login;
     this.password = password;
     this.apiBaseUrl = options.apiBaseUrl || 'https://api.dataforseo.com';
-    // Live Advanced is synchronous and usually answers in a few seconds; cap it
-    // so a hung connection can't wedge the tool. Overridable for tests/self-host.
-    this.timeoutMs = options.timeoutMs ?? 30000;
+    // Live Advanced is synchronous and runs a real-time Google scrape, so its
+    // latency swings widely with their capacity AND with `depth`. Measured on
+    // one account in a single session: ~13-15s at depth 10, and 30s / 45s /
+    // over 60s (twice) for the SAME depth-100 request. A killed request is
+    // still billed — DataForSEO has already run the scrape — so the cap exists
+    // only to stop a hung connection wedging the tool, and sits well above the
+    // slow end rather than through the middle of it.
+    // Overridable via DATAFORSEO_TIMEOUT_MS, or directly for tests/self-host.
+    this.timeoutMs =
+      options.timeoutMs ?? (Number(process.env.DATAFORSEO_TIMEOUT_MS) || 120000);
     // HTTP Basic auth header, computed once.
     this.authHeader = 'Basic ' + Buffer.from(`${login}:${password}`).toString('base64');
   }
@@ -40,7 +51,7 @@ export class DataForSEOSearchAdapter {
    * @param {number} [params.locationCode] - Numeric DataForSEO location code (overrides locationName)
    * @param {string} [params.languageCode='en'] - Language code
    * @param {('desktop'|'mobile')} [params.device='desktop'] - Device to emulate
-   * @param {number} [params.depth=100] - How many results to scan (100 = one page of cost)
+   * @param {number} [params.depth=20] - How many results to scan (billed per 10)
    * @returns {Promise<{items: Array<Object>, meta: Object}>} Normalized organic results + metadata
    */
   async search(params) {
@@ -50,7 +61,7 @@ export class DataForSEOSearchAdapter {
       locationCode,
       languageCode = 'en',
       device = 'desktop',
-      depth = 100,
+      depth = 20,
     } = params;
 
     if (!keyword) {

package/src/tools/search/serpRank.js

@@ -24,7 +24,10 @@ const SerpRankSchema = z.object({
   location_code: z.number().int().optional(),
   language_code: z.string().optional().default('en'),
   device: z.enum(['desktop', 'mobile']).optional().default('desktop'),
-  depth: z.number().int().min(10).max(200).optional().default(100), // DataForSEO caps depth at 200
+  // DataForSEO bills per 10 results scanned and gets slower the deeper it goes,
+  // so the default stays shallow: 20 covers Google's first two pages for $0.004.
+  // Raise it (max 200, their cap) when a deeper position is worth the spend.
+  depth: z.number().int().min(10).max(200).optional().default(20),
 });
 
 /** Reduce a domain or URL to a bare, comparable host: "https://www.Example.com/x" → "example.com". */