crawlforge-mcp-server 3.0.15 → 3.0.17

package/CLAUDE.md

@@ -60,7 +60,7 @@ These guidelines are working if: fewer unnecessary changes in diffs, fewer rewri
 
 ## Project Overview
 
-CrawlForge MCP Server - A professional MCP (Model Context Protocol) server providing 19 web scraping, crawling, and content processing tools.
+CrawlForge MCP Server - A professional MCP (Model Context Protocol) server providing 20 web scraping, crawling, and content processing tools.
 
 **Current Version:** 3.0.12
 
@@ -141,13 +141,13 @@ Tools are organized in subdirectories by category:
 - `tracking/` - trackChanges
 - `llmstxt/` - generateLLMsTxt
 
-### Available MCP Tools (19 total)
+### Available MCP Tools (20 total)
 
 **Basic Tools (server.js inline):**
 fetch_url, extract_text, extract_links, extract_metadata, scrape_structured
 
 **Advanced Tools:**
-search_web, crawl_deep, map_site, extract_content, process_document, summarize_content, analyze_content, batch_scrape, scrape_with_actions, deep_research, track_changes, generate_llms_txt, stealth_mode, localization
+search_web, crawl_deep, map_site, extract_content, process_document, summarize_content, analyze_content, extract_structured, batch_scrape, scrape_with_actions, deep_research, track_changes, generate_llms_txt, stealth_mode, localization
 
 ### MCP Server Entry Point
 

package/README.md

@@ -9,7 +9,7 @@ Professional web scraping and content extraction server implementing the Model C
 
 ## 🎯 Features
 
-- **18 Professional Tools**: Web scraping, deep research, stealth browsing, content analysis
+- **20 Professional Tools**: Web scraping, deep research, stealth browsing, content analysis
 - **Free Tier**: 1,000 credits to get started instantly
 - **MCP Compatible**: Works with Claude, Cursor, and other MCP-enabled AI tools
 - **Enterprise Ready**: Scale up with paid plans for production use
@@ -111,6 +111,8 @@ Restart Cursor to activate.
 - `search_web` - Search the web using Google Search API
 - `summarize_content` - Generate intelligent summaries
 - `analyze_content` - Comprehensive content analysis
+- `extract_structured` - LLM-powered schema-driven extraction
+- `track_changes` - Monitor content changes over time
 
 ### Premium Tools (5-10 credits)
 - `crawl_deep` - Deep crawl entire websites
@@ -136,7 +138,7 @@ Restart Cursor to activate.
 | **Enterprise** | 250,000 | Large scale operations |
 
 **All plans include:**
-- Access to all 18 tools
+- Access to all 20 tools
 - Credits never expire and roll over month-to-month
 - API access and webhook notifications
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "3.0.15",
-  "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 19 comprehensive web scraping, crawling, and content processing tools.",
+  "version": "3.0.17",
+  "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 20 comprehensive web scraping, crawling, and content processing tools.",
   "main": "server.js",
   "bin": {
     "crawlforge": "server.js",
@@ -90,15 +90,15 @@
   ],
   "dependencies": {
     "@googleapis/customsearch": "^5.0.1",
-    "@modelcontextprotocol/sdk": "^1.17.3",
+    "@modelcontextprotocol/sdk": "^1.29.0",
     "@mozilla/readability": "^0.6.0",
     "cheerio": "^1.1.2",
     "compromise": "^14.14.4",
     "diff": "^8.0.2",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",
-    "isomorphic-dompurify": "^2.26.0",
-    "jsdom": "^26.1.0",
+    "isomorphic-dompurify": "^3.9.0",
+    "jsdom": "^29.0.2",
     "lru-cache": "^11.1.0",
     "node-cron": "^3.0.3",
     "node-summarizer": "^1.0.7",
@@ -110,9 +110,18 @@
     "zod": "^3.23.8"
   },
   "devDependencies": {
-    "@jest/globals": "^30.0.5",
+    "@jest/globals": "^30.3.0",
     "cross-env": "^10.0.0",
-    "jest": "^30.0.5",
+    "jest": "^30.3.0",
     "shx": "^0.4.0"
+  },
+  "overrides": {
+    "undici": "^7.24.0",
+    "underscore": "^1.13.8",
+    "qs": "^6.14.2",
+    "path-to-regexp": "^8.4.2",
+    "@hono/node-server": "^1.19.13",
+    "hono": "^4.12.4",
+    "dompurify": "^3.4.0"
   }
 }

package/src/core/ResearchOrchestrator.js

@@ -508,23 +508,28 @@ export class ResearchOrchestrator extends EventEmitter {
 
             if (contentData && contentData.content) {
               this.metrics.contentExtracted++;
-              
+
+              // Normalize content to string (extract_content returns {text: "..."}, fallback returns string)
+              const contentText = typeof contentData.content === 'string'
+                ? contentData.content
+                : (contentData.content.text || JSON.stringify(contentData.content));
+
               // Enhance source with extracted content
               let enhancedSource = {
                 ...source,
-                extractedContent: contentData.content,
+                extractedContent: contentText,
                 metadata: contentData.metadata,
                 structuredData: contentData.structuredData,
                 extractedAt: new Date().toISOString(),
-                wordCount: contentData.content.split(' ').length,
-                readabilityScore: this.calculateReadabilityScore(contentData.content)
+                wordCount: contentText.split(' ').length,
+                readabilityScore: this.calculateReadabilityScore(contentText)
               };
 
               // LLM-powered relevance analysis
               if (this.enableLLMFeatures && topic) {
                 try {
                   const relevanceAnalysis = await this.llmManager.analyzeRelevance(
-                    contentData.content, 
+                    contentText,
                     topic,
                     { maxContentLength: 2000 }
                   );
@@ -546,11 +551,11 @@ export class ResearchOrchestrator extends EventEmitter {
                     error: llmError.message 
                   });
                   // Set default relevance score
-                  enhancedSource.relevanceScore = this.calculateTraditionalRelevance(contentData.content, topic);
+                  enhancedSource.relevanceScore = this.calculateTraditionalRelevance(contentText, topic);
                 }
               } else {
                 // Fallback relevance calculation
-                enhancedSource.relevanceScore = this.calculateTraditionalRelevance(contentData.content, topic);
+                enhancedSource.relevanceScore = this.calculateTraditionalRelevance(contentText, topic);
               }
 
               this.researchState.extractedContent.set(source.link, enhancedSource);