crawlforge-mcp-server 5.2.7 → 5.2.9

package/CLAUDE.md

@@ -62,7 +62,7 @@ These guidelines are working if: fewer unnecessary changes in diffs, fewer rewri
 
 CrawlForge MCP Server - A professional MCP (Model Context Protocol) server providing 28 web scraping, crawling, and content processing tools (5 inline + 23 advanced).
 
-**Current Version:** 5.2.7
+**Current Version:** 5.2.9
 
 ## Development Commands
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.2.7",
+  "version": "5.2.9",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 28 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",

package/server.js

@@ -100,7 +100,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.2.7",
+  version: "5.2.9",
   description: "Production-ready MCP server with 28 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",

package/src/core/ResearchOrchestrator.js

@@ -1254,7 +1254,7 @@ export class ResearchOrchestrator extends EventEmitter {
     return claimGroups
       .filter(group => group.sourceCount >= 2 && group.avgCredibility >= 0.6)
       .map(group => ({
-        topic: group.keywords.join(' '),
+        topic: this.claimGroupLabel(group),
         supportingClaims: group.claims.length,
         supportingSources: group.sourceCount,
         averageCredibility: group.avgCredibility,
@@ -1510,27 +1510,37 @@ export class ResearchOrchestrator extends EventEmitter {
     });
   }
 
+  // A claim group rendered for humans is its most credible claim. Claims are
+  // extractive sentences from source content, so this is readable prose —
+  // joining the group's keywords produces stopword-stripped gibberish
+  // ("scraping server model context protocol server that...").
+  mostCredibleClaim(group) {
+    return group.claims.reduce(
+      (best, c) => ((c.credibility || 0) > (best.credibility || 0) ? c : best),
+      group.claims[0]
+    );
+  }
+
+  // Compact label for a claim group: its best claim, cut at a word break.
+  claimGroupLabel(group, maxChars = 120) {
+    const claim = this.mostCredibleClaim(group)?.claim || '';
+    if (claim.length <= maxChars) return claim;
+    const cut = claim.slice(0, maxChars);
+    const lastSpace = cut.lastIndexOf(' ');
+    return (lastSpace > 40 ? cut.slice(0, lastSpace) : cut) + '…';
+  }
+
   generateKeyFindings(claimGroups, sources) {
     return claimGroups
       .filter(group => group.avgCredibility >= this.credibilityThreshold)
       .sort((a, b) => b.consensusStrength - a.consensusStrength)
       .slice(0, 10)
-      .map(group => {
-        // Findings must be readable prose. Claims are extractive sentences
-        // from the source content, so surface the most credible one verbatim
-        // — joining the group's keywords here produced stopword-stripped
-        // gibberish ("scraping server model context protocol server that...").
-        const representative = group.claims.reduce(
-          (best, c) => ((c.credibility || 0) > (best.credibility || 0) ? c : best),
-          group.claims[0]
-        );
-        return {
-          finding: representative.claim,
-          supportingClaims: group.claims.length,
-          credibility: group.avgCredibility,
-          sources: group.claims.map(c => c.source)
-        };
-      });
+      .map(group => ({
+        finding: this.mostCredibleClaim(group).claim,
+        supportingClaims: group.claims.length,
+        credibility: group.avgCredibility,
+        sources: group.claims.map(c => c.source)
+      }));
   }
 
   compileSupportingEvidence(sources) {
@@ -1584,10 +1594,11 @@ export class ResearchOrchestrator extends EventEmitter {
     );
     
     weakAreas.forEach(area => {
+      const label = this.claimGroupLabel(area);
       gaps.push({
-        area: area.keywords.join(' '),
+        area: label,
         issue: 'Limited reliable sources',
-        suggestion: `More research needed on ${area.keywords.join(' ')} related to ${topic}`
+        suggestion: `Corroborate with additional sources: "${label}"`
       });
     });
     
@@ -1609,7 +1620,7 @@ export class ResearchOrchestrator extends EventEmitter {
       recommendations.push({
         type: 'gap_filling',
         priority: 'medium',
-        description: `Address research gaps in ${synthesis.gaps.map(g => g.area).join(', ')}`
+        description: `Address ${synthesis.gaps.length} under-sourced claim(s) — see researchGaps, e.g. "${synthesis.gaps[0].area}"`
       });
     }
     

package/src/core/llm/LLMManager.js

@@ -259,7 +259,9 @@ Return a JSON object with:
   "keyPoints": ["point1", "point2", ...],
   "topicAlignment": "description of alignment",
   "credibilityIndicators": ["indicator1", "indicator2", ...]
-}`;
+}
+
+Be brief: at most 5 items per array, one short sentence each.`;
 
     const prompt = `Research Topic: "${topic}"
 
@@ -269,19 +271,48 @@ ${truncatedContent}
 Analyze the relevance of this content to the research topic:`;
 
     try {
-      const response = await this.generateCompletion(prompt, {
-        systemPrompt,
-        maxTokens: 500,
-        temperature: 0.3
-      });
-
-      const analysis = JSON.parse(response);
-      return {
-        relevanceScore: Math.max(0, Math.min(1, analysis.relevanceScore || 0.5)),
-        keyPoints: analysis.keyPoints || [],
-        topicAlignment: analysis.topicAlignment || '',
-        credibilityIndicators: analysis.credibilityIndicators || []
+      // Same discipline as synthesizeFindings: constrain the output shape
+      // (small local models otherwise wrap the JSON in markdown fences —
+      // the raw JSON.parse here failed on every Ollama run), strip fences,
+      // validate the load-bearing field, and retry a truncated response
+      // once before falling back.
+      const relevanceSchema = {
+        type: 'object',
+        properties: {
+          relevanceScore: { type: 'number' },
+          keyPoints: { type: 'array', items: { type: 'string' } },
+          topicAlignment: { type: 'string' },
+          credibilityIndicators: { type: 'array', items: { type: 'string' } }
+        },
+        required: ['relevanceScore']
       };
+
+      let lastError;
+      for (let attempt = 0; attempt < 2; attempt++) {
+        try {
+          const response = await this.generateCompletion(prompt, {
+            systemPrompt,
+            maxTokens: 800,
+            temperature: 0.3,
+            format: relevanceSchema
+          });
+
+          const cleaned = response.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '').trim();
+          const analysis = JSON.parse(cleaned);
+          if (!analysis || typeof analysis.relevanceScore !== 'number') {
+            throw new Error('Relevance response missing relevanceScore');
+          }
+          return {
+            relevanceScore: Math.max(0, Math.min(1, analysis.relevanceScore)),
+            keyPoints: analysis.keyPoints || [],
+            topicAlignment: analysis.topicAlignment || '',
+            credibilityIndicators: analysis.credibilityIndicators || []
+          };
+        } catch (error) {
+          lastError = error;
+        }
+      }
+      throw lastError;
     } catch (error) {
       this.logger.warn('LLM relevance analysis failed, using fallback', { error: error.message });
       return this.fallbackRelevanceAnalysis(content, topic);
@@ -306,10 +337,18 @@ Generate a JSON response with:
   "confidence": 0.0-1.0,
   "gaps": ["gap1", "gap2", ...],
   "recommendations": ["rec1", "rec2", ...]
-}`;
+}
+
+Be brief: summary at most 3 sentences; each array at most 5 items, one short sentence each.`;
 
     const findingsText = limitedFindings
-      .map((finding, index) => `${index + 1}. ${finding.finding || finding.text || finding}`)
+      .map((finding, index) => {
+        // A finding can be a whole flattened page section (sitemap dumps run
+        // 1500+ chars). Passing it whole bloats the prompt and pulls a long
+        // answer that overruns the token budget, truncating the JSON.
+        const text = String(finding.finding || finding.text || finding);
+        return `${index + 1}. ${text.length > 300 ? text.slice(0, 300) + '…' : text}`;
+      })
       .join('\n');
 
     const prompt = `Research Topic: "${topic}"
@@ -337,20 +376,32 @@ Synthesize these findings into a comprehensive analysis:`;
         required: ['summary', 'keyInsights', 'themes', 'confidence']
       };
 
-      const response = await this.generateCompletion(prompt, {
-        systemPrompt,
-        maxTokens: 800,
-        temperature: 0.4,
-        format: synthesisSchema
-      });
-
-      // Strip markdown code fences if present
-      const cleaned = response.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '').trim();
-      const parsed = JSON.parse(cleaned);
-      if (!parsed || typeof parsed.summary !== 'string' || parsed.summary.length === 0) {
-        throw new Error('Synthesis response missing summary');
+      // Two attempts: small local models occasionally overrun the token
+      // budget mid-string, and a truncated response cannot be parsed. 1600
+      // tokens gives the brevity-capped answer ~2x headroom (800 truncated
+      // roughly two runs in three on real findings).
+      let lastError;
+      for (let attempt = 0; attempt < 2; attempt++) {
+        try {
+          const response = await this.generateCompletion(prompt, {
+            systemPrompt,
+            maxTokens: 1600,
+            temperature: 0.4,
+            format: synthesisSchema
+          });
+
+          // Strip markdown code fences if present
+          const cleaned = response.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '').trim();
+          const parsed = JSON.parse(cleaned);
+          if (!parsed || typeof parsed.summary !== 'string' || parsed.summary.length === 0) {
+            throw new Error('Synthesis response missing summary');
+          }
+          return parsed;
+        } catch (error) {
+          lastError = error;
+        }
       }
-      return parsed;
+      throw lastError;
     } catch (error) {
       this.logger.warn('LLM synthesis failed, using fallback', { error: error.message });
       return this.fallbackSynthesis(findings, topic);