crawlforge-mcp-server 4.7.0 → 4.7.1

package/README.md

@@ -199,7 +199,7 @@ For the full canonical capabilities reference (all tools, CLI commands, stealth
 | **Business** ($399) | 250,000 | Large scale operations |
 
 **All plans include:**
-- Access to all 26 tools (the 15 local tools never consume credits)
+- Access to all 26 tools
 - Credits never expire and roll over month-to-month
 - API access and webhook notifications
 
@@ -298,7 +298,7 @@ Once configured, use these tools in your AI assistant:
 
 ## 🔒 Security & Privacy
 
-- **Secure Authentication**: API keys required for all metered premium tools (the 15 free local tools run without one)
+- **Secure Authentication**: API keys required for all metered tools
 - **Local Storage**: API keys stored securely at `~/.crawlforge/config.json`
 - **HTTPS Only**: All connections use encrypted HTTPS
 - **No Data Retention**: We don't store scraped data, only usage logs

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "4.7.0",
+  "version": "4.7.1",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 26 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",

package/server.js

@@ -89,7 +89,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "4.7.0",
+  version: "4.7.1",
   description: "Production-ready MCP server with 26 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png"

package/src/core/LLMsTxtAnalyzer.js

@@ -50,7 +50,16 @@ export class LLMsTxtAnalyzer {
       apis: [],
       contentTypes: {},
       securityAreas: [],
-      rateLimit: {},
+      // Conservative defaults so output never renders `undefined` when live
+      // rate-limit probing is skipped (analyzeRateLimiting only runs with
+      // probeRateLimit:true). Overwritten with measured values when probed.
+      rateLimit: {
+        recommendedDelay: 1000,
+        maxConcurrency: 5,
+        recommendedRPM: 30,
+        reasoning: 'Conservative defaults applied; live rate-limit probing was not performed (pass probeRateLimit:true to measure actual response times).',
+        averageResponseTime: null
+      },
       guidelines: {},
       metadata: {},
       errors: []

package/src/core/ResearchOrchestrator.js

@@ -32,6 +32,7 @@ export class ResearchOrchestrator extends EventEmitter {
       concurrency = 5,
       enableSourceVerification = true,
       enableConflictDetection = true,
+      credibilityThreshold = 0.3,
       cacheEnabled = true,
       cacheTTL = 1800000, // 30 minutes
       researchApproach = 'broad',
@@ -61,6 +62,7 @@ export class ResearchOrchestrator extends EventEmitter {
     this.concurrency = Math.min(Math.max(1, concurrency), 20);
     this.enableSourceVerification = enableSourceVerification;
     this.enableConflictDetection = enableConflictDetection;
+    this.credibilityThreshold = Math.min(Math.max(0, credibilityThreshold), 1);
 
     // Stealth fallback config + lazy state (browser launched only on first block)
     this.enableStealthFallback = enableStealthFallback;
@@ -859,7 +861,7 @@ export class ResearchOrchestrator extends EventEmitter {
         }
 
         // Only include sources that meet minimum credibility threshold
-        if (overallCredibility >= 0.3) {
+        if (overallCredibility >= this.credibilityThreshold) {
           verifiedSources.push({
             ...source,
             credibilityFactors,
@@ -1360,7 +1362,7 @@ export class ResearchOrchestrator extends EventEmitter {
 
   generateKeyFindings(claimGroups, sources) {
     return claimGroups
-      .filter(group => group.avgCredibility >= 0.3)
+      .filter(group => group.avgCredibility >= this.credibilityThreshold)
       .sort((a, b) => b.consensusStrength - a.consensusStrength)
       .slice(0, 10)
       .map(group => ({
@@ -1373,7 +1375,7 @@ export class ResearchOrchestrator extends EventEmitter {
 
   compileSupportingEvidence(sources) {
     return sources
-      .filter(source => source.overallCredibility >= 0.3)
+      .filter(source => source.overallCredibility >= this.credibilityThreshold)
       .map(source => ({
         title: source.title,
         url: source.link,

package/src/tools/llmstxt/generateLLMsTxt.js

@@ -391,7 +391,9 @@ export class GenerateLLMsTxtTool {
       lines.push('');
       lines.push('### Technical Justification');
       lines.push(`${analysis.rateLimit.reasoning}`);
-      lines.push(`Average response time: ${analysis.rateLimit.averageResponseTime}ms`);
+      if (analysis.rateLimit.averageResponseTime != null) {
+        lines.push(`Average response time: ${analysis.rateLimit.averageResponseTime}ms`);
+      }
       lines.push('');
     }
 

package/src/tools/research/deepResearch.js

@@ -272,6 +272,10 @@ export class DeepResearchTool {
       maxUrls: params.maxUrls,
       timeLimit: params.timeLimit,
       concurrency: params.concurrency,
+      // Minimum credibility a source must clear in verifySourceCredibility.
+      // Must be on the orchestrator *constructor* config (not the
+      // conductResearch options) — that is the only place it is now read.
+      credibilityThreshold: params.credibilityThreshold,
       // The orchestrator tunes its query expansion to the approach (commercial
       // vs academic vs current-events); without this it always used academic
       // variations, which poisoned commercial/comparative searches.
@@ -356,7 +360,6 @@ export class DeepResearchTool {
   buildResearchOptions(params) {
     return {
       sourceTypes: params.sourceTypes,
-      credibilityThreshold: params.credibilityThreshold,
       includeRecentOnly: params.includeRecentOnly,
       queryExpansion: params.queryExpansion,
       enableConflictDetection: params.enableConflictDetection,