crawlforge-mcp-server 4.9.0 → 5.0.0

package/src/tools/research/deepResearch.js

@@ -4,6 +4,7 @@ import { ElicitationHelper } from '../../core/ElicitationHelper.js';
 import { ResearchOrchestrator } from '../../core/ResearchOrchestrator.js';
 import { getToolConfig } from '../../constants/config.js';
 import { Logger } from '../../utils/Logger.js';
+import { safeFetch } from '../../utils/ssrfGuard.js';
 
 /**
  * DeepResearchTool - MCP tool for conducting comprehensive multi-stage research
@@ -190,12 +191,15 @@ export class DeepResearchTool {
         // Format results according to output preference
         const formattedResults = this.formatResults(researchResults, validated);
 
-        // Clean up session
+        // Capture startTime before deleting the session — looking it up
+        // afterward always returned undefined, so duration was always logged
+        // as 0 (Date.now() - undefined = NaN, then `NaN || 0` = 0).
+        const startTime = this.activeSessions.get(sessionId)?.startTime;
         this.activeSessions.delete(sessionId);
 
-        this.logger.info('Research completed successfully', { 
+        this.logger.info('Research completed successfully', {
           sessionId,
-          duration: Date.now() - this.activeSessions.get(sessionId)?.startTime || 0,
+          duration: startTime ? Date.now() - startTime : 0,
           findingsCount: researchResults.findings?.length || 0
         });
 
@@ -264,7 +268,12 @@ export class DeepResearchTool {
     if (params.llmConfig) {
       baseConfig.llmConfig = params.llmConfig;
     }
-    
+
+    // params.cacheResults was previously dropped entirely — only the
+    // DeepResearchTool-level default (cacheEnabled) ever reached the
+    // orchestrator, so a caller's per-request cacheResults:false was a no-op.
+    baseConfig.cacheEnabled = params.cacheResults;
+
     // Every approach must propagate the user's scope params (maxUrls,
     // timeLimit, concurrency) — only `broad` did before, so non-broad
     // approaches silently fell back to orchestrator defaults.
@@ -279,29 +288,37 @@ export class DeepResearchTool {
       // The orchestrator tunes its query expansion to the approach (commercial
       // vs academic vs current-events); without this it always used academic
       // variations, which poisoned commercial/comparative searches.
-      researchApproach: params.researchApproach
+      researchApproach: params.researchApproach,
+      // The user's explicit choice here was previously dropped entirely —
+      // only per-approach hardcoded values (or the orchestrator's own
+      // default) ever reached the constructor, so e.g.
+      // enableSourceVerification:false still ran verification.
+      enableSourceVerification: params.enableSourceVerification,
+      enableConflictDetection: params.enableConflictDetection
     };
 
     switch (params.researchApproach) {
-      case 'academic':
+      case 'academic': {
+        // Higher weight for academic sources. `rankingWeights` is kept for
+        // back-compat; `rankingOptions.weights` is the key SearchWebTool's
+        // constructor actually reads (searchWeb.js `rankingOptions`).
+        const weights = { authority: 0.4, semantic: 0.3, bm25: 0.2, freshness: 0.1 };
         return {
           ...baseConfig,
           ...scopeConfig,
           maxDepth: Math.min(params.maxDepth, 8),
-          enableSourceVerification: true,
           searchConfig: {
             ...baseConfig.searchConfig,
             enableRanking: true,
-            rankingWeights: {
-              authority: 0.4, // Higher weight for academic sources
-              semantic: 0.3,
-              bm25: 0.2,
-              freshness: 0.1
-            }
+            rankingWeights: weights,
+            rankingOptions: { weights }
           }
         };
+      }
 
-      case 'current_events':
+      case 'current_events': {
+        // Prioritize recent content.
+        const weights = { freshness: 0.4, semantic: 0.3, bm25: 0.2, authority: 0.1 };
         return {
           ...baseConfig,
           ...scopeConfig,
@@ -309,14 +326,11 @@ export class DeepResearchTool {
           searchConfig: {
             ...baseConfig.searchConfig,
             enableRanking: true,
-            rankingWeights: {
-              freshness: 0.4, // Prioritize recent content
-              semantic: 0.3,
-              bm25: 0.2,
-              authority: 0.1
-            }
+            rankingWeights: weights,
+            rankingOptions: { weights }
           }
         };
+      }
 
       case 'focused':
         return {
@@ -327,22 +341,22 @@ export class DeepResearchTool {
           concurrency: Math.min(params.concurrency, 3)
         };
 
-      case 'comparative':
+      case 'comparative': {
+        // `deduplicationThresholds` is kept for back-compat; `deduplicationOptions.thresholds`
+        // is the key SearchWebTool's constructor actually reads.
+        const thresholds = { url: 0.9, title: 0.8, content: 0.7 };
         return {
           ...baseConfig,
           ...scopeConfig,
-          enableConflictDetection: true,
           maxDepth: params.maxDepth,
           searchConfig: {
             ...baseConfig.searchConfig,
             enableDeduplication: true,
-            deduplicationThresholds: {
-              url: 0.9,
-              title: 0.8,
-              content: 0.7
-            }
+            deduplicationThresholds: thresholds,
+            deduplicationOptions: { thresholds }
           }
         };
+      }
 
       case 'broad':
       default:
@@ -713,14 +727,15 @@ export class DeepResearchTool {
         data
       };
 
-      const response = await fetch(webhook.url, {
+      const response = await safeFetch(webhook.url, {
         method: 'POST',
         headers: {
           'Content-Type': 'application/json',
           'User-Agent': 'MCP-WebScraper-DeepResearch/1.0',
           ...webhook.headers
         },
-        body: JSON.stringify(payload)
+        body: JSON.stringify(payload),
+        signal: AbortSignal.timeout(10000)
       });
 
       if (!response.ok) {
@@ -747,10 +762,15 @@ export class DeepResearchTool {
   }
 
   sanitizeConfigForLogging(config) {
-    const { webhook, ...safeConfig } = config;
+    const { webhook, llmConfig, ...safeConfig } = config;
     return {
       ...safeConfig,
-      webhook: webhook ? { url: webhook.url, events: webhook.events } : undefined
+      webhook: webhook ? { url: webhook.url, events: webhook.events } : undefined,
+      llmConfig: llmConfig ? {
+        ...llmConfig,
+        openai: llmConfig.openai ? { ...llmConfig.openai, apiKey: llmConfig.openai.apiKey ? '[redacted]' : undefined } : undefined,
+        anthropic: llmConfig.anthropic ? { ...llmConfig.anthropic, apiKey: llmConfig.anthropic.apiKey ? '[redacted]' : undefined } : undefined
+      } : undefined
     };
   }
 

package/src/tools/scrape/_brandingExtractor.js

@@ -140,19 +140,57 @@ async function collectCssSources($, pageUrl, opts) {
       if (href) hrefs.push(resolveUrl(href, pageUrl));
     });
     const max = clamp(opts.maxStylesheets ?? 10, 0, 20);
-    for (const href of hrefs.slice(0, max)) {
-      try {
-        const res = await safeFetch(href, { signal: AbortSignal.timeout(opts.perFileTimeoutMs ?? 8000) });
-        if (!res.ok) { warnings.push(`branding: stylesheet ${res.status} ${href}`); continue; }
-        let text = await res.text();
-        if (text.length > 512 * 1024) text = text.slice(0, 512 * 1024); // size cap
-        cssText += '\n' + text;
-        fetchedUrls.push(href);
-      } catch (err) {
-        warnings.push(`branding: could not fetch stylesheet ${href} — ${err.message}`);
+    const targets = hrefs.slice(0, max);
+    const perFileTimeoutMs = opts.perFileTimeoutMs ?? 8000;
+    // Single wall-clock budget for ALL stylesheets combined, not just each
+    // one individually — otherwise maxStylesheets slow/unresponsive hosts in
+    // series can still block for maxStylesheets * perFileTimeoutMs (up to
+    // ~160s at the schema max of 20). Capped to whatever the caller's overall
+    // timeoutMs allows, defaulting to 10s.
+    const overallTimeoutMs = opts.timeoutMs != null
+      ? Math.min(opts.timeoutMs, 10000)
+      : (opts.overallTimeoutMs ?? 10000);
+    const deadline = Date.now() + overallTimeoutMs;
+
+    // Small worker pool fetches stylesheets concurrently instead of one at a
+    // time; results are written back by index so cssText concatenation order
+    // stays deterministic regardless of completion order.
+    const concurrency = clamp(opts.stylesheetConcurrency ?? 4, 1, Math.max(targets.length, 1));
+    const results = new Array(targets.length);
+    let cursor = 0;
+
+    async function worker() {
+      while (true) {
+        const remaining = deadline - Date.now();
+        if (remaining <= 0) return; // overall deadline hit — leave rest unfetched
+        const i = cursor++;
+        if (i >= targets.length) return;
+        const href = targets[i];
+        try {
+          const res = await safeFetch(href, { signal: AbortSignal.timeout(Math.min(perFileTimeoutMs, remaining)) });
+          if (!res.ok) { results[i] = { warning: `branding: stylesheet ${res.status} ${href}` }; continue; }
+          let text = await res.text();
+          if (text.length > 512 * 1024) text = text.slice(0, 512 * 1024); // size cap
+          results[i] = { text, href };
+        } catch (err) {
+          results[i] = { warning: `branding: could not fetch stylesheet ${href} — ${err.message}` };
+        }
       }
     }
+
+    await Promise.all(Array.from({ length: concurrency }, () => worker()));
+
+    let skippedByDeadline = 0;
+    for (let i = 0; i < targets.length; i++) {
+      const r = results[i];
+      if (!r) { skippedByDeadline++; continue; }
+      if (r.warning) { warnings.push(r.warning); continue; }
+      cssText += '\n' + r.text;
+      fetchedUrls.push(r.href);
+    }
+
     if (hrefs.length > max) warnings.push(`branding: ${hrefs.length - max} stylesheet(s) skipped (maxStylesheets=${max})`);
+    if (skippedByDeadline > 0) warnings.push(`branding: ${skippedByDeadline} stylesheet(s) skipped — overall CSS fetch deadline (${overallTimeoutMs}ms) exceeded`);
   }
 
   return { cssText, fetchedUrls, styleBlocks, inlineStyleEls, warnings };
@@ -327,7 +365,7 @@ function extractTokens(cssText, cssVariables) {
  * Extract the full branding object from a loaded cheerio $.
  * @param {import('cheerio').CheerioAPI} $
  * @param {string} pageUrl
- * @param {{ fetchLinkedCss?: boolean, maxStylesheets?: number, perFileTimeoutMs?: number }} [opts]
+ * @param {{ fetchLinkedCss?: boolean, maxStylesheets?: number, perFileTimeoutMs?: number, timeoutMs?: number, overallTimeoutMs?: number, stylesheetConcurrency?: number }} [opts]
  * @returns {Promise<object>}
  */
 export async function extractBranding($, pageUrl, opts = {}) {

package/src/tools/scrape/unifiedScrape.js

@@ -52,29 +52,26 @@ export const UnifiedScrapeSchema = z.object({
 
 /**
  * Extract links from a loaded cheerio $ and the page URL.
+ * @param {import('cheerio').CheerioAPI} $
+ * @param {string} pageUrl - final URL of the fetched page (used for origin comparison)
+ * @param {string} [docBaseUrl] - resolution base for relative hrefs; defaults to pageUrl.
+ *   Pass the resolved <base href> here when the document declares one.
  */
-function extractLinksFromDom($, pageUrl) {
+function extractLinksFromDom($, pageUrl, docBaseUrl) {
   const links = [];
   const seen = new Set();
   let pageOrigin = '';
   try { pageOrigin = new URL(pageUrl).origin; } catch { /* ignore */ }
+  const resolveBase = docBaseUrl || pageUrl;
 
   $('a[href]').each((_, el) => {
     const href = $(el).attr('href');
     const text = $(el).text().trim();
     if (!href) return;
+    if (href.startsWith('#') || href.startsWith('javascript:')) return;
     try {
-      let absoluteUrl;
-      let isExternal = false;
-      if (href.startsWith('http://') || href.startsWith('https://')) {
-        absoluteUrl = href;
-        isExternal = new URL(href).origin !== pageOrigin;
-      } else if (href.startsWith('#') || href.startsWith('javascript:')) {
-        return;
-      } else {
-        absoluteUrl = new URL(href, pageUrl).toString();
-        isExternal = false;
-      }
+      const absoluteUrl = new URL(href, resolveBase).toString();
+      const isExternal = new URL(absoluteUrl).origin !== pageOrigin;
       if (!seen.has(absoluteUrl)) {
         seen.add(absoluteUrl);
         links.push({ href: absoluteUrl, text, is_external: isExternal, original_href: href });
@@ -208,6 +205,14 @@ export class UnifiedScrapeTool {
       throw new Error(`scrape: fetch failed for ${url}: ${err.message}`);
     }
 
+    // Resolve <base href> once per document (if present) so link resolution
+    // matches how a browser would navigate, instead of always using finalUrl.
+    let docBaseUrl = finalUrl;
+    try {
+      const baseHref = $('base[href]').first().attr('href');
+      if (baseHref) docBaseUrl = new URL(baseHref, finalUrl).toString();
+    } catch { /* ignore invalid <base href>, fall back to finalUrl */ }
+
     // For onlyMainContent: extract main-content html via Readability once
     let mainHtml = null;
     function getMainHtml() {
@@ -279,15 +284,18 @@ export class UnifiedScrapeTool {
 
         case 'text':
           try {
+            const { load } = await import('cheerio');
             if (onlyMainContent) {
               // Plain text from Readability main content via cheerio
-              const { load } = await import('cheerio');
               const $main = load(getMainHtml());
               $main('script, style').remove();
               content.text = extractBlockText($main);
             } else {
-              $('script, style').remove();
-              content.text = extractBlockText($);
+              // Strip script/style on a clone, not the shared $, so other
+              // formats reading $ later aren't affected by format ordering.
+              const $clone = load($.html());
+              $clone('script, style').remove();
+              content.text = extractBlockText($clone);
             }
           } catch (err) {
             content.text = '';
@@ -297,7 +305,7 @@ export class UnifiedScrapeTool {
 
         case 'links':
           try {
-            content.links = extractLinksFromDom($, finalUrl);
+            content.links = extractLinksFromDom($, finalUrl, docBaseUrl);
           } catch (err) {
             content.links = { links: [], total_count: 0, internal_count: 0, external_count: 0 };
             warnings.push(`links: ${err.message}`);
@@ -348,7 +356,9 @@ export class UnifiedScrapeTool {
               { headless: true, timeout: 30000 }
             );
             content.screenshots = Array.isArray(r?.screenshots) ? r.screenshots : [];
-            if (content.screenshots.length === 0) {
+            if (r?.success === false) {
+              warnings.push(`screenshot: ${r.error || 'capture failed'}`);
+            } else if (content.screenshots.length === 0) {
               warnings.push('screenshot: capture produced no image');
             }
           } catch (err) {

package/src/tools/search/providers/searxng.js

@@ -92,9 +92,13 @@ export async function searchViaSearxng(opts = {}) {
   let response;
   try {
     response = await fetch(url.toString(), {
-      headers: { Accept: 'application/json' }
+      headers: { Accept: 'application/json' },
+      signal: AbortSignal.timeout(15000)
     });
   } catch (err) {
+    if (err.name === 'TimeoutError' || err.name === 'AbortError') {
+      throw new Error('SearXNG request failed: timed out after 15000ms');
+    }
     throw new Error(`SearXNG request failed: ${err.message}`);
   }
 

package/src/tools/search/ranking/ResultDeduplicator.js

@@ -88,7 +88,15 @@ export class ResultDeduplicator {
       return results;
     }
 
-    const dedupeOptions = { ...this.options, ...options };
+    // Deep-merge thresholds instead of replacing wholesale — a caller
+    // passing a partial thresholds object (e.g. {url: 0.8}) previously wiped
+    // out the other thresholds, leaving them `undefined` and silently
+    // disabling title/content/combined duplicate detection.
+    const dedupeOptions = {
+      ...this.options,
+      ...options,
+      thresholds: { ...this.options.thresholds, ...(options.thresholds || {}) }
+    };
     this.stats.totalProcessed += results.length;
     
     // Generate cache key for deduplication computation

package/src/tools/search/ranking/ResultRanker.js

@@ -73,8 +73,19 @@ export class ResultRanker {
       return [];
     }
 
-    const rankingOptions = { ...this.options, ...options };
-    
+    // Deep-merge nested option objects instead of replacing them wholesale —
+    // a caller passing a partial weights object (e.g. {bm25: 0.7}) previously
+    // wiped out the other three weights, leaving them `undefined` and making
+    // computeFinalScore return NaN for every result.
+    const rankingOptions = {
+      ...this.options,
+      ...options,
+      weights: { ...this.options.weights, ...(options.weights || {}) },
+      bm25: { ...this.options.bm25, ...(options.bm25 || {}) },
+      authority: { ...this.options.authority, ...(options.authority || {}) },
+      freshness: { ...this.options.freshness, ...(options.freshness || {}) }
+    };
+
     // Generate cache key for ranking computation
     const cacheKey = this.cache ? this.cache.generateKey('ranking', {
       query,
@@ -179,6 +190,10 @@ export class ResultRanker {
 
     // Tokenize query and content
     const queryTerms = this.tokenize(query.toLowerCase());
+    // Queries whose tokens are all <=1 char after tokenization (e.g. "C#")
+    // tokenize to []; without this guard `score / queryTerms.length` below
+    // is 0/0 = NaN, which then poisons finalScore for every result.
+    if (queryTerms.length === 0) return 0;
     const contentTerms = this.tokenize(content);
     const contentLength = contentTerms.length;
 

package/src/tools/search/searchWeb.js

@@ -186,12 +186,18 @@ export class SearchWebTool {
         }
       }
       
-      // Try searches with expanded queries, starting with the best one
+      // Try searches with expanded queries, starting with the best one.
+      // Each retry (triggered when the previous query returned zero items) is
+      // a separate billed backend search — cap attempts so one search_web
+      // call can't silently fan out into up to maxExpansions backend requests.
+      const MAX_SEARCH_ATTEMPTS = 2;
+      const queriesToTry = searchQueries.slice(0, MAX_SEARCH_ATTEMPTS);
       let bestResults = null;
       let usedQuery = validated.query;
       let searchError = null;
-      
-      for (let i = 0; i < searchQueries.length; i++) {
+      let searchAttempts = 0;
+
+      for (let i = 0; i < queriesToTry.length; i++) {
         try {
           // Build search query with modifiers
           let searchQuery = searchQueries[i];
@@ -219,32 +225,34 @@ export class SearchWebTool {
           };
           
           const results = await this.searchAdapter.search(searchParams);
-          
+          searchAttempts++;
+
           // Check if we got good results
           if (results.items && results.items.length > 0) {
             bestResults = results;
-            usedQuery = searchQueries[i];
+            usedQuery = queriesToTry[i];
             break;
           } else if (i === 0) {
             // Save results from first query even if no items (might be the original query)
             bestResults = results;
-            usedQuery = searchQueries[i];
+            usedQuery = queriesToTry[i];
           }
         } catch (error) {
           searchError = error;
-          console.warn(`Search failed for query "${searchQueries[i]}":`, error.message);
-          
+          searchAttempts++;
+          console.warn(`Search failed for query "${queriesToTry[i]}":`, error.message);
+
           // If this is the last query and we haven't found results, throw the error
-          if (i === searchQueries.length - 1 && !bestResults) {
+          if (i === queriesToTry.length - 1 && !bestResults) {
             throw error;
           }
         }
       }
-      
+
       if (!bestResults) {
         throw searchError || new Error('All search queries failed');
       }
-      
+
       // Process and enrich results
       let processedResults = await this.processResults(bestResults);
       
@@ -282,7 +290,10 @@ export class SearchWebTool {
         
         rankingInfo = {
           algorithmsUsed: ['bm25', 'semantic', 'authority', 'freshness'],
-          weightsApplied: this.resultRanker.options.weights,
+          // rankingDetails.weights carries the actually-applied (merged) weights
+          // for this call; this.resultRanker.options.weights is only the
+          // constructor default and previously misreported partial overrides.
+          weightsApplied: processedResults[0]?.rankingDetails?.weights || this.resultRanker.options.weights,
           totalResults: processedResults.length
         };
       }
@@ -341,7 +352,11 @@ export class SearchWebTool {
           query_expansion: localizedParams.expand_query && expandedQueries.length > 1 ? {
             original_query: validated.query,
             expanded_count: expandedQueries.length,
-            used_query: usedQuery
+            used_query: usedQuery,
+            // Backend searches actually issued for this call (capped at
+            // MAX_SEARCH_ATTEMPTS) — surfaces the retry/billing cost that was
+            // previously invisible.
+            search_attempts: searchAttempts
           } : null,
           localization_applied: !!validated.localization
         }
@@ -411,7 +426,9 @@ export class SearchWebTool {
       );
       rankingInfo = {
         algorithmsUsed: ['bm25', 'semantic', 'authority', 'freshness'],
-        weightsApplied: this.resultRanker.options.weights,
+        // See execute()'s equivalent block: rankingDetails.weights carries the
+        // actually-applied (merged) weights for this call.
+        weightsApplied: processedResults[0]?.rankingDetails?.weights || this.resultRanker.options.weights,
         totalResults: processedResults.length
       };
     }

package/src/tools/search/serpRank.js

@@ -12,6 +12,11 @@
 import { z } from 'zod';
 import { DataForSEOSearchAdapter } from './adapters/dataforseoSearch.js';
 
+/** How many top organic results to return as the SERP listing (`results`).
+ * The first Google page is ~10; bounding it keeps the tool payload small while
+ * still surfacing the competitors that matter. */
+const RESULTS_LIMIT = 10;
+
 const SerpRankSchema = z.object({
   keyword: z.string().min(1),
   target: z.string().min(1), // domain or URL to locate in the SERP
@@ -94,6 +99,23 @@ export class SerpRankTool {
 
     const best = matches[0] || null;
 
+    // The SERP listing itself — the top organic competitors as Google actually
+    // ranks them, not just the target. Bounded to the first page so the payload
+    // stays small; each item already carries { position, rankAbsolute, domain,
+    // url, title, snippet } from the adapter. This is what "SERP results" means.
+    const results = items
+      .slice()
+      .sort((a, b) => (a.position ?? Infinity) - (b.position ?? Infinity))
+      .slice(0, RESULTS_LIMIT)
+      .map((it) => ({
+        position: it.position,
+        rankAbsolute: it.rankAbsolute,
+        domain: it.domain,
+        url: it.url,
+        title: it.title,
+        snippet: it.snippet,
+      }));
+
     return {
       configured: true,
       keyword: validated.keyword,
@@ -104,6 +126,7 @@ export class SerpRankTool {
       url: best ? best.url : null,
       title: best ? best.title : null,
       allPositions: matches, // every place the domain ranks on this SERP
+      results, // the top organic results (the SERP listing), best-first, capped
       location: meta.location,
       device: meta.device,
       depthScanned: meta.depth,

package/src/tools/templates/TemplateRegistry.js

@@ -113,7 +113,13 @@ const TEMPLATES = [
         description: attr($, 'meta[property="og:description"]', 'content'),
         thumbnail: attr($, 'meta[property="og:image"]', 'content'),
         duration: attr($, 'meta[itemprop="duration"]', 'content'),
-        video_id: new URL($('link[rel="canonical"]').attr('href') || 'https://youtube.com').searchParams.get('v')
+        video_id: (() => {
+          try {
+            return new URL($('link[rel="canonical"]').attr('href') || 'https://youtube.com').searchParams.get('v');
+          } catch {
+            return null;
+          }
+        })()
       };
     }
   },