crawlforge-mcp-server 6.21.1 → 6.22.0

package/src/tools/search/providers/searxng.js

@@ -94,6 +94,7 @@ export async function searchViaSearxng(opts = {}) {
   url.searchParams.set('safesearch', String(safesearch));
   url.searchParams.set('language', language);
 
+  const startedAt = Date.now();
   let response;
   try {
     response = await fetch(url.toString(), {
@@ -104,7 +105,10 @@ export async function searchViaSearxng(opts = {}) {
     if (err.name === 'TimeoutError' || err.name === 'AbortError') {
       throw new Error('SearXNG request failed: timed out after 15000ms');
     }
-    throw new Error(`SearXNG request failed: ${err.message}`);
+    throw new Error(
+      `SearXNG is not reachable at ${instanceUrl}: ${err.cause?.code || err.message}.\n` +
+        'Next step: start the instance (for the Docker container: "docker start searxng"), or call search_web without provider:"searxng".'
+    );
   }
 
   if (!response.ok) {
@@ -121,13 +125,26 @@ export async function searchViaSearxng(opts = {}) {
   }
 
   const rawResults = Array.isArray(data.results) ? data.results : [];
+
+  // Every engine failing still comes back HTTP 200 with no results; that is
+  // not an empty search (R25 3.3). Entries are [engine, reason] pairs.
+  const unresponsive = Array.isArray(data.unresponsive_engines) ? data.unresponsive_engines : [];
+  if (rawResults.length === 0 && unresponsive.length > 0) {
+    const engines = unresponsive.map((e) => (Array.isArray(e) ? `${e[0]} (${e[1]})` : String(e))).join(', ');
+    throw new Error(
+      `SearXNG returned no results because its engines failed: ${engines}.\n` +
+        'Next step: retry in a minute, or call search_web without provider:"searxng".'
+    );
+  }
+
   const items = rawResults.slice(0, limit).map(normalizeSearxngResult);
 
   return {
     items,
     searchInformation: {
-      totalResults: rawResults.length,
-      searchTime: 0
+      // SearXNG reports 0 when no engine gave a count; that is unknown, not zero.
+      totalResults: data.number_of_results || null,
+      searchTime: (Date.now() - startedAt) / 1000
     },
     queries: {},
     context: {}

package/src/tools/search/ranking/ResultDeduplicator.js

@@ -119,7 +119,6 @@ export class ResultDeduplicator {
         ...result,
         originalIndex: index,
         normalizedUrl: this.normalizeUrl(result.link || result.url, dedupeOptions.urlNormalization),
-        contentHash: this.computeContentHash(result, dedupeOptions.contentSimilarity),
         titleTokens: this.tokenizeTitle(result.title || ''),
         deduplicationInfo: {
           originalUrl: result.link || result.url,
@@ -188,11 +187,13 @@ export class ResultDeduplicator {
       const currentGroup = [results[i]];
       processed.add(i);
 
-      // Find all duplicates of current result
+      // Find all duplicates of current result. A result joins the group only
+      // when it duplicates EVERY member, so a hub page (e.g. MDN `Array`)
+      // cannot pull in its children one by one.
       for (let j = i + 1; j < results.length; j++) {
         if (processed.has(j)) continue;
 
-        if (this.areDuplicates(results[i], results[j], options)) {
+        if (currentGroup.every(member => this.areDuplicates(member, results[j], options))) {
           currentGroup.push(results[j]);
           processed.add(j);
           
@@ -216,21 +217,24 @@ export class ResultDeduplicator {
   areDuplicates(result1, result2, options) {
     const similarities = this.computeSimilarities(result1, result2, options);
 
-    // Near-identical URLs are always duplicates (e.g. http vs https of same page)
-    if (similarities.url >= 0.95) {
+    // The same page written differently (protocol, www, trailing slash,
+    // index.html, tracking parameters) normalizes to the same URL.
+    if (result1.normalizedUrl && result1.normalizedUrl === result2.normalizedUrl) {
       this.stats.urlDuplicates++;
       return true;
     }
 
-    // Require at least 2 high-similarity signals to mark as duplicate
-    let matchCount = 0;
-    if (similarities.url >= options.thresholds.url) matchCount++;
-    if (similarities.title >= options.thresholds.title) matchCount++;
-    if (similarities.content >= options.thresholds.content) matchCount++;
+    // Different URLs merge only on similar content. Sibling pages share a
+    // URL prefix and a site title template (" - JavaScript | MDN"), so URL
+    // and title similarity alone never prove two pages are the same.
+    if (similarities.content < options.thresholds.content) {
+      return false;
+    }
 
-    if (matchCount >= 2) {
+    // Content plus one more high-similarity signal
+    if (similarities.url >= options.thresholds.url || similarities.title >= options.thresholds.title) {
       if (similarities.title >= options.thresholds.title) this.stats.titleDuplicates++;
-      if (similarities.content >= options.thresholds.content) this.stats.contentDuplicates++;
+      this.stats.contentDuplicates++;
       return true;
     }
 
@@ -294,28 +298,23 @@ export class ResultDeduplicator {
   }
 
   /**
-   * Compute content similarity using SimHash
+   * Compute content similarity: Jaccard over the snippets' words. A 64-bit
+   * SimHash of a ~30-word snippet is too noisy here: unrelated MDN sibling
+   * snippets scored up to 0.86 against the 0.85 threshold (R25 1.3).
    */
   computeContentSimilarity(result1, result2, contentOptions) {
     const content1 = this.extractContent(result1);
     const content2 = this.extractContent(result2);
-    
+
     if (content1.length < contentOptions.minLength || content2.length < contentOptions.minLength) {
       return 0;
     }
-    
-    // Use pre-computed content hashes
-    const hash1 = result1.contentHash;
-    const hash2 = result2.contentHash;
-    
-    // Compute Hamming distance between hashes
-    const hammingDistance = this.hammingDistance(hash1, hash2);
-    
-    // Convert to similarity score
-    const maxDistance = contentOptions.simhashBits;
-    const similarity = 1 - (hammingDistance / maxDistance);
-    
-    return Math.max(0, similarity);
+
+    const words1 = new Set(content1.split(' '));
+    const words2 = new Set(content2.split(' '));
+    const intersection = [...words1].filter(word => words2.has(word)).length;
+
+    return intersection / new Set([...words1, ...words2]).size;
   }
 
   /**
@@ -378,7 +377,7 @@ export class ResultDeduplicator {
       // Update deduplication info
       primaryResult.deduplicationInfo.merged = true;
       primaryResult.deduplicationInfo.mergedDuplicates = duplicates.length;
-      primaryResult.deduplicationInfo.duplicateUrls = duplicates.map(d => d.originalUrl);
+      primaryResult.deduplicationInfo.duplicateUrls = duplicates.map(d => d.deduplicationInfo.originalUrl);
       
       this.stats.merged += duplicates.length;
       
@@ -456,16 +455,12 @@ export class ResultDeduplicator {
     if (!url) return '';
     
     try {
-      let normalized = url;
-      
       // Parse URL
       const urlObj = new URL(url);
-      
-      // Remove protocol if specified
-      if (normalizationOptions.removeProtocol) {
-        normalized = normalized.replace(/^https?:\/\//, '');
-      }
-      
+
+      // A fragment points into the same page
+      urlObj.hash = '';
+
       // Remove www prefix
       if (normalizationOptions.removeWww) {
         urlObj.hostname = urlObj.hostname.replace(/^www\./, '');
@@ -484,19 +479,24 @@ export class ResultDeduplicator {
         }
       }
       
+      // A directory index is the directory itself
+      urlObj.pathname = urlObj.pathname.replace(/\/index\.(html?|php)$/i, '/');
+
       // Remove trailing slash
       if (normalizationOptions.removeTrailingSlash) {
         urlObj.pathname = urlObj.pathname.replace(/\/$/, '') || '/';
       }
-      
+
       // Sort and clean query parameters
       if (normalizationOptions.sortQueryParams || normalizationOptions.removeEmptyParams) {
         const params = new URLSearchParams(urlObj.search);
         const sortedParams = new URLSearchParams();
-        
-        // Get sorted parameter names
-        const paramNames = Array.from(params.keys()).sort();
-        
+
+        // Get sorted parameter names, without tracking parameters
+        const paramNames = Array.from(params.keys())
+          .filter(name => !/^(utm_.*|gclid|fbclid|msclkid|mc_cid|mc_eid)$/i.test(name))
+          .sort();
+
         for (const name of paramNames) {
           const value = params.get(name);
           if (!normalizationOptions.removeEmptyParams || (value && value.trim())) {
@@ -506,8 +506,12 @@ export class ResultDeduplicator {
         
         urlObj.search = sortedParams.toString();
       }
-      
-      return urlObj.toString();
+
+      // Remove protocol if specified (http vs https of the same page)
+      const normalized = urlObj.toString();
+      return normalizationOptions.removeProtocol
+        ? normalized.replace(/^https?:\/\//, '')
+        : normalized;
     } catch (error) {
       // If URL parsing fails, return cleaned original
       return url.toLowerCase().replace(/^https?:\/\/(www\.)?/, '');
@@ -515,30 +519,12 @@ export class ResultDeduplicator {
   }
 
   /**
-   * Compute content hash using SimHash algorithm
-   */
-  computeContentHash(result, contentOptions) {
-    const content = this.extractContent(result);
-    
-    if (content.length < contentOptions.minLength) {
-      return '0'; // Default hash for short content
-    }
-    
-    return this.simHash(content, contentOptions.simhashBits);
-  }
-
-  /**
-   * Extract content from result for comparison
+   * Extract content from result for comparison. The snippet only: the title
+   * and domain are shared by a site's pages (a title template), so they made
+   * sibling pages look alike; the title is compared on its own anyway.
    */
   extractContent(result) {
-    const parts = [
-      result.title || '',
-      result.snippet || '',
-      result.htmlSnippet || '',
-      result.displayLink || ''
-    ];
-    
-    return parts.join(' ').toLowerCase().replace(/[^\w\s]/g, ' ').replace(/\s+/g, ' ').trim();
+    return (result.snippet || '').toLowerCase().replace(/[^\w\s]/g, ' ').replace(/\s+/g, ' ').trim();
   }
 
   /**

package/src/tools/tracking/trackChanges/differ.js

@@ -104,7 +104,8 @@ export async function fetchContent(url, options = {}) {
     // undici reports every network failure as bare "fetch failed" and puts
     // the real reason (ENOTFOUND, ECONNREFUSED, timeout) in error.cause —
     // without it, a dead host and a typoed domain read identically.
-    const cause = error.cause?.message ? ` (${error.cause.message})` : '';
+    // safeFetch now names it itself (withFetchCause); don't say it twice.
+    const cause = error.cause?.message && !error.message.includes(error.cause.message) ? ` (${error.cause.message})` : '';
     throw new Error(`Failed to fetch content: ${error.message}${cause}`);
   }
 }

package/src/tools/tracking/trackChanges/index.js

@@ -14,6 +14,7 @@
  */
 
 import { EventEmitter } from 'events';
+import { z } from 'zod';
 import os from 'os';
 import path from 'path';
 import ChangeTracker from '../../../core/ChangeTracker.js';
@@ -191,7 +192,11 @@ export class TrackChangesTool extends EventEmitter {
         default:                        throw new Error(`Unknown operation: ${operation}`);
       }
     } catch (error) {
-      return { success: false, error: error.message, timestamp: Date.now() };
+      // A ZodError's message is its issue array as JSON; one line per issue reads better.
+      const message = error instanceof z.ZodError
+        ? `Invalid input - ${error.issues.map((i) => `${i.path.join('.') || '(root)'}: ${i.message}`).join('; ')}`
+        : error.message;
+      return { success: false, error: message, timestamp: Date.now() };
     }
   }
 

package/src/tools/tracking/trackChanges/schema.js

@@ -16,7 +16,7 @@ import { z } from 'zod';
  * `.default({})` would not.
  */
 export const TRACK_CHANGES_INPUT_SHAPE = {
-  url: z.string().url().optional().describe("The URL to track changes for (optional for list_scheduled_monitors)"),
+  url: z.string().url().optional().describe("The URL to track changes for. Required for create_baseline, compare, monitor, get_history and create_scheduled_monitor; stop_scheduled_monitor takes it or scheduledMonitorOptions.monitorId; the other operations run without one"),
   operation: z.enum([
     'create_baseline',
     'compare',
@@ -146,8 +146,22 @@ export const TRACK_CHANGES_INPUT_SHAPE = {
   }).optional().describe("Dashboard display options")
 };
 
+// The operations that act on one page. The rest are global (list, dashboard,
+// templates) or fall back to every tracked URL when url is omitted (get_stats,
+// export_history, create_alert_rule, generate_trend_report);
+// stop_scheduled_monitor also accepts a monitorId and checks that itself.
+const URL_REQUIRED_OPERATIONS = new Set(['create_baseline', 'compare', 'monitor', 'get_history', 'create_scheduled_monitor']);
+
+// The refinement lives here, not in the registered shape: server.js registers
+// TRACK_CHANGES_INPUT_SHAPE as a raw shape, which cannot carry one, so
+// execute() enforces it. compare without a url used to fail with "Failed to
+// parse URL from undefined" plus a create_baseline hint (R25 3.3).
 export const TrackChangesSchema = z.object({
   ...TRACK_CHANGES_INPUT_SHAPE,
   respect_robots: z.boolean().optional(),
   user_agent: z.string().optional()
+}).superRefine((params, ctx) => {
+  if (!params.url && URL_REQUIRED_OPERATIONS.has(params.operation)) {
+    ctx.addIssue({ code: 'custom', path: ['url'], message: `required for operation "${params.operation}"` });
+  }
 });

package/src/utils/fetchError.js

@@ -0,0 +1,47 @@
+/**
+ * undici rejects a DNS, refused or reset connection with a bare TypeError
+ * "fetch failed" and keeps the reason (`getaddrinfo ENOTFOUND host`,
+ * `ECONNREFUSED`) in `cause`. A tool that reports only `error.message` makes a
+ * typoed domain and a dead host read the same, and the not_found hint in
+ * fallbackHints.js, which keys on ENOTFOUND, never matches (R25 3.1).
+ *
+ * @param {Error} error
+ * @param {string} url
+ * @returns {string} "Could not reach <url> (<reason>)" for that error, as the
+ *   REST route's FETCH_FAILED says it; any other error's own message
+ */
+export function fetchErrorMessage(error, url) {
+  if (!(error instanceof TypeError) || error.message !== 'fetch failed') return error.message;
+  const reason = error.cause?.message || error.cause?.code;
+  return `Could not reach ${url}${reason ? ` (${reason})` : ''}`;
+}
+
+/**
+ * The error to rethrow: undici's bare "fetch failed" becomes a TypeError
+ * with fetchErrorMessage's text, keeping undici's cause and its code, so
+ * `err.cause?.code` checks and RetryManager's code list still see
+ * ENOTFOUND / ECONNREFUSED. Any other error comes back unchanged. Applied in
+ * safeFetch and fetchWithTimeout, so every tool that prints only
+ * `err.message` names the cause.
+ *
+ * The URL is named without its query or credentials: these two also fetch
+ * key-based connector URLs (`?key=...`), and the message reaches the caller.
+ *
+ * @param {Error} error
+ * @param {string|URL} url
+ * @returns {Error}
+ */
+export function withFetchCause(error, url) {
+  let shown;
+  try {
+    const parsed = new URL(url);
+    shown = `${parsed.origin}${parsed.pathname}`;
+  } catch {
+    shown = 'the URL';
+  }
+  const message = fetchErrorMessage(error, shown);
+  if (message === error.message) return error;
+  const wrapped = new TypeError(message, { cause: error.cause });
+  if (error.cause?.code) wrapped.code = error.cause.code;
+  return wrapped;
+}

package/src/utils/fetchLadder.js

@@ -24,6 +24,7 @@ import { fetchWithTimeout } from '../tools/basic/_fetch.js';
 import { plainVerdict, targetFailure } from '../tools/basic/_targetFailure.js';
 import { stealthDocumentVerdict } from './stealthVerdict.js';
 import { parseRetryAfter } from './hostRateLimiter.js';
+import { fetchErrorMessage } from './fetchError.js';
 import { setActualCost } from '../server/requestContext.js';
 import { SCRAPE_ESCALATION_SHAPE, SCRAPE_ESCALATION_CREDITS, aBrowserMightPass } from '../tools/scrape/escalation.js';
 
@@ -93,16 +94,15 @@ function retryAfterMs(response) {
 
 /**
  * fetchWithTimeout, with undici's bare "fetch failed" (DNS, refused, reset)
- * given the URL and the reason it keeps in `cause`, as the REST route's
- * FETCH_FAILED "Could not reach <url>" does.
+ * given the URL and the reason it keeps in `cause` (fetchErrorMessage).
  */
 async function fetchOnce(url, options) {
   try {
     return await fetchWithTimeout(url, options);
   } catch (error) {
-    if (!(error instanceof TypeError) || error.message !== 'fetch failed') throw error;
-    const reason = error.cause?.message || error.cause?.code;
-    throw new Error(`Could not reach ${url}${reason ? ` (${reason})` : ''}`, { cause: error });
+    const message = fetchErrorMessage(error, url);
+    if (message === error.message) throw error;
+    throw new Error(message, { cause: error });
   }
 }
 

package/src/utils/htmlToMarkdown.js

@@ -52,6 +52,39 @@ export function promoteCornerHeaderCells(html) {
   }
 }
 
+const ROW_CONTAINER = /^(table|thead|tbody|tfoot)$/;
+
+/**
+ * Readability serializes a layout table it half-kept as `<tbody>`/`<tr>` with
+ * no `<table>` around them. An HTML parser drops row and cell tags outside a
+ * table, so on Hacker News every story's cells ran together into one line
+ * ("…(github.com/x)62 points by … 39 comments2.") and the layout-table rules
+ * never saw a row (R25, 2026-10-19). htmlparser2 keeps the orphans, so they
+ * are found there and each run of adjacent ones is wrapped in a `<table>`.
+ * @param {string} html
+ * @returns {string}
+ */
+export function wrapOrphanRows(html) {
+  if (!/<(tr|tbody|thead|tfoot)[\s>]/i.test(html)) return html;
+  try {
+    const $ = load(html, { xml: { xmlMode: false, decodeEntities: false } }, false);
+    const created = new Set();
+    $('thead, tbody, tfoot, tr').each((_, el) => {
+      if (el.parent && ROW_CONTAINER.test(el.parent.name)) return;
+      const prev = $(el).prev();
+      if (prev.length && created.has(prev[0])) {
+        prev.append(el);
+      } else {
+        $(el).wrap('<table></table>');
+        created.add(el.parent);
+      }
+    });
+    return created.size > 0 ? $.html() : html;
+  } catch {
+    return html;
+  }
+}
+
 // Mirrors turndown-plugin-gfm's own heading-row test, which is what decides
 // whether it converts a table or keeps it as raw HTML.
 function isHeadingRow(tr) {
@@ -141,9 +174,10 @@ export function htmlToMarkdown(html, options = {}) {
     // Drop content a browser would not paint before converting. Turndown keeps
     // the text of screen-reader-only labels and state-gated badges, which then
     // reads as live page content once the CSS is gone.
+    const rows = wrapOrphanRows(html);
     const visible = options.keepHiddenContent
-      ? html
-      : stripHiddenHtml(html, { css: options.css });
+      ? rows
+      : stripHiddenHtml(rows, { css: options.css });
     return getTurndown().turndown(promoteCornerHeaderCells(visible)).trim();
   } catch {
     // Fallback: strip tags, return plain text

package/src/utils/ssrfGuard.js

@@ -36,6 +36,7 @@ import { Agent, buildConnector } from 'undici';
 import { config } from '../constants/config.js';
 import { SSRFProtection } from './ssrfProtection.js';
 import { fetchResigned } from './resignedFetch.js';
+import { withFetchCause } from './fetchError.js';
 
 // Reused only for its (well-tested) CIDR range math — no network state.
 const _ssrf = new SSRFProtection();
@@ -342,7 +343,7 @@ export async function safeFetch(url, options = {}) {
     if (isSsrfError(err)) {
       throw new Error(err.cause?.message || err.message);
     }
-    throw err;
+    throw withFetchCause(err, url);
   }
 }