crawlforge-mcp-server 6.21.1 → 6.22.1

package/src/tools/scrape/unifiedScrape.js

@@ -27,7 +27,8 @@ import { toPublicUnit, parseChosenIndexes, groundingCheck, headingUnits, NOT_IN_
 import { setActualCost } from '../../server/requestContext.js';
 import { fenceUntrusted } from '../../utils/untrustedContent.js';
 import { fetchAndParse } from '../extract/_fetchAndParse.js';
-import { extractMainContent, isThinMainContent, keepPageHeading } from './_mainContent.js';
+import { fetchErrorMessage } from '../../utils/fetchError.js';
+import { extractMainContent, isThinMainContent, keepPageHeading, MAX_READABILITY_ELEMENTS } from './_mainContent.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
 import { stripHiddenFromDom } from '../../utils/hiddenContent.js';
 import { extractBlockText } from '../basic/extractText.js';
@@ -53,6 +54,85 @@ export const JSON_TIMEOUT_MS = 60000;
 
 const isQueryFormat = (fmt) => Boolean(fmt) && typeof fmt === 'object' && (fmt.type === 'highlights' || fmt.type === 'question');
 
+// ── Question evidence support (R25 1.4) ──────────────────────────────────────
+// A ranked unit is not support for an answer by being ranked: before
+// crawlforge-extractors 1.18.0 the ranker scored link URLs, and Wikipedia's
+// reference list answered "Who created Python…?" with citation titles — as
+// grounded:true. These checks read only the units this call returns, so they
+// hold with any extractors version.
+const EVIDENCE_LINK = /(!?)\[((?:\\.|[^[\]\\])*)\]\((?:\\.|[^()\\\s]|\([^()\s]*\))*(?:\s+"(?:\\.|[^"\\])*")?\)/g;
+const REFERENCE_HEADING = /^(?:(?:references?|notes?|citations?|sources?|footnotes?|bibliography|works cited)(?:\s*(?:,|&|and)\s*|\s*$))+$/i;
+// Words any question shares with any page; they show no unit answers it.
+const QUESTION_STOPWORDS = new Set([
+  'a', 'an', 'the', 'and', 'or', 'but', 'of', 'to', 'in', 'on', 'at', 'by', 'for', 'with', 'from', 'as', 'into', 'about',
+  'is', 'are', 'was', 'were', 'be', 'been', 'being', 'am', 'do', 'does', 'did', 'has', 'have', 'had', 'can', 'could',
+  'will', 'would', 'should', 'may', 'might', 'it', 'its', 'this', 'that', 'these', 'those', 'there', 'their', 'they',
+  'he', 'she', 'his', 'her', 'i', 'you', 'we', 'my', 'your', 'our', 'me', 'us', 'them', 'any', 'some', 'not', 'no',
+  'who', 'whom', 'whose', 'what', 'which', 'when', 'where', 'why', 'how', 'many', 'much'
+]);
+// crawlforge-extractors' word runs: a CJK run and any other run never merge.
+const CJK_CHAR = '[\\p{scx=Han}\\p{scx=Hiragana}\\p{scx=Katakana}\\p{scx=Hangul}]';
+const QUESTION_TOKEN = new RegExp(`(?:(?=${CJK_CHAR})[\\p{L}\\p{N}])+|(?:(?!${CJK_CHAR})[\\p{L}\\p{N}\\p{M}])+`, 'gu');
+const CJK_RUN = new RegExp(`^${CJK_CHAR}`, 'u');
+
+/** A unit's text as a reader sees it: link labels without URLs, no citation markers. */
+function visibleEvidenceText(text) {
+  let out = String(text ?? '');
+  for (let pass = 0; pass < 2 && out.includes(']('); pass++) {
+    out = out.replace(EVIDENCE_LINK, (match, _bang, label) => (/\]\(#cite_(?:note|ref)/.test(match) ? ' ' : label));
+  }
+  return out.toLowerCase();
+}
+
+/** An entry of a references-type list: under such a heading, or carrying a Wikipedia back-link. */
+function isReferenceUnit(unit) {
+  if (/\]\(#cite_ref/.test(String(unit.text ?? ''))) return true;
+  return typeof unit.heading === 'string' && REFERENCE_HEADING.test(unit.heading.trim());
+}
+
+/**
+ * The question's content terms, stemmed lightly ("released" → "releas") so a
+ * substring test finds their inflections; a CJK run becomes bigrams.
+ */
+function questionTerms(question) {
+  const terms = new Set();
+  for (const [run] of String(question ?? '').toLowerCase().matchAll(QUESTION_TOKEN)) {
+    if (CJK_RUN.test(run)) {
+      const chars = Array.from(run);
+      if (chars.length === 1) terms.add(run);
+      for (let i = 0; i + 1 < chars.length; i++) terms.add(chars[i] + chars[i + 1]);
+    } else if (!QUESTION_STOPWORDS.has(run)) {
+      terms.add(run.length > 4 ? run.replace(/(?:ing|ed|es|s)$/, '').replace(/(.{4,})e$/, '$1') : run);
+    }
+  }
+  return [...terms];
+}
+
+/**
+ * Whether the ranked evidence can ground an answer: some unit outside a
+ * reference list whose visible text (or heading) holds a content term of the
+ * question. Returns the reason when it cannot.
+ * @param {Array<{ text: string, heading?: string | null }>} evidence
+ * @param {string} question
+ * @returns {{ supported: boolean, reason?: string }}
+ */
+export function evidenceSupport(evidence, question) {
+  if (evidence.length === 0) return { supported: false };
+  const body = evidence.filter((unit) => !isReferenceUnit(unit));
+  if (body.length === 0) {
+    return { supported: false, reason: 'every evidence unit is a reference-list entry (References, Notes, Citations or Sources)' };
+  }
+  const terms = questionTerms(question);
+  if (terms.length === 0) return { supported: true };
+  const holds = body.some((unit) => {
+    const visible = `${visibleEvidenceText(unit.text)}\n${visibleEvidenceText(unit.heading)}`;
+    return terms.some((term) => visible.includes(term));
+  });
+  return holds
+    ? { supported: true }
+    : { supported: false, reason: 'no evidence unit outside a reference list contains a term of the question in its visible text' };
+}
+
 // The six public fields in tools/list order, each carrying the description
 // the client sees. server.js spreads this with COMPLIANCE_PARAMS instead of
 // keeping its own copy (Phase 0, 0.3).
@@ -359,7 +439,7 @@ export class UnifiedScrapeTool {
           errorDocuments: true
         }));
       } catch (err) {
-        throw new Error(`scrape: fetch failed for ${url}: ${err.message}`);
+        throw new Error(`scrape: fetch failed for ${url}: ${fetchErrorMessage(err, url)}`);
       }
       warnings.push(...fetchWarnings);
 
@@ -504,6 +584,11 @@ export class UnifiedScrapeTool {
       if (mainHtml !== null) return mainHtml;
       const main = extractMainContent(html, finalUrl);
       mainHtml = main.html ?? html;
+      if (main.tooLarge) {
+        warnings.push(
+          `mainContent: the page has more than ${MAX_READABILITY_ELEMENTS.toLocaleString('en-US')} elements, too many for main-content extraction; the whole page is used instead`
+        );
+      }
       if (main.tablesRecovered > 0) {
         warnings.push(
           `mainContent: re-attached ${main.tablesRecovered} data table(s) that main-content extraction had dropped`
@@ -638,7 +723,7 @@ export class UnifiedScrapeTool {
                 `json: output did not match the requested schema — ${(result.validationErrors || []).join('; ')}`
               );
             }
-            if (result.truncated) {
+            if (result.input_truncated) {
               warnings.push(
                 `json: page text was truncated from ${result.original_length} chars before extraction; ` +
                 'fields appearing late in the page may be missing'
@@ -686,8 +771,11 @@ export class UnifiedScrapeTool {
             const evidence = rankUnits(candidates, fmt.question, { maxUnits: QUESTION_EVIDENCE_UNITS });
             const evidenceText = evidence.map((unit) => unit.text).join('\n');
             let text = evidenceText;
-            // An empty answer rests on nothing, so it is never grounded.
-            let grounded = evidence.length > 0;
+            // An empty answer rests on nothing, so it is never grounded; nor
+            // does one resting on reference-list entries or link URLs alone.
+            const support = evidenceSupport(evidence, fmt.question);
+            if (support.reason) warnings.push(`question: ${support.reason}; grounded: false`);
+            let grounded = support.supported;
             if (fmt.mode === 'model' && evidence.length > 0) {
               try {
                 text = await this._answerQuestion(evidence, fmt.question);
@@ -698,7 +786,7 @@ export class UnifiedScrapeTool {
                   warnings.push('question: the model found no answer in the evidence; grounded: false');
                 } else {
                   const check = groundingCheck(text, evidenceText, fmt.question);
-                  grounded = check.grounded;
+                  grounded = check.grounded && support.supported;
                   if (!grounded) {
                     warnings.push(`question: the model's answer has ${check.unbacked.length} token(s) not found in the evidence or the question (${check.unbacked.join(', ')}); grounded: false`);
                   }

package/src/tools/search/providers/searxng.js

@@ -94,6 +94,7 @@ export async function searchViaSearxng(opts = {}) {
   url.searchParams.set('safesearch', String(safesearch));
   url.searchParams.set('language', language);
 
+  const startedAt = Date.now();
   let response;
   try {
     response = await fetch(url.toString(), {
@@ -104,7 +105,10 @@ export async function searchViaSearxng(opts = {}) {
     if (err.name === 'TimeoutError' || err.name === 'AbortError') {
       throw new Error('SearXNG request failed: timed out after 15000ms');
     }
-    throw new Error(`SearXNG request failed: ${err.message}`);
+    throw new Error(
+      `SearXNG is not reachable at ${instanceUrl}: ${err.cause?.code || err.message}.\n` +
+        'Next step: start the instance (for the Docker container: "docker start searxng"), or call search_web without provider:"searxng".'
+    );
   }
 
   if (!response.ok) {
@@ -121,13 +125,26 @@ export async function searchViaSearxng(opts = {}) {
   }
 
   const rawResults = Array.isArray(data.results) ? data.results : [];
+
+  // Every engine failing still comes back HTTP 200 with no results; that is
+  // not an empty search (R25 3.3). Entries are [engine, reason] pairs.
+  const unresponsive = Array.isArray(data.unresponsive_engines) ? data.unresponsive_engines : [];
+  if (rawResults.length === 0 && unresponsive.length > 0) {
+    const engines = unresponsive.map((e) => (Array.isArray(e) ? `${e[0]} (${e[1]})` : String(e))).join(', ');
+    throw new Error(
+      `SearXNG returned no results because its engines failed: ${engines}.\n` +
+        'Next step: retry in a minute, or call search_web without provider:"searxng".'
+    );
+  }
+
   const items = rawResults.slice(0, limit).map(normalizeSearxngResult);
 
   return {
     items,
     searchInformation: {
-      totalResults: rawResults.length,
-      searchTime: 0
+      // SearXNG reports 0 when no engine gave a count; that is unknown, not zero.
+      totalResults: data.number_of_results || null,
+      searchTime: (Date.now() - startedAt) / 1000
     },
     queries: {},
     context: {}

package/src/tools/search/ranking/ResultDeduplicator.js

@@ -119,7 +119,6 @@ export class ResultDeduplicator {
         ...result,
         originalIndex: index,
         normalizedUrl: this.normalizeUrl(result.link || result.url, dedupeOptions.urlNormalization),
-        contentHash: this.computeContentHash(result, dedupeOptions.contentSimilarity),
         titleTokens: this.tokenizeTitle(result.title || ''),
         deduplicationInfo: {
           originalUrl: result.link || result.url,
@@ -188,11 +187,13 @@ export class ResultDeduplicator {
       const currentGroup = [results[i]];
       processed.add(i);
 
-      // Find all duplicates of current result
+      // Find all duplicates of current result. A result joins the group only
+      // when it duplicates EVERY member, so a hub page (e.g. MDN `Array`)
+      // cannot pull in its children one by one.
       for (let j = i + 1; j < results.length; j++) {
         if (processed.has(j)) continue;
 
-        if (this.areDuplicates(results[i], results[j], options)) {
+        if (currentGroup.every(member => this.areDuplicates(member, results[j], options))) {
           currentGroup.push(results[j]);
           processed.add(j);
           
@@ -216,21 +217,24 @@ export class ResultDeduplicator {
   areDuplicates(result1, result2, options) {
     const similarities = this.computeSimilarities(result1, result2, options);
 
-    // Near-identical URLs are always duplicates (e.g. http vs https of same page)
-    if (similarities.url >= 0.95) {
+    // The same page written differently (protocol, www, trailing slash,
+    // index.html, tracking parameters) normalizes to the same URL.
+    if (result1.normalizedUrl && result1.normalizedUrl === result2.normalizedUrl) {
       this.stats.urlDuplicates++;
       return true;
     }
 
-    // Require at least 2 high-similarity signals to mark as duplicate
-    let matchCount = 0;
-    if (similarities.url >= options.thresholds.url) matchCount++;
-    if (similarities.title >= options.thresholds.title) matchCount++;
-    if (similarities.content >= options.thresholds.content) matchCount++;
+    // Different URLs merge only on similar content. Sibling pages share a
+    // URL prefix and a site title template (" - JavaScript | MDN"), so URL
+    // and title similarity alone never prove two pages are the same.
+    if (similarities.content < options.thresholds.content) {
+      return false;
+    }
 
-    if (matchCount >= 2) {
+    // Content plus one more high-similarity signal
+    if (similarities.url >= options.thresholds.url || similarities.title >= options.thresholds.title) {
       if (similarities.title >= options.thresholds.title) this.stats.titleDuplicates++;
-      if (similarities.content >= options.thresholds.content) this.stats.contentDuplicates++;
+      this.stats.contentDuplicates++;
       return true;
     }
 
@@ -294,28 +298,23 @@ export class ResultDeduplicator {
   }
 
   /**
-   * Compute content similarity using SimHash
+   * Compute content similarity: Jaccard over the snippets' words. A 64-bit
+   * SimHash of a ~30-word snippet is too noisy here: unrelated MDN sibling
+   * snippets scored up to 0.86 against the 0.85 threshold (R25 1.3).
    */
   computeContentSimilarity(result1, result2, contentOptions) {
     const content1 = this.extractContent(result1);
     const content2 = this.extractContent(result2);
-    
+
     if (content1.length < contentOptions.minLength || content2.length < contentOptions.minLength) {
       return 0;
     }
-    
-    // Use pre-computed content hashes
-    const hash1 = result1.contentHash;
-    const hash2 = result2.contentHash;
-    
-    // Compute Hamming distance between hashes
-    const hammingDistance = this.hammingDistance(hash1, hash2);
-    
-    // Convert to similarity score
-    const maxDistance = contentOptions.simhashBits;
-    const similarity = 1 - (hammingDistance / maxDistance);
-    
-    return Math.max(0, similarity);
+
+    const words1 = new Set(content1.split(' '));
+    const words2 = new Set(content2.split(' '));
+    const intersection = [...words1].filter(word => words2.has(word)).length;
+
+    return intersection / new Set([...words1, ...words2]).size;
   }
 
   /**
@@ -378,7 +377,7 @@ export class ResultDeduplicator {
       // Update deduplication info
       primaryResult.deduplicationInfo.merged = true;
       primaryResult.deduplicationInfo.mergedDuplicates = duplicates.length;
-      primaryResult.deduplicationInfo.duplicateUrls = duplicates.map(d => d.originalUrl);
+      primaryResult.deduplicationInfo.duplicateUrls = duplicates.map(d => d.deduplicationInfo.originalUrl);
       
       this.stats.merged += duplicates.length;
       
@@ -456,16 +455,12 @@ export class ResultDeduplicator {
     if (!url) return '';
     
     try {
-      let normalized = url;
-      
       // Parse URL
       const urlObj = new URL(url);
-      
-      // Remove protocol if specified
-      if (normalizationOptions.removeProtocol) {
-        normalized = normalized.replace(/^https?:\/\//, '');
-      }
-      
+
+      // A fragment points into the same page
+      urlObj.hash = '';
+
       // Remove www prefix
       if (normalizationOptions.removeWww) {
         urlObj.hostname = urlObj.hostname.replace(/^www\./, '');
@@ -484,19 +479,24 @@ export class ResultDeduplicator {
         }
       }
       
+      // A directory index is the directory itself
+      urlObj.pathname = urlObj.pathname.replace(/\/index\.(html?|php)$/i, '/');
+
       // Remove trailing slash
       if (normalizationOptions.removeTrailingSlash) {
         urlObj.pathname = urlObj.pathname.replace(/\/$/, '') || '/';
       }
-      
+
       // Sort and clean query parameters
       if (normalizationOptions.sortQueryParams || normalizationOptions.removeEmptyParams) {
         const params = new URLSearchParams(urlObj.search);
         const sortedParams = new URLSearchParams();
-        
-        // Get sorted parameter names
-        const paramNames = Array.from(params.keys()).sort();
-        
+
+        // Get sorted parameter names, without tracking parameters
+        const paramNames = Array.from(params.keys())
+          .filter(name => !/^(utm_.*|gclid|fbclid|msclkid|mc_cid|mc_eid)$/i.test(name))
+          .sort();
+
         for (const name of paramNames) {
           const value = params.get(name);
           if (!normalizationOptions.removeEmptyParams || (value && value.trim())) {
@@ -506,8 +506,12 @@ export class ResultDeduplicator {
         
         urlObj.search = sortedParams.toString();
       }
-      
-      return urlObj.toString();
+
+      // Remove protocol if specified (http vs https of the same page)
+      const normalized = urlObj.toString();
+      return normalizationOptions.removeProtocol
+        ? normalized.replace(/^https?:\/\//, '')
+        : normalized;
     } catch (error) {
       // If URL parsing fails, return cleaned original
       return url.toLowerCase().replace(/^https?:\/\/(www\.)?/, '');
@@ -515,30 +519,12 @@ export class ResultDeduplicator {
   }
 
   /**
-   * Compute content hash using SimHash algorithm
-   */
-  computeContentHash(result, contentOptions) {
-    const content = this.extractContent(result);
-    
-    if (content.length < contentOptions.minLength) {
-      return '0'; // Default hash for short content
-    }
-    
-    return this.simHash(content, contentOptions.simhashBits);
-  }
-
-  /**
-   * Extract content from result for comparison
+   * Extract content from result for comparison. The snippet only: the title
+   * and domain are shared by a site's pages (a title template), so they made
+   * sibling pages look alike; the title is compared on its own anyway.
    */
   extractContent(result) {
-    const parts = [
-      result.title || '',
-      result.snippet || '',
-      result.htmlSnippet || '',
-      result.displayLink || ''
-    ];
-    
-    return parts.join(' ').toLowerCase().replace(/[^\w\s]/g, ' ').replace(/\s+/g, ' ').trim();
+    return (result.snippet || '').toLowerCase().replace(/[^\w\s]/g, ' ').replace(/\s+/g, ' ').trim();
   }
 
   /**

package/src/tools/tracking/trackChanges/differ.js

@@ -104,7 +104,8 @@ export async function fetchContent(url, options = {}) {
     // undici reports every network failure as bare "fetch failed" and puts
     // the real reason (ENOTFOUND, ECONNREFUSED, timeout) in error.cause —
     // without it, a dead host and a typoed domain read identically.
-    const cause = error.cause?.message ? ` (${error.cause.message})` : '';
+    // safeFetch now names it itself (withFetchCause); don't say it twice.
+    const cause = error.cause?.message && !error.message.includes(error.cause.message) ? ` (${error.cause.message})` : '';
     throw new Error(`Failed to fetch content: ${error.message}${cause}`);
   }
 }

package/src/tools/tracking/trackChanges/index.js

@@ -14,6 +14,7 @@
  */
 
 import { EventEmitter } from 'events';
+import { z } from 'zod';
 import os from 'os';
 import path from 'path';
 import ChangeTracker from '../../../core/ChangeTracker.js';
@@ -191,7 +192,11 @@ export class TrackChangesTool extends EventEmitter {
         default:                        throw new Error(`Unknown operation: ${operation}`);
       }
     } catch (error) {
-      return { success: false, error: error.message, timestamp: Date.now() };
+      // A ZodError's message is its issue array as JSON; one line per issue reads better.
+      const message = error instanceof z.ZodError
+        ? `Invalid input - ${error.issues.map((i) => `${i.path.join('.') || '(root)'}: ${i.message}`).join('; ')}`
+        : error.message;
+      return { success: false, error: message, timestamp: Date.now() };
     }
   }
 

package/src/tools/tracking/trackChanges/schema.js

@@ -16,7 +16,7 @@ import { z } from 'zod';
  * `.default({})` would not.
  */
 export const TRACK_CHANGES_INPUT_SHAPE = {
-  url: z.string().url().optional().describe("The URL to track changes for (optional for list_scheduled_monitors)"),
+  url: z.string().url().optional().describe("The URL to track changes for. Required for create_baseline, compare, monitor, get_history and create_scheduled_monitor; stop_scheduled_monitor takes it or scheduledMonitorOptions.monitorId; the other operations run without one"),
   operation: z.enum([
     'create_baseline',
     'compare',
@@ -146,8 +146,22 @@ export const TRACK_CHANGES_INPUT_SHAPE = {
   }).optional().describe("Dashboard display options")
 };
 
+// The operations that act on one page. The rest are global (list, dashboard,
+// templates) or fall back to every tracked URL when url is omitted (get_stats,
+// export_history, create_alert_rule, generate_trend_report);
+// stop_scheduled_monitor also accepts a monitorId and checks that itself.
+const URL_REQUIRED_OPERATIONS = new Set(['create_baseline', 'compare', 'monitor', 'get_history', 'create_scheduled_monitor']);
+
+// The refinement lives here, not in the registered shape: server.js registers
+// TRACK_CHANGES_INPUT_SHAPE as a raw shape, which cannot carry one, so
+// execute() enforces it. compare without a url used to fail with "Failed to
+// parse URL from undefined" plus a create_baseline hint (R25 3.3).
 export const TrackChangesSchema = z.object({
   ...TRACK_CHANGES_INPUT_SHAPE,
   respect_robots: z.boolean().optional(),
   user_agent: z.string().optional()
+}).superRefine((params, ctx) => {
+  if (!params.url && URL_REQUIRED_OPERATIONS.has(params.operation)) {
+    ctx.addIssue({ code: 'custom', path: ['url'], message: `required for operation "${params.operation}"` });
+  }
 });

package/src/utils/fetchError.js

@@ -0,0 +1,47 @@
+/**
+ * undici rejects a DNS, refused or reset connection with a bare TypeError
+ * "fetch failed" and keeps the reason (`getaddrinfo ENOTFOUND host`,
+ * `ECONNREFUSED`) in `cause`. A tool that reports only `error.message` makes a
+ * typoed domain and a dead host read the same, and the not_found hint in
+ * fallbackHints.js, which keys on ENOTFOUND, never matches (R25 3.1).
+ *
+ * @param {Error} error
+ * @param {string} url
+ * @returns {string} "Could not reach <url> (<reason>)" for that error, as the
+ *   REST route's FETCH_FAILED says it; any other error's own message
+ */
+export function fetchErrorMessage(error, url) {
+  if (!(error instanceof TypeError) || error.message !== 'fetch failed') return error.message;
+  const reason = error.cause?.message || error.cause?.code;
+  return `Could not reach ${url}${reason ? ` (${reason})` : ''}`;
+}
+
+/**
+ * The error to rethrow: undici's bare "fetch failed" becomes a TypeError
+ * with fetchErrorMessage's text, keeping undici's cause and its code, so
+ * `err.cause?.code` checks and RetryManager's code list still see
+ * ENOTFOUND / ECONNREFUSED. Any other error comes back unchanged. Applied in
+ * safeFetch and fetchWithTimeout, so every tool that prints only
+ * `err.message` names the cause.
+ *
+ * The URL is named without its query or credentials: these two also fetch
+ * key-based connector URLs (`?key=...`), and the message reaches the caller.
+ *
+ * @param {Error} error
+ * @param {string|URL} url
+ * @returns {Error}
+ */
+export function withFetchCause(error, url) {
+  let shown;
+  try {
+    const parsed = new URL(url);
+    shown = `${parsed.origin}${parsed.pathname}`;
+  } catch {
+    shown = 'the URL';
+  }
+  const message = fetchErrorMessage(error, shown);
+  if (message === error.message) return error;
+  const wrapped = new TypeError(message, { cause: error.cause });
+  if (error.cause?.code) wrapped.code = error.cause.code;
+  return wrapped;
+}

package/src/utils/fetchLadder.js

@@ -24,6 +24,7 @@ import { fetchWithTimeout } from '../tools/basic/_fetch.js';
 import { plainVerdict, targetFailure } from '../tools/basic/_targetFailure.js';
 import { stealthDocumentVerdict } from './stealthVerdict.js';
 import { parseRetryAfter } from './hostRateLimiter.js';
+import { fetchErrorMessage } from './fetchError.js';
 import { setActualCost } from '../server/requestContext.js';
 import { SCRAPE_ESCALATION_SHAPE, SCRAPE_ESCALATION_CREDITS, aBrowserMightPass } from '../tools/scrape/escalation.js';
 
@@ -93,16 +94,15 @@ function retryAfterMs(response) {
 
 /**
  * fetchWithTimeout, with undici's bare "fetch failed" (DNS, refused, reset)
- * given the URL and the reason it keeps in `cause`, as the REST route's
- * FETCH_FAILED "Could not reach <url>" does.
+ * given the URL and the reason it keeps in `cause` (fetchErrorMessage).
  */
 async function fetchOnce(url, options) {
   try {
     return await fetchWithTimeout(url, options);
   } catch (error) {
-    if (!(error instanceof TypeError) || error.message !== 'fetch failed') throw error;
-    const reason = error.cause?.message || error.cause?.code;
-    throw new Error(`Could not reach ${url}${reason ? ` (${reason})` : ''}`, { cause: error });
+    const message = fetchErrorMessage(error, url);
+    if (message === error.message) throw error;
+    throw new Error(message, { cause: error });
   }
 }
 

package/src/utils/htmlToMarkdown.js

@@ -52,6 +52,39 @@ export function promoteCornerHeaderCells(html) {
   }
 }
 
+const ROW_CONTAINER = /^(table|thead|tbody|tfoot)$/;
+
+/**
+ * Readability serializes a layout table it half-kept as `<tbody>`/`<tr>` with
+ * no `<table>` around them. An HTML parser drops row and cell tags outside a
+ * table, so on Hacker News every story's cells ran together into one line
+ * ("…(github.com/x)62 points by … 39 comments2.") and the layout-table rules
+ * never saw a row (R25, 2026-10-19). htmlparser2 keeps the orphans, so they
+ * are found there and each run of adjacent ones is wrapped in a `<table>`.
+ * @param {string} html
+ * @returns {string}
+ */
+export function wrapOrphanRows(html) {
+  if (!/<(tr|tbody|thead|tfoot)[\s>]/i.test(html)) return html;
+  try {
+    const $ = load(html, { xml: { xmlMode: false, decodeEntities: false } }, false);
+    const created = new Set();
+    $('thead, tbody, tfoot, tr').each((_, el) => {
+      if (el.parent && ROW_CONTAINER.test(el.parent.name)) return;
+      const prev = $(el).prev();
+      if (prev.length && created.has(prev[0])) {
+        prev.append(el);
+      } else {
+        $(el).wrap('<table></table>');
+        created.add(el.parent);
+      }
+    });
+    return created.size > 0 ? $.html() : html;
+  } catch {
+    return html;
+  }
+}
+
 // Mirrors turndown-plugin-gfm's own heading-row test, which is what decides
 // whether it converts a table or keeps it as raw HTML.
 function isHeadingRow(tr) {
@@ -141,9 +174,10 @@ export function htmlToMarkdown(html, options = {}) {
     // Drop content a browser would not paint before converting. Turndown keeps
     // the text of screen-reader-only labels and state-gated badges, which then
     // reads as live page content once the CSS is gone.
+    const rows = wrapOrphanRows(html);
     const visible = options.keepHiddenContent
-      ? html
-      : stripHiddenHtml(html, { css: options.css });
+      ? rows
+      : stripHiddenHtml(rows, { css: options.css });
     return getTurndown().turndown(promoteCornerHeaderCells(visible)).trim();
   } catch {
     // Fallback: strip tags, return plain text

package/src/utils/ssrfGuard.js

@@ -36,6 +36,7 @@ import { Agent, buildConnector } from 'undici';
 import { config } from '../constants/config.js';
 import { SSRFProtection } from './ssrfProtection.js';
 import { fetchResigned } from './resignedFetch.js';
+import { withFetchCause } from './fetchError.js';
 
 // Reused only for its (well-tested) CIDR range math — no network state.
 const _ssrf = new SSRFProtection();
@@ -342,7 +343,7 @@ export async function safeFetch(url, options = {}) {
     if (isSsrfError(err)) {
       throw new Error(err.cause?.message || err.message);
     }
-    throw err;
+    throw withFetchCause(err, url);
   }
 }