crawlforge-mcp-server 6.21.0 → 6.22.0

package/src/tools/advanced/batchScrape/worker.js

@@ -11,10 +11,14 @@ import { ssrfGuard, isSsrfError } from '../../../utils/ssrfGuard.js';
 import { noteRetryAfter } from '../../../utils/hostRateLimiter.js';
 import { preflightFetch } from '../../../utils/robotsGate.js';
 import { fetchResigned } from '../../../utils/resignedFetch.js';
+import { fetchErrorMessage } from '../../../utils/fetchError.js';
 import { htmlToMarkdown } from '../../../utils/htmlToMarkdown.js';
+import { extractMainContent, isThinMainContent, keepPageHeading } from '../../scrape/_mainContent.js';
 import { elementText } from '../../../utils/elementText.js';
 import { pageTitle } from '../../../utils/pageTitle.js';
 import { classifyContentType, unsupportedContentTypeError } from '../../extract/_fetchAndParse.js';
+import { requestContext } from '../../../server/requestContext.js';
+import { classifyError, CLASS_HINTS, FALLBACK_HINTS } from '../../../server/fallbackHints.js';
 
 /**
  * Fetch a URL with AbortController timeout (SSRF-guarded + per-host throttled).
@@ -57,7 +61,9 @@ export async function fetchUrl(url, options = {}) {
   } catch (error) {
     if (isSsrfError(error)) throw new Error(error.cause?.message || error.message);
     if (error.name === 'AbortError') throw new Error(`Request timeout after ${timeout}ms`);
-    throw error;
+    const message = fetchErrorMessage(error, url);
+    if (message === error.message) throw error;
+    throw new Error(message, { cause: error });
   } finally {
     // Cleared only after the body read finishes (or the fetch itself fails),
     // so the abort timer keeps protecting against a trickling body, not just
@@ -103,13 +109,14 @@ export async function scrapeUrl(config, options, defaultTimeout) {
       result.extracted = extractStructuredData($, { ...config.selectors, ...options.extractionSchema });
     }
 
-    result.content = generateFormats($, html, options.formats);
+    result.content = generateFormats($, html, options.formats, config.url);
     return result;
   } catch (error) {
     return {
       success: false,
       url: config.url,
       error: error.message,
+      next_step: failedEntryHint(error.message),
       timestamp: Date.now(),
       executionTime: Date.now() - startTime,
       metadata: config.metadata || {}
@@ -117,6 +124,17 @@ export async function scrapeUrl(config, options, defaultTimeout) {
   }
 }
 
+/**
+ * The next step for one failed URL (R25 3.1): its error class's hint, else
+ * scrape's. A batch with failed URLs still succeeds, so withAuth never hints
+ * it. Classified from the message alone: the invocation's preflight-refusal
+ * flag belongs to the whole batch, and would give every URL one URL's refusal.
+ */
+export function failedEntryHint(message) {
+  const c = requestContext.exit(() => classifyError(null, message));
+  return c ? CLASS_HINTS[c.cls]('batch_scrape', c.params) : FALLBACK_HINTS.scrape;
+}
+
 function extractStructuredData($, selectors) {
   const extracted = {};
   for (const [key, selector] of Object.entries(selectors)) {
@@ -144,7 +162,7 @@ function valueText($, el) {
     : flattenText($, $el);
 }
 
-export function generateFormats($, html, formats) {
+export function generateFormats($, html, formats, url) {
   // Script, style, noscript and template bodies are never rendered text.
   // Left in, the text format carried an entire theme-toggle script
   // (erlang.org/downloads) and CSS rules (opennet.ru) — R15, 2026-09-04.
@@ -153,7 +171,7 @@ export function generateFormats($, html, formats) {
   const content = {};
   if (formats.includes('html')) content.html = html;
   if (formats.includes('text')) content.text = flattenText($);
-  if (formats.includes('markdown')) content.markdown = buildMarkdown($);
+  if (formats.includes('markdown')) content.markdown = buildMarkdown($, html, url);
   if (formats.includes('json')) {
     content.json = {
       title: pageTitle($),
@@ -166,8 +184,15 @@ export function generateFormats($, html, formats) {
   return content;
 }
 
-function buildMarkdown($) {
-  const title = pageTitle($);
+/**
+ * The page's main content, by the same pass scrape's markdown uses: converting
+ * the whole <main> handed back Wikipedia's language list and menus (R25,
+ * 2026-10-19). A page Readability finds no article in, or only a thin
+ * fragment of, falls back to the first content container below.
+ */
+function mainContentHtml($, html, url) {
+  const main = extractMainContent(html, url);
+  if (main.html && !isThinMainContent(main.html, html)) return keepPageHeading(main.html, $);
 
   const selectors = ['article', 'main', '.content', '#content', '.post-content', '.entry-content'];
   let $body = null;
@@ -176,17 +201,31 @@ function buildMarkdown($) {
     if ($body.length > 0) break;
   }
   if (!$body || $body.length === 0) $body = $('body');
+  return $.html($body);
+}
+
+// "Web scraping - Wikipedia" names the page "Web scraping": the H1 plus a
+// separator and the site's name.
+function titleRepeatsHeading(title, heading) {
+  const t = title.toLowerCase();
+  const h = heading.toLowerCase();
+  return t === h || (t.startsWith(h) && /^\s+[-–—|·]\s+\S/.test(t.slice(h.length)));
+}
+
+function buildMarkdown($, html, url) {
+  const title = pageTitle($);
+  const mainHtml = mainContentHtml($, html, url);
 
   // Full-fidelity conversion via the shared Turndown helper (same converter
   // the unified `scrape` tool uses). The previous hand-rolled h1–h3/p/li walk
   // silently dropped any text living outside those tags (e.g. quotes in
   // <span>/<small> on quotes.toscrape.com).
-  let md = htmlToMarkdown($.html($body));
+  let md = htmlToMarkdown(mainHtml);
 
-  // C3: de-dup title — only emit the <title> heading if the page has no <h1>
-  // or if the first <h1> text differs from the <title> text (case-insensitive).
-  const firstH1 = $body.find('h1').first().text().trim();
-  const titleDuplicated = firstH1 && firstH1.toLowerCase() === title.toLowerCase();
+  // C3: de-dup title — only emit the <title> heading if the content has no
+  // <h1> or its first <h1> is not what the <title> names (case-insensitive).
+  const firstH1 = load(mainHtml)('h1').first().text().replace(/\s+/g, ' ').trim();
+  const titleDuplicated = firstH1 && titleRepeatsHeading(title, firstH1);
   if (title && !titleDuplicated) md = `# ${title}\n\n${md}`;
 
   return md.trim();

package/src/tools/agent/agent.js

@@ -16,7 +16,7 @@ import { AGENT_ESCALATION_CREDITS } from './escalation.js';
 export const AGENT_BASE_CREDITS = 8;
 
 export const AgentInputSchema = z.object({
-  prompt: z.string().min(1).max(2000).describe('Natural-language task or question'),
+  prompt: z.string().trim().min(1).max(2000).describe('Natural-language task or question'),
   urls: z.array(z.string().url()).max(20).optional().describe('Optional seed URLs to include (max 20)'),
   schema: z.record(z.any()).optional().describe('Optional JSON schema for structured output'),
   model: z.enum(['default', 'pro']).optional().default('default').describe('"default" = SamplingClient loop; "pro" = full ResearchOrchestrator'),

package/src/tools/basic/_fetch.js

@@ -9,6 +9,7 @@ import { ssrfGuard, isSsrfError } from '../../utils/ssrfGuard.js';
 import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
 import { preflightFetch } from '../../utils/robotsGate.js';
 import { fetchResigned } from '../../utils/resignedFetch.js';
+import { withFetchCause } from '../../utils/fetchError.js';
 
 /**
  * Fetch a URL with a configurable timeout and body-size cap.
@@ -68,7 +69,7 @@ export async function fetchWithTimeout(url, options = {}) {
       if (error.name === 'AbortError') {
         throw new Error(`Request timeout after ${timeout}ms`);
       }
-      throw error;
+      throw withFetchCause(error, url);
     }
 
     // A host asking us to back off is honoured on the *next* request to it,

package/src/tools/basic/extractMetadata.js

@@ -7,6 +7,7 @@
 
 import { load } from 'cheerio';
 import { fetchWithTimeout } from './_fetch.js';
+import { fetchErrorMessage } from '../../utils/fetchError.js';
 import { parseJsonLd, filterJsonLdByType } from '../../utils/jsonLd.js';
 
 /**
@@ -161,7 +162,7 @@ export async function extractMetadataHandler({ url, user_agent, respect_robots,
     };
   } catch (error) {
     return {
-      content: [{ type: 'text', text: `Failed to extract metadata: ${error.message}` }],
+      content: [{ type: 'text', text: `Failed to extract metadata: ${fetchErrorMessage(error, url)}` }],
       isError: true
     };
   }

package/src/tools/basic/extractText.js

@@ -15,6 +15,7 @@ import { JSDOM } from 'jsdom';
 import { Readability } from '@mozilla/readability';
 import { fetchLadder, ladderErrorResult, isJsonType } from '../../utils/fetchLadder.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
+import { tooLargeForReadability } from '../scrape/_mainContent.js';
 
 // Block-level elements whose boundaries should become paragraph breaks
 const BLOCK_ELEMENTS = new Set([
@@ -66,14 +67,17 @@ export function extractBlockText($) {
  * @returns {string}
  */
 export function readabilityToMarkdown(html, pageUrl) {
-  let articleHtml;
-  try {
-    const dom = new JSDOM(html, { url: pageUrl });
-    const reader = new Readability(dom.window.document);
-    const article = reader.parse();
-    articleHtml = article ? article.content : html;
-  } catch {
-    articleHtml = html;
+  let articleHtml = html;
+  // A page too large for Readability is converted whole (R25).
+  if (!tooLargeForReadability(html)) {
+    try {
+      const dom = new JSDOM(html, { url: pageUrl });
+      const reader = new Readability(dom.window.document);
+      const article = reader.parse();
+      if (article) articleHtml = article.content;
+    } catch {
+      // the whole page
+    }
   }
   const $ = load(articleHtml);
   absoluteUrls($, pageUrl);

package/src/tools/crawl/mapSite.js

@@ -167,6 +167,11 @@ export class MapSiteTool {
       // unresolvable host used to return a success with total_urls: 0 (R24).
       try {
         const pageUrls = await this.fetchPageUrls(validated.url, domainFilter, identity);
+        // The start page is a page of the site too: a one-page site with no
+        // sitemap used to map to 0 URLs (R25 1.2).
+        if (!domainFilter || domainFilter.isAllowed(validated.url).allowed) {
+          urls.add(normalizeUrl(validated.url));
+        }
         pageUrls.forEach(url => {
           if (urls.size < poolLimit) {
             urls.add(normalizeUrl(url));
@@ -200,6 +205,9 @@ export class MapSiteTool {
 
       // Convert to array and limit
       const urlArray = pool.slice(0, validated.max_urls);
+      if (urlArray.length === 0) {
+        warnings.push('No URLs were found: no sitemap listed any, and the start page and its links were all excluded by the domain filter.');
+      }
 
       // Fetch metadata if requested
       if (validated.include_metadata) {

package/src/tools/extract/extractContent.js

@@ -401,7 +401,8 @@ export class ExtractContentTool {
    */
   extractTitleFromHTML(html) {
     const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
-    return titleMatch ? titleMatch[1].trim() : null;
+    // Decode entities: MDN's <table> page is titled "&lt;table&gt; ...".
+    return titleMatch ? load(titleMatch[1], null, false).root().text().trim() : null;
   }
 
   /**

package/src/tools/extract/extractStructured.js

@@ -10,6 +10,7 @@ import { load } from 'cheerio';
 import { LLMManager } from '../../core/llm/LLMManager.js';
 import { CRAWLFORGE_USER_AGENT } from '../../utils/fetchIdentity.js';
 import { fetchAndParse, flattenBodyText } from './_fetchAndParse.js';
+import { fetchErrorMessage } from '../../utils/fetchError.js';
 import { extractMainContent } from '../scrape/_mainContent.js';
 import { verifyNumericProvenance } from '../../utils/provenance.js';
 import { validateAgainstSchema, validateFieldsAgainstSchema } from '../../utils/schemaValidate.js';
@@ -532,6 +533,7 @@ export class ExtractStructuredTool {
       };
 
     } catch (error) {
+      const message = fetchErrorMessage(error, params.url);
       return {
         success: false,
         url: params.url || 'unknown',
@@ -540,8 +542,8 @@ export class ExtractStructuredTool {
         confidence: 0,
         schema_used: params.schema || {},
         processingTime: Date.now() - startTime,
-        error: `Structured extraction failed: ${error.message}`,
-        validation: { valid: false, errors: [error.message] }
+        error: `Structured extraction failed: ${message}`,
+        validation: { valid: false, errors: [message] }
       };
     }
   }

package/src/tools/extract/extractWithLlm.js

@@ -529,8 +529,10 @@ export class ExtractWithLlm {
       }));
     } catch (llmErr) {
       if (signal?.aborted) return { success: false, error: `LLM call aborted: ${llmErr.message}` };
-      // D1.3: If provider is 'auto'/'ollama' and it failed, try MCP sampling as final fallback
-      if (providerParam === 'auto' || providerParam === 'ollama') {
+      // D1.3: If provider is 'auto'/'ollama' and it failed, try MCP sampling as final fallback.
+      // Not when the caller named a model: sampling would answer with a
+      // different one, silently (R25 3.3).
+      if ((providerParam === 'auto' || providerParam === 'ollama') && !modelParam) {
         try {
           const SamplingClient = await getSamplingClient();
           const samplingClient = new SamplingClient({ mcpServer: this._mcpServer });
@@ -639,7 +641,9 @@ export class ExtractWithLlm {
       if (checked.skipped) provenance.skipped = checked.skipped;
     }
 
-    // C3: surface truncation metadata so callers know the input was clipped
+    // C3: surface truncation metadata so callers know the input was clipped.
+    // `input_truncated`, not `truncated`: a top-level `truncated: true` means
+    // "this result is a preview, read the rest with read_result" (R25).
     const result = {
       success: true,
       data: parsed,
@@ -649,7 +653,7 @@ export class ExtractWithLlm {
       usage
     };
     if (inputTruncated) {
-      result.truncated = true;
+      result.input_truncated = true;
       result.original_length = original_length;
     }
     if (fetchWarnings.length > 0) result.warnings = fetchWarnings;

package/src/tools/extract/processDocument.js

@@ -300,6 +300,33 @@ export class ProcessDocumentTool {
         tool: 'process_document'
       });
       if (gate.warnings.length > 0) result.warnings = gate.warnings;
+
+      // What came back is checked before the PDF parser sees it, as the url
+      // branch does: a PNG fetched as pdf_url failed as "Invalid PDF
+      // structure" (R25 3.3).
+      const response = await safeFetch(source, {
+        headers: { ...gate.headers },
+        onRedirect: gate.onRedirect,
+        signal: AbortSignal.timeout(30000)
+      });
+      if (!response.ok) {
+        if (response.status === 429 || response.status === 503) {
+          noteRetryAfter(source, response.headers.get('retry-after'));
+        }
+        throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+      }
+      const contentType = response.headers.get('content-type') || '';
+      const buffer = await this.pdfProcessor.readBodyWithSizeCap(response);
+      const kind = sniffDocumentKind(contentType, buffer, source);
+      if (kind !== 'pdf') {
+        throw new Error(
+          `${contentType.split(';')[0] || kind} content is not a PDF: sourceType "pdf_url" needs a URL that serves one. ` +
+            'process_document reads PDF, DOCX, HTML and plain text; sourceType "url" detects which.'
+        );
+      }
+      source = buffer;
+      sourceType = 'pdf_buffer';
+      gate = undefined;
     }
 
     const pdfResult = await this.pdfProcessor.processPDF({
@@ -619,7 +646,8 @@ export class ProcessDocumentTool {
    */
   extractTitleFromHTML(html) {
     const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
-    return titleMatch ? titleMatch[1].trim() : null;
+    // Decode entities: MDN's <table> page is titled "&lt;table&gt; ...".
+    return titleMatch ? load(titleMatch[1], null, false).root().text().trim() : null;
   }
 
   /**

package/src/tools/extract/summarizeContent.js

@@ -211,11 +211,15 @@ export class SummarizeContentTool {
       return result;
 
     } catch (error) {
+      // A ZodError's message is its issue array as JSON; one line per issue reads better.
+      const reason = error instanceof z.ZodError
+        ? `Invalid input - ${error.issues.map((i) => `${i.path.join('.') || '(root)'}: ${i.message}`).join('; ')}`
+        : error.message;
       return {
         originalText: params.text?.substring(0, 100) || 'unknown',
         summarizedAt: new Date().toISOString(),
         success: false,
-        error: `Content summarization failed: ${error.message}`,
+        error: `Content summarization failed: ${reason}`,
         processingTime: Date.now() - startTime,
         summary: {
           text: '',

package/src/tools/llmstxt/generateLLMsTxt.js

@@ -33,7 +33,7 @@ const GenerateLLMsTxtSchema = z.object({
     robotsStyle: z.boolean().optional().default(false).describe('Emit legacy robots.txt-style directives instead of spec-compliant llmstxt.org markdown')
   }).optional().prefault({}),
 
-  complianceLevel: z.enum(['basic', 'standard', 'strict']).optional().default('standard').describe('Compliance level for generated guidelines'),
+  complianceLevel: z.enum(['basic', 'standard', 'strict']).optional().default('standard').describe('Applies only with outputOptions.robotsStyle:true, where it sets how many paths the llms.txt Disallow list covers (basic < standard < strict). The default llmstxt.org markdown is the same at every level; llms-full.txt only records the level as a label.'),
   
   format: z.enum(['both', 'llms-txt', 'llms-full-txt']).optional().default('both').describe('Which files to generate')
 });
@@ -740,7 +740,11 @@ export class GenerateLLMsTxtTool {
     lines.push('If you encounter issues or need clarification on these guidelines:');
     lines.push('1. Check this document for guidance');
     lines.push('2. Review the basic llms.txt file');
-    lines.push('3. Contact us using the information above');
+    // Only when section 1 printed something to contact: without either field
+    // this pointed at contact information the file did not contain.
+    if (outputOptions.contactEmail || outputOptions.organizationName) {
+      lines.push('3. Contact us using the information above');
+    }
     lines.push('');
     lines.push('### Updates');
     lines.push('These guidelines may be updated periodically. Check the generation date above');

package/src/tools/scrape/_mainContent.js

@@ -49,6 +49,12 @@ import { load } from 'cheerio';
 // Readability already kept it. Long enough to be unique to that table.
 const SIGNATURE_LENGTH = 120;
 
+// Navigation tables are link lists laid out as tables, not data. Wikipedia's
+// navbox and sidebar templates pass the data-table test (a `<th>` and 2+
+// cells a row), so List_of_HTTP_status_codes came back with its "HTTP" sidebar
+// and footer navbox re-attached as pipe tables (R25, 2026-10-19).
+const NAVIGATION_TABLE = 'table.navbox, .navbox-inner, .vertical-navbox, [role="navigation"], nav';
+
 // Copied verbatim from @mozilla/readability's REGEXPS. Two of them delete
 // content, and both are unanchored: `unlikelyCandidates` removes an element
 // outright, `negative` docks 25 from its class weight, which is enough for
@@ -151,6 +157,26 @@ export function sanitizeAccidentalClasses(document) {
   return sanitized;
 }
 
+/**
+ * Undo MediaWiki's heading wrapper before Readability sees the page.
+ *
+ * Since 2024 MediaWiki emits every section heading as
+ * `<div class="mw-heading"><h2>History</h2><span class="mw-editsection">[edit]</span></div>`.
+ * Readability scores that div as a link-heavy fragment: the History and
+ * Techniques h2s of Wikipedia's *Web scraping* article were dropped, and the
+ * headings that survived each came back followed by a "[edit]" link line
+ * (R25, 2026-10-19). Deleting the edit links and unwrapping the div leaves the
+ * bare heading Readability keeps.
+ *
+ * @param {Document} document - mutated in place
+ */
+export function unwrapMediaWikiHeadings(document) {
+  for (const el of Array.from(document.querySelectorAll('.mw-editsection'))) el.remove();
+  for (const wrapper of Array.from(document.querySelectorAll('div.mw-heading'))) {
+    wrapper.replaceWith(...Array.from(wrapper.childNodes));
+  }
+}
+
 /**
  * Readability's own data-table test (`_markDataTables`): 10+ rows or more than
  * 4 columns means the table carries data rather than layout, counting
@@ -232,6 +258,7 @@ export function recoverDroppedTables(html, url, keptText = '') {
       // A nested table travels with its parent; re-attaching it separately
       // would duplicate it.
       .filter((table) => !table.parentElement?.closest('table'))
+      .filter((table) => !table.closest(NAVIGATION_TABLE))
       .filter((table) => isDataTable(table) || isHeadedTable(table))
       .filter((table) => {
         const signature = normalizeWhitespace(table.textContent || '').slice(0, SIGNATURE_LENGTH);
@@ -243,6 +270,28 @@ export function recoverDroppedTables(html, url, keptText = '') {
   }
 }
 
+// Readability's time grows faster than the page, and all of it blocks the
+// event loop: about 1 s at 25k elements (the largest Wikipedia articles), 9 s
+// at 114k (docs.python.org's whatsnew/changelog, 7.3 MB), which starved other
+// crawls past their 30 s queue budget (R25 Phase 2 follow-up). Above this many
+// elements a page skips the main-content pass and callers use the whole page.
+export const MAX_READABILITY_ELEMENTS = 50000;
+
+/**
+ * True when `html` has more elements than Readability should be given. Counts
+ * opening tags with a regex (about 5 ms on 7 MB) rather than building a DOM,
+ * which would itself cost seconds.
+ * @param {string} html
+ */
+export function tooLargeForReadability(html) {
+  const tag = /<[a-zA-Z]/g;
+  let count = 0;
+  while (tag.exec(html)) {
+    if (++count > MAX_READABILITY_ELEMENTS) return true;
+  }
+  return false;
+}
+
 /**
  * Extract a page's main content, re-attaching data tables Readability dropped.
  *
@@ -252,14 +301,17 @@ export function recoverDroppedTables(html, url, keptText = '') {
  *
  * @param {string} html - the page's HTML
  * @param {string} [url] - document URL, used as the base for relative links
- * @returns {{ html: string|null, title: string, tablesRecovered: number }} `html`
- *   is null when Readability found no article; callers decide what to fall back to.
+ * @returns {{ html: string|null, title: string, tablesRecovered: number, tooLarge?: boolean }}
+ *   `html` is null when Readability found no article, or was not run because the
+ *   page is too large (`tooLarge`); callers decide what to fall back to.
  */
 export function extractMainContent(html, url) {
+  if (tooLargeForReadability(html)) return { html: null, title: '', tablesRecovered: 0, tooLarge: true };
   let article;
   try {
     const dom = new JSDOM(html, { url });
     sanitizeAccidentalClasses(dom.window.document);
+    unwrapMediaWikiHeadings(dom.window.document);
     article = new Readability(dom.window.document).parse();
   } catch {
     return { html: null, title: '', tablesRecovered: 0 };

package/src/tools/scrape/unifiedScrape.js

@@ -27,7 +27,8 @@ import { toPublicUnit, parseChosenIndexes, groundingCheck, headingUnits, NOT_IN_
 import { setActualCost } from '../../server/requestContext.js';
 import { fenceUntrusted } from '../../utils/untrustedContent.js';
 import { fetchAndParse } from '../extract/_fetchAndParse.js';
-import { extractMainContent, isThinMainContent, keepPageHeading } from './_mainContent.js';
+import { fetchErrorMessage } from '../../utils/fetchError.js';
+import { extractMainContent, isThinMainContent, keepPageHeading, MAX_READABILITY_ELEMENTS } from './_mainContent.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
 import { stripHiddenFromDom } from '../../utils/hiddenContent.js';
 import { extractBlockText } from '../basic/extractText.js';
@@ -53,6 +54,85 @@ export const JSON_TIMEOUT_MS = 60000;
 
 const isQueryFormat = (fmt) => Boolean(fmt) && typeof fmt === 'object' && (fmt.type === 'highlights' || fmt.type === 'question');
 
+// ── Question evidence support (R25 1.4) ──────────────────────────────────────
+// A ranked unit is not support for an answer by being ranked: before
+// crawlforge-extractors 1.18.0 the ranker scored link URLs, and Wikipedia's
+// reference list answered "Who created Python…?" with citation titles — as
+// grounded:true. These checks read only the units this call returns, so they
+// hold with any extractors version.
+const EVIDENCE_LINK = /(!?)\[((?:\\.|[^[\]\\])*)\]\((?:\\.|[^()\\\s]|\([^()\s]*\))*(?:\s+"(?:\\.|[^"\\])*")?\)/g;
+const REFERENCE_HEADING = /^(?:(?:references?|notes?|citations?|sources?|footnotes?|bibliography|works cited)(?:\s*(?:,|&|and)\s*|\s*$))+$/i;
+// Words any question shares with any page; they show no unit answers it.
+const QUESTION_STOPWORDS = new Set([
+  'a', 'an', 'the', 'and', 'or', 'but', 'of', 'to', 'in', 'on', 'at', 'by', 'for', 'with', 'from', 'as', 'into', 'about',
+  'is', 'are', 'was', 'were', 'be', 'been', 'being', 'am', 'do', 'does', 'did', 'has', 'have', 'had', 'can', 'could',
+  'will', 'would', 'should', 'may', 'might', 'it', 'its', 'this', 'that', 'these', 'those', 'there', 'their', 'they',
+  'he', 'she', 'his', 'her', 'i', 'you', 'we', 'my', 'your', 'our', 'me', 'us', 'them', 'any', 'some', 'not', 'no',
+  'who', 'whom', 'whose', 'what', 'which', 'when', 'where', 'why', 'how', 'many', 'much'
+]);
+// crawlforge-extractors' word runs: a CJK run and any other run never merge.
+const CJK_CHAR = '[\\p{scx=Han}\\p{scx=Hiragana}\\p{scx=Katakana}\\p{scx=Hangul}]';
+const QUESTION_TOKEN = new RegExp(`(?:(?=${CJK_CHAR})[\\p{L}\\p{N}])+|(?:(?!${CJK_CHAR})[\\p{L}\\p{N}\\p{M}])+`, 'gu');
+const CJK_RUN = new RegExp(`^${CJK_CHAR}`, 'u');
+
+/** A unit's text as a reader sees it: link labels without URLs, no citation markers. */
+function visibleEvidenceText(text) {
+  let out = String(text ?? '');
+  for (let pass = 0; pass < 2 && out.includes(']('); pass++) {
+    out = out.replace(EVIDENCE_LINK, (match, _bang, label) => (/\]\(#cite_(?:note|ref)/.test(match) ? ' ' : label));
+  }
+  return out.toLowerCase();
+}
+
+/** An entry of a references-type list: under such a heading, or carrying a Wikipedia back-link. */
+function isReferenceUnit(unit) {
+  if (/\]\(#cite_ref/.test(String(unit.text ?? ''))) return true;
+  return typeof unit.heading === 'string' && REFERENCE_HEADING.test(unit.heading.trim());
+}
+
+/**
+ * The question's content terms, stemmed lightly ("released" → "releas") so a
+ * substring test finds their inflections; a CJK run becomes bigrams.
+ */
+function questionTerms(question) {
+  const terms = new Set();
+  for (const [run] of String(question ?? '').toLowerCase().matchAll(QUESTION_TOKEN)) {
+    if (CJK_RUN.test(run)) {
+      const chars = Array.from(run);
+      if (chars.length === 1) terms.add(run);
+      for (let i = 0; i + 1 < chars.length; i++) terms.add(chars[i] + chars[i + 1]);
+    } else if (!QUESTION_STOPWORDS.has(run)) {
+      terms.add(run.length > 4 ? run.replace(/(?:ing|ed|es|s)$/, '').replace(/(.{4,})e$/, '$1') : run);
+    }
+  }
+  return [...terms];
+}
+
+/**
+ * Whether the ranked evidence can ground an answer: some unit outside a
+ * reference list whose visible text (or heading) holds a content term of the
+ * question. Returns the reason when it cannot.
+ * @param {Array<{ text: string, heading?: string | null }>} evidence
+ * @param {string} question
+ * @returns {{ supported: boolean, reason?: string }}
+ */
+export function evidenceSupport(evidence, question) {
+  if (evidence.length === 0) return { supported: false };
+  const body = evidence.filter((unit) => !isReferenceUnit(unit));
+  if (body.length === 0) {
+    return { supported: false, reason: 'every evidence unit is a reference-list entry (References, Notes, Citations or Sources)' };
+  }
+  const terms = questionTerms(question);
+  if (terms.length === 0) return { supported: true };
+  const holds = body.some((unit) => {
+    const visible = `${visibleEvidenceText(unit.text)}\n${visibleEvidenceText(unit.heading)}`;
+    return terms.some((term) => visible.includes(term));
+  });
+  return holds
+    ? { supported: true }
+    : { supported: false, reason: 'no evidence unit outside a reference list contains a term of the question in its visible text' };
+}
+
 // The six public fields in tools/list order, each carrying the description
 // the client sees. server.js spreads this with COMPLIANCE_PARAMS instead of
 // keeping its own copy (Phase 0, 0.3).
@@ -359,7 +439,7 @@ export class UnifiedScrapeTool {
           errorDocuments: true
         }));
       } catch (err) {
-        throw new Error(`scrape: fetch failed for ${url}: ${err.message}`);
+        throw new Error(`scrape: fetch failed for ${url}: ${fetchErrorMessage(err, url)}`);
       }
       warnings.push(...fetchWarnings);
 
@@ -504,6 +584,11 @@ export class UnifiedScrapeTool {
       if (mainHtml !== null) return mainHtml;
       const main = extractMainContent(html, finalUrl);
       mainHtml = main.html ?? html;
+      if (main.tooLarge) {
+        warnings.push(
+          `mainContent: the page has more than ${MAX_READABILITY_ELEMENTS.toLocaleString('en-US')} elements, too many for main-content extraction; the whole page is used instead`
+        );
+      }
       if (main.tablesRecovered > 0) {
         warnings.push(
           `mainContent: re-attached ${main.tablesRecovered} data table(s) that main-content extraction had dropped`
@@ -638,7 +723,7 @@ export class UnifiedScrapeTool {
                 `json: output did not match the requested schema — ${(result.validationErrors || []).join('; ')}`
               );
             }
-            if (result.truncated) {
+            if (result.input_truncated) {
               warnings.push(
                 `json: page text was truncated from ${result.original_length} chars before extraction; ` +
                 'fields appearing late in the page may be missing'
@@ -686,8 +771,11 @@ export class UnifiedScrapeTool {
             const evidence = rankUnits(candidates, fmt.question, { maxUnits: QUESTION_EVIDENCE_UNITS });
             const evidenceText = evidence.map((unit) => unit.text).join('\n');
             let text = evidenceText;
-            // An empty answer rests on nothing, so it is never grounded.
-            let grounded = evidence.length > 0;
+            // An empty answer rests on nothing, so it is never grounded; nor
+            // does one resting on reference-list entries or link URLs alone.
+            const support = evidenceSupport(evidence, fmt.question);
+            if (support.reason) warnings.push(`question: ${support.reason}; grounded: false`);
+            let grounded = support.supported;
             if (fmt.mode === 'model' && evidence.length > 0) {
               try {
                 text = await this._answerQuestion(evidence, fmt.question);
@@ -698,7 +786,7 @@ export class UnifiedScrapeTool {
                   warnings.push('question: the model found no answer in the evidence; grounded: false');
                 } else {
                   const check = groundingCheck(text, evidenceText, fmt.question);
-                  grounded = check.grounded;
+                  grounded = check.grounded && support.supported;
                   if (!grounded) {
                     warnings.push(`question: the model's answer has ${check.unbacked.length} token(s) not found in the evidence or the question (${check.unbacked.join(', ')}); grounded: false`);
                   }