crawlforge-mcp-server 6.21.1 → 6.22.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "6.21.1",
+  "version": "6.22.1",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 31 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -121,7 +121,7 @@
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",
-    "crawlforge-extractors": "^1.17.0",
+    "crawlforge-extractors": "^1.18.1",
     "diff": "^9.0.0",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",
@@ -132,6 +132,7 @@
     "pdf-parse": "2.4.5",
     "playwright": "^1.54.2",
     "robots-parser": "^3.0.1",
+    "sentiment": "5.0.2",
     "turndown": "^7.2.4",
     "turndown-plugin-gfm": "^1.0.2",
     "undici": "^7.24.0",

package/server.js

@@ -113,7 +113,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "6.21.1",
+  version: "6.22.1",
   description: "Production-ready MCP server with 31 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, stateful browser sessions with element refs, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -572,10 +572,10 @@ registerToolIfEnabled("search_web", {
   try {
     if (!query && !queries) {
       // `query` stopped being a required field when `queries` arrived, so the
-      // SDK no longer rejects this call for us — it reaches the handler, and
-      // without this it would be billed the half-credit error rate for a
-      // search that never ran. Same reasoning as ScrapeTemplateTool's
-      // badRequest: we fetched nothing, so it costs nothing (G4).
+      // SDK no longer rejects this call for us — it reaches the handler, which
+      // refuses it before any search runs. Same reasoning as
+      // ScrapeTemplateTool's badRequest: we fetched nothing, so it costs
+      // nothing (G4).
       markPreflightRefusal('BAD_REQUEST');
       return { content: [{ type: "text", text: EXACTLY_ONE_QUERY_MESSAGE }], isError: true };
     }
@@ -1258,8 +1258,8 @@ registerToolIfEnabled("scrape", {
         return shot;
       });
     }
-    // A blocked verdict (0.1) is an error result: withAuth bills it at the
-    // error rate and appends the hint naming stealth_mode, and the SDK skips
+    // A blocked verdict (0.1) is an error result: withAuth charges it nothing
+    // and appends the hint naming stealth_mode, and the SDK skips
     // outputSchema validation on isError. The JSON body still carries the
     // verdict (success:false, error, blocked) the way stealth_mode's does.
     if (result.success === false) return { ...dualOutput(result), isError: true };
@@ -1274,7 +1274,7 @@ registerToolIfEnabled("agent", {
   description: "Use this when you need an autonomous agent to research, navigate, and synthesise an answer from the web - no URLs required. The agent plans search queries, fetches and filters relevant pages, and returns a prose or structured answer. model:\"pro\" uses deep multi-source research. Hard limits: maxSteps<=10, maxUrls<=20, 120s wall-clock. Confirms before pro runs. Degraded-but-useful output if no LLM keys/Ollama. Not for a URL you already have (scrape) or a question one search answers (search_web). Pages that block a plain fetch are retried in the stealth browser automatically (at most 2 a run, URLs you name first; evidence marked via:\"stealth\"). Cost: 18 credits at most - 8, plus 5 per stealth retry that gets the page; a retry that is blocked again is free. Example: agent({prompt:\"What are the top 5 MCP servers in 2025?\", maxUrls:10})",
   annotations: { title: "Agent (Autonomous)", readOnlyHint: true, destructiveHint: false, idempotentHint: false, openWorldHint: true },
   inputSchema: {
-    prompt: z.string().min(1).max(2000).describe("Natural-language task or question"),
+    prompt: z.string().trim().min(1).max(2000).describe("Natural-language task or question"),
     urls: z.array(z.string().url()).max(20).optional().describe("Optional seed URLs to include (max 20)"),
     schema: z.record(z.any()).optional().describe("Optional JSON schema for structured output"),
     model: z.enum(["default", "pro"]).optional().default("default").describe("\"default\" = SamplingClient loop (no keys needed); \"pro\" = full ResearchOrchestrator"),
@@ -1335,7 +1335,7 @@ registerToolIfEnabled("generate_llms_txt", {
       customRestrictions: z.array(z.string()).optional(),
       robotsStyle: z.boolean().optional().default(false)
     }).optional().describe("Output customization and organization details"),
-    complianceLevel: z.enum(['basic', 'standard', 'strict']).optional().default('standard').describe("Compliance level for generated guidelines"),
+    complianceLevel: z.enum(['basic', 'standard', 'strict']).optional().default('standard').describe("Applies only with outputOptions.robotsStyle:true, where it sets how many paths the llms.txt Disallow list covers (basic < standard < strict). The default llmstxt.org markdown is the same at every level; llms-full.txt only records the level as a label."),
     format: z.enum(['both', 'llms-txt', 'llms-full-txt']).optional().default('both').describe("Output format: llms.txt, llms-full.txt, or both"),
     ...MAX_INLINE_CHARS_PARAM
   }
@@ -1423,10 +1423,10 @@ function stealthScrapeFormats(formats, scraped) {
 
 // Tool: stealth_mode
 registerToolIfEnabled("stealth_mode", {
-  description: "Use this when a site blocks normal scraping - Cloudflare, Datadome, or other bot-detection systems. Renders in a real browser with randomized fingerprints, human behavior simulation, WebRTC/canvas spoofing - Camoufox (Firefox) when it is installed, Chromium otherwise, and the result names the one that ran. operation:\"scrape\" is the one-shot path: it creates a context, navigates, returns the requested formats and tears down. The create_context -> create_page -> cleanup operations remain for multi-step work. operation:\"configure\" only validates a stealthConfig and returns it with the defaults filled in - it stores nothing, so pass stealthConfig on each scrape or create_context call that should use it. robots.txt is respected on every navigation. Not a first choice: try scrape first and switch here after a 403/429/CAPTCHA/challenge page or an empty shell. Cost: 5 credits per browser operation; configure, get_stats and cleanup cost 1. Example: stealth_mode({operation:\"scrape\", url:\"https://example.com\", formats:[\"markdown\",\"links\"]})",
+  description: "Use this when a site blocks normal scraping - Cloudflare, Datadome, or other bot-detection systems. Renders in a real browser with randomized fingerprints, human behavior simulation, WebRTC/canvas spoofing - Camoufox (Firefox) when it is installed, Chromium otherwise, and the result names the one that ran. operation:\"scrape\" is the one-shot path: it creates a context, navigates, returns the requested formats and tears down. create_context then create_page is a navigation and challenge check: create_page loads urlToTest, reports its status, title and any challenge page, and closes the page before the call returns, so nothing on it carries over to a later call. For work over several calls on one page (log in, then read; click, then read), use browser_session with stealth:true. operation:\"configure\" only validates a stealthConfig and returns it with the defaults filled in - it stores nothing, so pass stealthConfig on each scrape or create_context call that should use it. robots.txt is respected on every navigation. Not a first choice: try scrape first and switch here after a 403/429/CAPTCHA/challenge page or an empty shell. Cost: 5 credits per browser operation; configure, get_stats and cleanup cost 1. Example: stealth_mode({operation:\"scrape\", url:\"https://example.com\", formats:[\"markdown\",\"links\"]})",
   annotations: { title: "Stealth Mode", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: true },
   inputSchema: {
-    operation: z.enum(['scrape', 'configure', 'create_context', 'create_page', 'get_stats', 'cleanup']).default('configure').describe("Stealth operation to perform. scrape: render one URL and return the formats. configure: validate stealthConfig and return it with defaults filled in; nothing is stored. create_context / create_page: a context to open pages in. get_stats: this server process's open contexts, browser and proxy state. cleanup: close the browser and every context."),
+    operation: z.enum(['scrape', 'configure', 'create_context', 'create_page', 'get_stats', 'cleanup']).default('configure').describe("Stealth operation to perform. scrape: render one URL and return the formats. configure: validate stealthConfig and return it with defaults filled in; nothing is stored. create_context: a stealth context to run create_page in. create_page: navigate urlToTest in that context and report status, title and any challenge page; the page is closed before the call returns. get_stats: this server process's open contexts, browser and proxy state. cleanup: close the browser and every context."),
     stealthConfig: z.object({
       level: z.enum(['basic', 'medium', 'advanced']).default('medium'),
       randomizeFingerprint: z.boolean().default(true),
@@ -1467,7 +1467,7 @@ registerToolIfEnabled("stealth_mode", {
     }).optional().describe("Stealth browser configuration with anti-detection settings. Applies to the call it is passed on and is not remembered. Chromium applies customUserAgent, customViewport, locale and timezone per call. On camoufox customUserAgent and customViewport are not applied, locale is fixed when the browser launches (run operation:\"cleanup\" first to change it), and behind a proxy locale and timezone come from the exit IP; each of these is reported in `warnings`."),
     engine: z.enum(["auto", "chromium", "camoufox", "playwright"]).optional().default("auto").describe("Browser engine: \"auto\" (default — camoufox when it is installed, Chromium otherwise, and the result says which), \"camoufox\" (Firefox-based, higher anti-detect score; fails if not installed), or \"chromium\" (\"playwright\" is the same engine under its old name)"),
     contextId: z.string().optional().describe("Browser context ID for page operations"),
-    urlToTest: z.string().url().optional().describe("URL to navigate to when creating a page"),
+    urlToTest: z.string().url().optional().describe("create_page: the URL to navigate to and check"),
     url: z.string().url().optional().describe("URL to scrape — required for operation:\"scrape\""),
     formats: z.array(z.enum(["markdown", "html", "text", "links", "metadata", "screenshot"])).optional().default(["markdown"]).describe("Formats to return from operation:\"scrape\" (default: [\"markdown\"]). \"screenshot\" returns a crawlforge://screenshot/{id} resource URI."),
     wait_for: z.number().min(0).max(30000).optional().describe("Extra wait after page load, in ms — for content that renders after DOMContentLoaded"),

package/src/core/ActionExecutor.js

@@ -51,6 +51,17 @@ const SELECTOR_WAIT_STATES = new Set(['attached', 'detached', 'visible', 'hidden
 
 const withoutHash = (url) => url.split('#')[0];
 
+/**
+ * A failed action's message, naming its selector. Playwright names it only in
+ * the "Call log:", which is stripped before the model reads the error (R25 3.2).
+ */
+function actionErrorMessage(action, error) {
+  const firstLine = String(error.message).split('\n')[0];
+  return action.selector && !firstLine.includes(action.selector)
+    ? `${action.type} on selector "${action.selector}": ${error.message}`
+    : error.message;
+}
+
 // Action schemas
 const BaseActionSchema = z.object({
   type: z.string(),
@@ -854,6 +865,11 @@ export class ActionExecutor extends EventEmitter {
           action.timeout) {
         timeout = Math.max(this.defaultTimeout, action.timeout + 5000);
       }
+      // A pause before a selector/text wait (executeWaitAction) runs on top of
+      // that wait's deadline.
+      if (action.type === 'wait' && (action.selector || action.text)) {
+        timeout += action.duration || action.milliseconds || 0;
+      }
 
       // Execute based on action type. Playwright owns the real deadline (every
       // call below is given `timeout`), so this race is only a backstop for a
@@ -902,7 +918,7 @@ export class ActionExecutor extends EventEmitter {
         id: actionId,
         type: action.type,
         success: false,
-        error: error.message,
+        error: actionErrorMessage(action, error),
         // Only when the gate refused (a `navigate` to a URL it would not
         // load), so the chain can tell an answer from a fault.
         ...(refusalCode ? { errorCode: refusalCode } : {}),
@@ -1094,14 +1110,17 @@ export class ActionExecutor extends EventEmitter {
       (!action.selector && !action.text ? action.timeout : undefined);
     if (waitTime) {
       await this.delay(waitTime);
-      return { waited: waitTime };
+      // A pause given with a selector or text waits for that too, after the
+      // pause; the selector and condition used to be dropped (R25 3.3).
+      if (!action.selector && !action.text) return { waited: waitTime };
     }
+    const waited = waitTime ? { waited: waitTime } : {};
 
     const timeout = this.actionTimeout(action);
 
     if (action.selector) {
       await this.waitForCondition(page, action.selector, action.condition, timeout);
-      return { selector: action.selector, condition: action.condition };
+      return { ...waited, selector: action.selector, condition: action.condition };
     }
 
     if (action.text) {
@@ -1110,7 +1129,7 @@ export class ActionExecutor extends EventEmitter {
         action.text,
         { timeout }
       );
-      return { text: action.text };
+      return { ...waited, text: action.text };
     }
 
     throw new Error('Wait action requires duration/milliseconds/timeout, selector, or text');

package/src/core/AgentOrchestrator.js

@@ -12,12 +12,15 @@
  * Hard stops (enforced here, not by the LLM):
  *   1. maxSteps iterations of the ACT loop
  *   2. maxUrls total URLs fetched
- *   3. wallClockMs wall-clock milliseconds (default 120 000)
+ *   3. wallClockMs wall-clock milliseconds (default 120 000); every LLM call
+ *      gets only the time left, and a run that reaches it returns degraded
+ *      with the evidence gathered so far
  *
  * No-LLM-key path: if all LLM calls fail, return collected evidence + {degraded:true}.
  * pro model: delegates to ResearchOrchestrator.conductResearch() for richer synthesis.
  */
 
+import { load } from 'cheerio';
 import { fetchAndParse } from '../tools/extract/_fetchAndParse.js';
 import { SamplingClient } from './SamplingClient.js';
 import { fenceUntrusted } from '../utils/untrustedContent.js';
@@ -31,6 +34,12 @@ const DEFAULT_MAX_URLS = 10;
 // A stealth retry launches a browser and waits out a challenge; one started
 // with less than this left on the wall clock would overrun the hard stop.
 const ESCALATION_MIN_REMAINING_MS = 20_000;
+// PLAN is optional (the raw prompt is the fallback query), so a slow model
+// may spend at most this share of the wall clock on it, never all of it.
+const PLAN_MAX_SHARE = 0.25;
+// A corrective rewrite is a whole synthesis call; with less than this left it
+// would run into the hard stop, so the first answer is kept and flagged.
+const REWRITE_MIN_REMAINING_MS = 15_000;
 
 /** A refusal (robots.txt, blocklist) is a decision, not a wall — a browser must not retry it. */
 function isRefusal(err) {
@@ -67,7 +76,7 @@ function truncate(text, maxChars = 8000) {
  * strings that do not (deduplicated, in answer order). Comparison is
  * case-insensitive with whitespace and thousands separators removed, so
  * "6.3.3" matches "6.3.3" and "1,234" matches "1 234". URLs in the answer are
- * ignored: they are citations, not claims.
+ * skipped here: they are citations, checked by unverifiedUrls.
  */
 const MONTHS = '(?:jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)[a-z]*';
 const VALUE_PATTERNS = [
@@ -101,16 +110,81 @@ export function unverifiedValues(answer, sourceText) {
   return missing.filter((v, i) => !missing.some((o, j) => j !== i && o.length > v.length && o.includes(v)));
 }
 
+/**
+ * The http(s) URLs in a text, trailing punctuation removed. A closing
+ * parenthesis stays only when it closes one opened inside the URL, so
+ * "[HN](https://x/item?id=1)" yields the link and "/wiki/Foo_(bar)" survives.
+ */
+export function urlsIn(text) {
+  return (String(text || '').match(/https?:\/\/[^\s<>"'`\]]+/g) || []).map(url => {
+    for (;;) {
+      if (/[.,;:!?*'"]$/.test(url)) url = url.slice(0, -1);
+      else if (url.endsWith(')') && url.split('(').length < url.split(')').length) url = url.slice(0, -1);
+      else return url;
+    }
+  });
+}
+
+/**
+ * A URL as a citation is compared: scheme, "www.", fragment and trailing
+ * slash dropped, host lowercased, "&amp;" decoded. Null when it does not parse.
+ */
+function citationKey(url) {
+  try {
+    const u = new URL(String(url).replace(/&amp;/g, '&'));
+    return `${u.host.toLowerCase().replace(/^www\./, '')}${u.pathname.replace(/\/+$/, '')}${u.search}`;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * URLs the answer cites that no fetched page supports: not a fetched URL
+ * (before or after redirects), not a link on a fetched page, and not written
+ * in the source text. The R25 re-check (2026-10-19) answered the current HN
+ * top story with the right title and a link to a 2021 story that none of the
+ * sources contained, and the value-only check above skipped it as a citation.
+ *
+ * @param {string} answer
+ * @param {Iterable<string>} citable fetched URLs, their final URLs and the links on them
+ * @param {string} [sourceText]
+ * @returns {string[]} the cited URLs as written, deduplicated
+ */
+export function unverifiedUrls(answer, citable, sourceText = '') {
+  const known = new Set([...citable, ...urlsIn(sourceText)].map(citationKey).filter(Boolean));
+  const seen = new Set();
+  return urlsIn(answer).filter(url => {
+    const key = citationKey(url);
+    if (!key || seen.has(key)) return false;
+    seen.add(key);
+    return !known.has(key);
+  });
+}
+
+/** Absolute http(s) hrefs of the links on a parsed page. */
+function pageLinks($, baseUrl) {
+  const links = [];
+  $('a[href]').each((_, el) => {
+    try {
+      const url = new URL($(el).attr('href'), baseUrl);
+      if (url.protocol === 'http:' || url.protocol === 'https:') links.push(url.href);
+    } catch {
+      // An unparsable href cites nothing.
+    }
+  });
+  return links;
+}
+
 /**
  * Current-state task gate: prompts about the live "now" ("right now",
- * "currently", "today", "latest", "#1 … now") go stale through search alone —
+ * "current", "currently", "today", "latest", "#1 … now") go stale through search alone —
  * dated articles rank for the task's words while the live page does not
  * (live repro 2026-08-26: "#1 story on Hacker News right now" answered from
  * a January thread; news.ycombinator.com was never fetched). Detected
  * deterministically here (never LLM-trusted) so PLAN steers search toward
  * the live entity page and SHAPE answers from it, not from dated results.
  */
-const CURRENT_STATE_RE = /\b(right now|currently|today|tonight|at the moment|as of (now|today)|latest|this (week|month|morning))\b|#\d+[^.?!]*\bnow\b/i;
+const CURRENT_STATE_RE = /\b(right now|current|currently|today|tonight|at the moment|as of (now|today)|latest|this (week|month|morning))\b|#\d+[^.?!]*\bnow\b/i;
 
 /** Version numbers as they appear in an answer, e.g. "2.11.4". */
 const VERSION_IN_TEXT = /\b\d+\.\d+(?:\.\d+){0,2}\b/g;
@@ -172,6 +246,66 @@ export function isCurrentStateTask(prompt) {
   return CURRENT_STATE_RE.test(prompt || '');
 }
 
+/** The registrable label of a hostname: "python" for www.python.org, "bbc" for bbc.co.uk. */
+function siteLabel(hostname) {
+  const labels = hostname.toLowerCase().split('.');
+  const ccSecondLevel = labels.length >= 3 && labels.at(-1).length === 2 && labels.at(-2).length <= 3;
+  return labels[Math.max(labels.length - (ccSecondLevel ? 3 : 2), 0)];
+}
+
+/**
+ * True when a search result is the entity's own site: its domain is the
+ * entity's name (python.org for "Python"), or it is a site root whose title
+ * leads with exactly that name (news.ycombinator.com, titled "Hacker News").
+ *
+ * The domain vote alone picked the wrong site in both R25 repros (2026-10-19).
+ * Brave returns each origin about once, so for "Python" every origin tied and
+ * the top-ranked one, en.wikipedia.org, was promoted — its root is the
+ * Wikipedia Main Page. For "Hacker News" thehackernews.com appeared twice and
+ * outvoted news.ycombinator.com. Neither look-alike passes this test: its
+ * domain is "thehackernews", its title "The Hacker News | …".
+ */
+export function isOfficialResult(entity, url, title) {
+  const name = (entity || '').trim().toLowerCase().replace(/\s+/g, ' ');
+  if (!name) return false;
+  let u;
+  try { u = new URL(url); } catch { return false; }
+  const compact = s => s.replace(/[^\p{L}\p{N}]/gu, '');
+  if (compact(siteLabel(u.hostname)) === compact(name)) return true;
+  const lead = (title || '').split(/\s+[|–—-]\s+|:\s/)[0].trim().toLowerCase().replace(/\s+/g, ' ');
+  return u.pathname === '/' && !u.search && lead === name;
+}
+
+/**
+ * A page's share of the synthesis budget. Cut from the top, a page whose
+ * answer sits below its navigation menus loses it: python.org's
+ * "Latest: Python 3.15.0" sits ~3 500 characters in, under a wall of menu
+ * links, past a 2 400-character share (R25, 2026-10-19). When a line past the
+ * cut matches more of the task's terms than any line before it, the second
+ * half of the share goes to the text around that line.
+ */
+export function sliceForSynthesis(text, cap, terms) {
+  if (!text || text.length <= cap) return text;
+  const score = line => {
+    const lc = line.toLowerCase();
+    return terms.filter(t => lc.includes(t)).length;
+  };
+  let keptBest = 0;
+  let best = 0;
+  let bestAt = -1;
+  let pos = 0;
+  for (const line of text.split('\n')) {
+    const s = score(line);
+    if (pos + line.length <= cap) keptBest = Math.max(keptBest, s);
+    else if (s > best) { best = s; bestAt = pos; }
+    pos += line.length + 1;
+  }
+  if (best <= keptBest) return truncate(text, cap);
+  const head = Math.floor(cap / 2);
+  const from = Math.max(head, bestAt - 200);
+  return `${text.slice(0, head)}\n[...]\n${truncate(text.slice(from), cap - head)}`;
+}
+
 /**
  * The names a prompt uses: words carrying a capital letter that is not just
  * the capital opening a sentence ("TypeScript", "Hacker", "News", "NASA"; not
@@ -309,7 +443,9 @@ export class AgentOrchestrator {
     usage.charged = 0;
 
     const startTime = Date.now();
-    const deadline = () => (Date.now() - startTime) >= wallClockMs;
+    const remaining = () => wallClockMs - (Date.now() - startTime);
+    const deadline = () => remaining() <= 0;
+    const timeLimitReason = `Stopped at the ${wallClockMs} ms time limit; the evidence gathered before it is returned.`;
 
     // Hard-cap params regardless of what caller sends
     const capSteps = Math.min(maxSteps, 10);
@@ -358,7 +494,10 @@ export class AgentOrchestrator {
           ? `The task asks about a CURRENT live state: the FIRST query must be ONLY the name of the site or thing whose current state is asked, nothing else. `
           : '') +
         `Output ONLY the queries, one per line, no preamble or numbering:\n\n${prompt}`;
-      const { text } = await this._getSamplingClient().complete(planPrompt, { maxTokens: 200 });
+      const { text } = await this._getSamplingClient().complete(planPrompt, {
+        maxTokens: 200,
+        timeoutMs: Math.min(remaining(), wallClockMs * PLAN_MAX_SHARE)
+      });
       const lines = text.split('\n')
         .map(l => l.replace(/^[-*\d.)\s]+/, '').trim())
         // A model that fences its list ("```\nq1\nq2\n```") or quotes each
@@ -477,8 +616,10 @@ export class AgentOrchestrator {
     // queue and first at SHAPE time so synthesis answers from it, never from
     // a dated article. A wrong root self-heals: it fails the relevance gate
     // and never enters evidence.
+    let promotedOrigin = null;
     if (currentState && searchResults.length > 0) {
       const originCounts = new Map();
+      const official = new Set();
       // Only the bare entity query votes: the fact query added above returns
       // guides and news sites, which must not outvote the official domain.
       const entityQuery = searchQueries[0];
@@ -487,14 +628,23 @@ export class AgentOrchestrator {
         try {
           const origin = new URL(s.url).origin;
           originCounts.set(origin, (originCounts.get(origin) || 0) + 1);
+          if (isOfficialResult(entityQuery, s.url, s.title)) official.add(origin);
         } catch { /* unparsable result url */ }
       }
+      // The entity's own site wins over any vote count; only when no result
+      // is official does the vote decide, and a tied vote decides nothing —
+      // with one result per origin it would just promote the top-ranked one.
       let bestOrigin = null;
       let bestCount = 0;
+      let tied = false;
       for (const [origin, count] of originCounts) {
+        if (official.size > 0 && !official.has(origin)) continue;
         // Strict > keeps the earliest-seen (top-ranked) origin on ties.
-        if (count > bestCount) { bestOrigin = origin; bestCount = count; }
+        if (count > bestCount) { bestOrigin = origin; bestCount = count; tied = false; }
+        else if (count === bestCount) tied = true;
       }
+      if (tied && official.size === 0) bestOrigin = null;
+      promotedOrigin = bestOrigin;
       if (bestOrigin) {
         const liveRoot = `${bestOrigin}/`;
         if (!priorityUrls.includes(liveRoot)) priorityUrls.push(liveRoot);
@@ -512,6 +662,9 @@ export class AgentOrchestrator {
     // cap whenever it was smaller than capUrls, leaving maxUrls unreachable
     // at its default. Both caps remain fully enforced (neither is weakened).
     const evidence = [];
+    // URLs an answer may cite: every evidence page's URL (requested and
+    // final) and the links on it.
+    const citable = [];
     let urlsFetched = 0;
     let step = 0;
     const warnings = [];
@@ -527,7 +680,7 @@ export class AgentOrchestrator {
       let plainError = null;
       let verdict = null;
       try {
-        plain = await fetchAndParse(url, { timeoutMs: 10000, errorDocuments: true });
+        plain = await fetchAndParse(url, { timeoutMs: Math.max(1, Math.min(10000, remaining())), errorDocuments: true });
         verdict = documentVerdict(
           { url: plain.finalUrl, title: plain.$('title').first().text().trim(), text: plain.textContent, html: plain.html, status: plain.status },
           { fetcher: 'a plain fetch', rendered: false, contentReturned: false }
@@ -547,6 +700,7 @@ export class AgentOrchestrator {
             text: truncate(plain.textContent),
             step
           });
+          citable.push(url, plain.finalUrl, ...pageLinks(plain.$, plain.finalUrl));
           continue;
         }
         // A real page that is simply off-topic: nothing to retry.
@@ -591,6 +745,7 @@ export class AgentOrchestrator {
               via: 'stealth',
               step
             });
+            citable.push(url, stealth.url || url, ...pageLinks(load(stealth.html || ''), stealth.url || url));
             continue;
           }
           warnings.push(`stealth retry of ${url} ${rendered.success ? 'returned a page unrelated to the task' : `did not get the page either${rendered.blocked?.vendor ? ` (${rendered.blocked.vendor})` : ''}`}`);
@@ -601,6 +756,7 @@ export class AgentOrchestrator {
 
       if (relevantExcerpt) {
         evidence.push({ url, title: relevantExcerpt.title, text: relevantExcerpt.text, snippet: true, step });
+        citable.push(url);
       }
     }
 
@@ -622,15 +778,18 @@ export class AgentOrchestrator {
       }))
       .sort((a, b) => b._score - a._score);
     const perSourceCap = Math.max(1500, Math.floor(12000 / Math.max(evidence.length, 1)));
+    // Question words match any prose ("What is your name?" in a code sample).
+    const sliceTerms = promptTerms.map(t => t.replace(/[^\p{L}\p{N}]/gu, ''))
+      .filter(t => t.length > 3 && !/^(what|which|when|where|does|that|this|with|from|have|about)$/.test(t));
     const combinedText = orderedEvidence
-      .map(e => `--- Source: ${e.url}${e.snippet ? ' (search-result snippet; the page itself could not be fetched)' : ''} ---\n${truncate(e.text, perSourceCap)}`)
+      .map(e => `--- Source: ${e.url}${e.snippet ? ' (search-result snippet; the page itself could not be fetched)' : ''} ---\n${sliceForSynthesis(e.text, perSourceCap, sliceTerms)}`)
       .join('\n\n');
 
     if (!combinedText.trim()) {
       return {
         success: true,
         degraded: true,
-        reason: 'No content could be fetched for the given prompt.',
+        reason: deadline() ? timeLimitReason : 'No content could be fetched for the given prompt.',
         search_results: searchResults,
         evidence: [],
         answer: null,
@@ -643,15 +802,19 @@ export class AgentOrchestrator {
     }
 
     // Schema path: use ExtractWithLlm for structured output
-    if (schema && Object.keys(schema).length > 0) {
+    if (schema && Object.keys(schema).length > 0 && !deadline()) {
       try {
         const extractWithLlm = await this._getExtractWithLlm();
         const result = await extractWithLlm.execute({
           content: combinedText,
           prompt: `From the following research sources, answer this task and extract structured data:\n${prompt}`,
           schema,
-          provider: 'auto'
+          provider: 'auto',
+          signal: AbortSignal.timeout(Math.max(0, remaining()))
         });
+        // Out of time: fall through so the time-limit result below carries
+        // the full evidence.
+        if (!result.success && deadline()) throw new Error(result.error);
         // A required field left null is an incomplete answer, not a
         // structured success (R24, 2026-10-03).
         const missingRequired = result.success && Array.isArray(schema.required)
@@ -686,12 +849,14 @@ export class AgentOrchestrator {
     let degraded = false;
     let degradedReason;
     let unverified = [];
+    let unverifiedLinks = [];
     let forumOnlyVersions = [];
     // Distinct from `degraded`: a run can be degraded because its version
     // claims are unsupported and still have had its provenance checked.
     let synthesisFailed = false;
 
     try {
+      if (deadline()) throw new Error('time limit reached before synthesis');
       // Wording matters for small local models (llama3.2-class): without the
       // "already fetched / do not refuse" framing they answer "I cannot access
       // the internet" or "the sources do not contain it" even when the answer
@@ -706,9 +871,13 @@ export class AgentOrchestrator {
         `Rules:\n` +
         // Short and imperative on purpose: the executing model is a small
         // local one (gemma3:4b-class) and ignores hedged phrasing.
-        (currentState
+        // "The first source is the live page" is only true when GATHER
+        // promoted one; otherwise the first source is just the best term match.
+        (currentState && promotedOrigin
           ? `- The task asks about the CURRENT state. Answer from the FIRST source below (the live page); if the FIRST source does not state the answer, take it from the next sources in order. NEVER present older or dated content as the current answer.\n`
-          : '') +
+          : currentState
+            ? `- The task asks about the CURRENT state. Prefer the subject's own official site over third-party articles. NEVER present older or dated content as the current answer.\n`
+            : '') +
         `- Answer ONLY from the provided sources; do not use outside knowledge.\n` +
         `- Read the sources carefully before concluding anything is missing from them.\n` +
         `- Cite the exact source URL(s) you used.\n` +
@@ -716,7 +885,7 @@ export class AgentOrchestrator {
         `- NEVER invent or guess a URL; cite only URLs that appear in the sources above.\n\n` +
         `Provide a clear, concise answer.`;
 
-      const { text } = await this._getSamplingClient().complete(synthesisPrompt, { maxTokens: 1024 });
+      const { text } = await this._getSamplingClient().complete(synthesisPrompt, { maxTokens: 1024, timeoutMs: remaining() });
       answer = text;
 
       // A small model fills gaps from memory: "Swift 5.9.3", "commander 8.6.2
@@ -724,18 +893,24 @@ export class AgentOrchestrator {
       // 2026-09-04). Versions, dates and counts in the answer must appear in
       // the source text; what does not gets one corrective rewrite, and
       // whatever still remains is named in the answer and in `provenance`.
+      // A cited URL is checked the same way (R25 re-check, 2026-10-19: the
+      // right HN title with an invented item link).
+      const sourceText = evidence.map(e => e.text || '').join('\n');
       unverified = unverifiedValues(answer, combinedText);
-      if (unverified.length > 0) {
+      unverifiedLinks = unverifiedUrls(answer, citable, sourceText);
+      if ((unverified.length > 0 || unverifiedLinks.length > 0) && remaining() >= REWRITE_MIN_REMAINING_MS) {
         const retryPrompt =
           `${synthesisPrompt}\n\nYour previous answer was:\n${fenceUntrusted(answer, 'previous answer')}\n` +
-          `It contains values that appear nowhere in the source text: ${unverified.map(v => `"${v}"`).join(', ')}. ` +
+          (unverified.length > 0 ? `It contains values that appear nowhere in the source text: ${unverified.map(v => `"${v}"`).join(', ')}. ` : '') +
+          (unverifiedLinks.length > 0 ? `It cites URLs that are not among the sources or the links on them: ${unverifiedLinks.join(', ')}. Cite only the source URLs given above. ` : '') +
           `Rewrite the answer. Keep only values that the sources state, spelled as the sources spell them. ` +
           `Where the sources do not state a value, say that it is not stated instead of guessing.`;
         try {
-          const { text: rewritten } = await this._getSamplingClient().complete(retryPrompt, { maxTokens: 1024 });
+          const { text: rewritten } = await this._getSamplingClient().complete(retryPrompt, { maxTokens: 1024, timeoutMs: remaining() });
           if (rewritten && rewritten.trim()) {
             answer = rewritten;
             unverified = unverifiedValues(answer, combinedText);
+            unverifiedLinks = unverifiedUrls(answer, citable, sourceText);
           }
         } catch {
           // Keep the first answer; it is flagged below.
@@ -746,6 +921,11 @@ export class AgentOrchestrator {
           `\n\nProvenance warning: ${unverified.map(v => `"${v}"`).join(', ')} ` +
           `${unverified.length === 1 ? 'does' : 'do'} not appear in the fetched sources and may be invented.`;
       }
+      if (unverifiedLinks.length > 0) {
+        answer +=
+          `\n\nProvenance warning: ${unverifiedLinks.join(', ')} ` +
+          `${unverifiedLinks.length === 1 ? 'is' : 'are'} not among the fetched pages or the links on them and may be invented.`;
+      }
 
       // Asked for a current version, the agent answered "2.4.3" from a forum
       // thread about a different project while the project's own site and its
@@ -756,7 +936,7 @@ export class AgentOrchestrator {
       // presented as the answer.
       if (currentState && isVersionQuestion(prompt)) {
         forumOnlyVersions = unsupportedVersionClaims(answer, evidence);
-        if (forumOnlyVersions.length > 0) {
+        if (forumOnlyVersions.length > 0 && remaining() >= REWRITE_MIN_REMAINING_MS) {
           const versionRetryPrompt =
             `${synthesisPrompt}\n\nYour previous answer was:\n${fenceUntrusted(answer, 'previous answer')}\n` +
             `It states ${forumOnlyVersions.map(v => `"${v}"`).join(', ')}, which ${forumOnlyVersions.length === 1 ? 'appears' : 'appear'} ` +
@@ -765,7 +945,7 @@ export class AgentOrchestrator {
             `Rewrite the answer using only versions stated by the project's own sources. ` +
             `If none of them states a current version, say plainly that the fetched sources do not state one.`;
           try {
-            const { text: rewritten } = await this._getSamplingClient().complete(versionRetryPrompt, { maxTokens: 1024 });
+            const { text: rewritten } = await this._getSamplingClient().complete(versionRetryPrompt, { maxTokens: 1024, timeoutMs: remaining() });
             if (rewritten && rewritten.trim()) {
               answer = rewritten;
               forumOnlyVersions = unsupportedVersionClaims(answer, evidence);
@@ -786,10 +966,36 @@ export class AgentOrchestrator {
             `only in discussion pages. The authoritative pages fetched state no version.`;
         }
       }
+
+      // A promoted live page is only worth its +1000 if the answer came from
+      // it. R25 (2026-10-19) answered "not stated" over a promoted Wikipedia
+      // Main Page, and the HN question from thehackernews.com, both with
+      // degraded:false. The answer counts as the live page's when it cites
+      // that host or states a value (version, date, number) the page shows.
+      if (currentState && promotedOrigin && !degraded) {
+        const host = u => { try { return new URL(u).hostname.replace(/^www\./, ''); } catch { return null; } };
+        const liveHost = host(promotedOrigin);
+        const liveText = evidence.filter(e => host(e.url) === liveHost).map(e => e.text || '').join('\n');
+        // unverifiedValues against an empty text lists every value the answer states.
+        const stated = unverifiedValues(answer, '');
+        const fromLivePage = answer.toLowerCase().includes(liveHost) ||
+          (liveText !== '' && unverifiedValues(answer, liveText).length < stated.length);
+        if (!fromLivePage) {
+          degraded = true;
+          degradedReason =
+            `The answer does not come from the live page ${promotedOrigin}/: ` +
+            (liveText
+              ? 'it neither cites that page nor states a value shown on it.'
+              : 'that page could not be fetched or did not match the task.') +
+            ' It may describe a dated or third-party source rather than the current state.';
+        }
+      }
     } catch (err) {
       degraded = true;
       synthesisFailed = true;
-      degradedReason = `LLM synthesis unavailable: ${err.message}`;
+      degradedReason = err.code === 'LLM_TIMEOUT' || deadline()
+        ? timeLimitReason
+        : `LLM synthesis unavailable: ${err.message}`;
       // Return raw evidence so the host LLM can synthesize
       answer = null;
     }
@@ -809,6 +1015,7 @@ export class AgentOrchestrator {
       provenance: {
         checked: !synthesisFailed,
         unverified,
+        ...(unverifiedLinks.length > 0 ? { unverified_urls: unverifiedLinks } : {}),
         ...(forumOnlyVersions.length > 0 ? { unsupported_versions: forumOnlyVersions } : {})
       }
     };