crawlforge-mcp-server 6.22.3 → 6.23.0

package/README.md

@@ -276,6 +276,12 @@ export CRAWLFORGE_STEALTH_PROXIES="http://user:pass@gw.provider.net:8080"
 
 # Optional: turn off the clearance jar (on by default) — see "Stealth engines and proxies"
 export CRAWLFORGE_CLEARANCE_JAR="off"
+
+# Optional: also refuse private networks (RFC1918, ULA, CGNAT, ...) — see "Security & Approvals"
+export SSRF_STRICT="true"
+
+# Optional: the only directories process_document may read local files from (":"-separated, ";" on Windows)
+export CRAWLFORGE_ALLOWED_DIRS="$HOME/Documents:$HOME/Downloads"
 ```
 
 ### MCP Spec Features
@@ -380,7 +386,10 @@ Once configured, use these tools in your AI assistant:
 
 ### Security & Approvals
 
-- **SSRF enforcement**: Every scraped URL is validated before the request is sent — http/https only; blocks loopback, RFC1918, IPv6 private/link-local ranges, cloud metadata endpoints (GCP, Azure), and dangerous ports (SSH, SMTP, DNS, MySQL, Postgres, Redis, MongoDB, etc.). Redirects are re-validated each hop, capped at 5.
+- **SSRF enforcement**: Every scraped URL is validated before the request is sent, and every connection (each redirect hop included, capped at 5) is checked against the address it actually connects to — http/https only, cloud metadata hostnames (GCP, Azure) refused. Two levels:
+  - **Default (Stage 1)** blocks loopback (`127.0.0.0/8`, `::1`), the unspecified addresses (`0.0.0.0/8`, `::`), link-local and cloud metadata (`169.254.0.0/16` incl. `169.254.169.254`, `fe80::/10`). It **allows private networks** (RFC1918 `10/8`, `172.16/12`, `192.168/16`, IPv6 ULA): run on your own machine, the server acts for you, and reading a router page, a NAS or an intranet docs site on your LAN is a legitimate use that the stricter level would break.
+  - **`SSRF_STRICT=true`** adds every private and reserved range: RFC1918, ULA (`fc00::/7`), CGNAT (`100.64/10`), multicast, reserved, IPv4-compatible (`::/96`) and NAT64 (`64:ff9b::/96`) IPv6. Set it if an agent that reads untrusted pages (and could be prompt-injected into fetching a LAN address) should not reach your local network, or whenever the server is shared.
+- **Local files** (`process_document` with `sourceType` `file` or `pdf_file`; refused outright when the server is served to a network): a path is judged by its realpath, so a symlink counts as its target. Only `.pdf`, `.docx`, `.txt`, `.md`, `.html` and `.htm` files are read, and nothing inside a dot-directory (`~/.ssh`, `~/.aws`, `~/.config`, `~/.crawlforge`, ...). `CRAWLFORGE_ALLOWED_DIRS` (directories separated by `:`, `;` on Windows) narrows reads to those directories instead; inside them the dot-directory rule is lifted, the extension rule is not.
 - **Backend endpoint guard** (v3.0.18): The server's own calls to CrawlForge.dev use a separate fail-closed allow-list (`{crawlforge.dev, www.crawlforge.dev, api.crawlforge.dev}`, HTTPS required). Setting `CRAWLFORGE_API_URL` to an arbitrary host is blocked at parse time.
 - **Action allowlist**: `scrape_with_actions` accepts only 11 action types (`snapshot`, `wait`, `click`, `type`, `press`, `scroll`, `screenshot`, `executeJavaScript`, `select`, `hover`, `navigate`). No download, file-write, or arbitrary cross-page navigation primitives exist — `navigate` goes through the same SSRF and robots.txt gate as the initial URL.
 - **JavaScript gate**: The `executeJavaScript` action throws by default. Set `ALLOW_JAVASCRIPT_EXECUTION=true` at deploy time to enable (not recommended in production).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "6.22.3",
+  "version": "6.23.0",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 31 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",

package/server.js

@@ -113,7 +113,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "6.22.3",
+  version: "6.23.0",
   description: "Production-ready MCP server with 31 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, stateful browser sessions with element refs, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -427,7 +427,7 @@ const refusedByGate = (result) => result?.success === false && preflightRefusal(
 
 // Tool: fetch_url
 registerToolIfEnabled("fetch_url", {
-  description: "Use this for a raw HTTP body - JSON, XML, plain text, an API response - or for the status code, headers and response time. Returns the body unprocessed. Not for HTML you intend to read: scrape returns markdown from one fetch, so fetch_url followed by extract_* is a double fetch. Not for JS-rendered or bot-protected pages (scrape, then stealth_mode). Supports custom headers (e.g. auth tokens) and a timeout. Cost: 1 credit. Example: fetch_url({url: \"https://api.example.com/v1/items\", timeout: 15000})",
+  description: "Use this for a raw HTTP body - JSON, XML, plain text, an API response - or for the status code, headers and response time. Returns the body unprocessed. Not for HTML you intend to read: scrape returns markdown from one fetch, so fetch_url followed by extract_* is a double fetch. Not for JS-rendered or bot-protected pages (scrape, then stealth_mode). Supports custom headers (e.g. auth tokens) and a timeout; the result reports redirected and final_url. Cost: 1 credit. Example: fetch_url({url: \"https://api.example.com/v1/items\", timeout: 15000})",
   annotations: { title: "Fetch URL", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The URL to fetch content from"),
@@ -504,7 +504,10 @@ registerToolIfEnabled("scrape_structured", {
   annotations: { title: "Scrape Structured Data", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The URL to scrape"),
-    selectors: z.record(z.string()).describe("CSS selectors mapping field names to selectors. Append @attr to extract an attribute instead of text (e.g. \"a.link@href\", \"img@src\")"),
+    selectors: z.record(z.union([
+      z.string(),
+      z.object({ selector: z.string(), multiple: z.boolean().optional() })
+    ])).describe("CSS selectors mapping field names to selectors. Append @attr to extract an attribute instead of text (e.g. \"a.link@href\", \"img@src\"); @href and @src values are resolved to absolute URLs. A field may also be {selector, multiple: true} to return every match as an array - inside a row_selector row a field otherwise keeps only the first match"),
     row_selector: z.string().optional().describe("CSS selector for the repeating row/container element. When set, each field in selectors is matched inside each row and data is an array of row-aligned records ({field: value|null}) instead of parallel arrays"),
     max_results: z.number().int().min(1).optional().describe("Maximum number of matches to return per field when a selector matches multiple elements, or the maximum number of rows when row_selector is set"),
     ...COMPLIANCE_PARAMS
@@ -754,7 +757,7 @@ registerToolIfEnabled("extract_content", {
 
 // Tool: process_document
 registerToolIfEnabled("process_document", {
-  description: "Use this to extract text from a PDF or DOCX URL or file - research papers, contracts, reports. The body decides how it is read: a PDF or Word document served under sourceType \"url\" still reaches its parser, and a body this tool cannot read (an image, an archive) is refused by name. Returns structured sections, metadata, and word count; for a PDF, pagesRead names the pages read (maxPages defaults to 100). Not for ordinary web pages (scrape), though an HTML URL is accepted. Cost: 2 credits. Example: process_document({source: \"https://example.com/report.pdf\", sourceType: \"pdf_url\"})",
+  description: "Use this to extract text from a PDF or DOCX URL or file - research papers, contracts, reports. The body decides how it is read: a PDF or Word document served under sourceType \"url\" still reaches its parser, and a body this tool cannot read (an image, an archive) is refused by name. Returns structured sections, metadata, and word count; for a PDF, pagesRead names the pages read (maxPages defaults to 100). Not for ordinary web pages (scrape), though an HTML URL is accepted. A local file (sourceType file/pdf_file) must be .pdf, .docx, .txt, .md or .html and outside dot-directories; CRAWLFORGE_ALLOWED_DIRS limits reads to the listed directories. Cost: 2 credits. Example: process_document({source: \"https://example.com/report.pdf\", sourceType: \"pdf_url\"})",
   annotations: { title: "Process Document", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     source: z.string().describe("Document source - URL or file path"),
@@ -1060,7 +1063,7 @@ registerToolIfEnabled("scrape_with_actions", {
       includeMetadata: z.boolean().default(true),
       includeLinks: z.boolean().default(true),
       includeImages: z.boolean().default(true)
-    }).optional().describe("Content extraction options. selectors results are returned as content.json.extracted, so include \"json\" in formats when passing selectors — without it the extraction is not part of the response."),
+    }).optional().describe("Content extraction options. selectors results are returned as content.json.extracted, so include \"json\" in formats when passing selectors — without it the extraction is not part of the response. Append @attr to a selector to read an attribute instead of the text (\"a.next@href\"); @value on an input, textarea or select reads what it holds after the actions (typed or selected), and any other @attr reads the attribute as written in the page's HTML."),
     continueOnActionError: z.boolean().default(false).describe("Continue executing actions if one fails"),
     maxRetries: z.number().min(0).max(3).default(0).describe("Whole-chain retries on failure (0-3). A retry re-navigates to the starting URL and replays every action; each attempt is reported under attempts[]."),
     screenshotOnError: z.boolean().default(true).describe("Capture screenshot when an error occurs"),
@@ -1294,7 +1297,7 @@ registerToolIfEnabled("agent", {
 
 // Tool: track_changes
 registerToolIfEnabled("track_changes", {
-  description: "Use this to monitor a URL for content changes over time - competitor pricing, regulation updates, product availability. Start with operation:\"create_baseline\", then periodically use operation:\"compare\" to diff; repeated compare calls on the same URL are expected. Supports webhooks and scheduled monitoring, and scheduledMonitorOptions.hosted:true runs the monitor on CrawlForge's servers with email and signed webhooks. Not for a one-off read (scrape). Cost: 3 credits. Example: track_changes({url: \"https://example.com/pricing\", operation: \"create_baseline\"})",
+  description: "Use this to monitor a URL for content changes over time - competitor pricing, regulation updates, product availability. Start with operation:\"create_baseline\", then periodically use operation:\"compare\" to diff; repeated compare calls on the same URL are expected. Supports webhooks and scheduled monitoring, and scheduledMonitorOptions.hosted:true runs the monitor on CrawlForge's servers with email and signed webhooks. Not for a one-off read (scrape). A compare always uses the baseline's granularity/customSelectors/excludeSelectors; different values are reported in ignoredOptions. Cost: 3 credits. Example: track_changes({url: \"https://example.com/pricing\", operation: \"create_baseline\"})",
   annotations: { title: "Track Changes", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: true },
   // The tool module owns the schema (G5); this is the same shape it validates with.
   inputSchema: {
@@ -1523,7 +1526,11 @@ registerToolIfEnabled("stealth_mode", {
         };
         if (verdict.blocked) result.blocked = verdict.blocked;
         if (verdict.error) result.error = verdict.error;
-        if (scraped.gracedMs > 0) result.waited_for_render_ms = scraped.gracedMs;
+        // The requested wait_for and the render grace the browser added on top
+        // are separate numbers; reporting only the grace made wait_for look
+        // unapplied (wait_for:5000 showed as 4 ms, live sweep 2026-10-20).
+        if (scraped.waitedMs > 0) result.waited_ms = scraped.waitedMs;
+        if (scraped.gracedMs > 0) result.render_grace_ms = scraped.gracedMs;
 
         // Screenshots follow the same crawlforge://screenshot/{id} pattern as
         // scrape and scrape_with_actions, so the base64 never bloats the result.

package/src/core/ActionExecutor.js

@@ -9,7 +9,7 @@ import { EventEmitter } from 'events';
 import { createHash } from 'node:crypto';
 import { assertUrlAllowed, assertNavigationAllowed } from '../utils/ssrfGuard.js';
 import { browserPreflight, redirectGate, pageMoveGate, gateRefusalCode, paceBrowserAction } from '../utils/robotsGate.js';
-import { isRef, resolveRef, captureSnapshot } from './browser/snapshot.js';
+import { isRef, resolveRef, reresolveRef, captureSnapshot } from './browser/snapshot.js';
 import { settlePage } from './browser/settle.js';
 import { handleConsent } from './browser/consent.js';
 import { stealthDocumentVerdict } from '../utils/stealthVerdict.js';
@@ -55,6 +55,27 @@ const withoutHash = (url) => url.split('#')[0];
  * A failed action's message, naming its selector. Playwright names it only in
  * the "Call log:", which is stripped before the model reads the error (R25 3.2).
  */
+/**
+ * The live `.value` of every input, textarea and select, in document order.
+ *
+ * page.content() serializes attributes, and typing changes the property, not
+ * the attribute: `input[type=number]@value` came back null after typing "42"
+ * (live sweep 2026-10-20, 3.4). The serialized HTML's controls are matched to
+ * these by position (ScrapeWithActionsTool.extractWithSelectors).
+ * @param {Page} page - Playwright page
+ * @returns {Promise<Array<{tag: string, value: string}>>} [] when the page cannot answer
+ */
+async function readFormValues(page) {
+  try {
+    return await page.evaluate(() => Array.from(
+      document.querySelectorAll('input, textarea, select'),
+      (el) => ({ tag: el.localName, value: el.value })
+    ));
+  } catch (_) {
+    return [];
+  }
+}
+
 function actionErrorMessage(action, error) {
   const firstLine = String(error.message).split('\n')[0];
   return action.selector && !firstLine.includes(action.selector)
@@ -363,6 +384,7 @@ export class ActionExecutor extends EventEmitter {
         if (finalState) {
           executionContext.finalHtml = finalState.html;
           executionContext.finalUrl = finalState.url;
+          executionContext.finalFormValues = finalState.formValues;
         }
 
         this.stats.successfulChains++;
@@ -433,6 +455,7 @@ export class ActionExecutor extends EventEmitter {
           ...executionContext,
           page: undefined, // Don't store page in history
           finalHtml: undefined,
+          finalFormValues: undefined, // what was typed, passwords included
           screenshots: undefined,
           screenshotCount: executionContext.screenshots.length,
           capturedStates: undefined,
@@ -459,6 +482,7 @@ export class ActionExecutor extends EventEmitter {
         url,
         finalUrl: executionContext.finalUrl || url,
         finalHtml: executionContext.finalHtml,
+        finalFormValues: executionContext.finalFormValues || [],
         navigationStatus: executionContext.page?.__crawlforgeNavigation?.status ?? null,
         navigations: executionContext.navigations || [],
         consent: executionContext.consent,
@@ -579,6 +603,7 @@ export class ActionExecutor extends EventEmitter {
                 afterActionId: actionResult.id,
                 url: captured.url,
                 html: captured.html,
+                formValues: captured.formValues,
                 timestamp: Date.now()
               });
             }
@@ -780,18 +805,19 @@ export class ActionExecutor extends EventEmitter {
   }
 
   /**
-   * The page's URL and HTML, read natively (no in-page JS execution, so it
-   * works regardless of ALLOW_JAVASCRIPT_EXECUTION). A page that cannot be
-   * read yields null and a warning; a page the gate refuses throws.
+   * The page's URL and HTML, read natively (no caller-supplied JS, so it
+   * works regardless of ALLOW_JAVASCRIPT_EXECUTION), and the live value of
+   * each form control. A page that cannot be read yields null and a warning;
+   * a page the gate refuses throws.
    * @param {Page} page - Playwright page
    * @param {Object} [browserOptions] - Browser options (`respectRobots` override)
    * @param {string} what - names the capture in the warning
-   * @returns {Promise<{ url: string, html: string }|null>}
+   * @returns {Promise<{ url: string, html: string, formValues: Array<{tag: string, value: string}> }|null>}
    */
   async captureState(page, browserOptions, what) {
     return await this.readGated(page, browserOptions, async () => {
       try {
-        return { html: await page.content(), url: page.url() };
+        return { html: await page.content(), url: page.url(), formValues: await readFormValues(page) };
       } catch (captureErr) {
         this.log('warn', 'Failed to capture ' + what + ': ' + captureErr.message);
         return null;
@@ -879,7 +905,7 @@ export class ActionExecutor extends EventEmitter {
       // "Action timeout".
       const backstopMs = timeout + ACTION_TIMEOUT_GRACE_MS;
       let backstopTimer;
-      const executionPromise = this.executeActionByType(page, action, executionContext);
+      const executionPromise = this.executeActionWithRefRetry(page, action, executionContext);
       const timeoutPromise = new Promise((_, reject) => {
         backstopTimer = setTimeout(
           () => reject(new Error(
@@ -1058,6 +1084,42 @@ export class ActionExecutor extends EventEmitter {
     return await settlePage(page, { timeout: remaining });
   }
 
+  /**
+   * Run the action; if its ref names an element the page has since replaced,
+   * re-find the element by the ref's role and name once and run it again.
+   *
+   * Checked up front as well as on the error, because a detached ref does not
+   * always fail fast: Playwright reports "Element is not attached to the DOM"
+   * when the swap lands mid-action, but waits out the whole action timeout for
+   * a ref that was already detached when the action began.
+   * @param {Page} page - Playwright page
+   * @param {Object} action - Action configuration
+   * @param {Object} [executionContext] - Execution context
+   * @returns {Promise<any>} Action result
+   */
+  async executeActionWithRefRetry(page, action, executionContext) {
+    const ref = [action.selector, action.toElement].find(isRef);
+    let reresolved = false;
+    if (ref) {
+      let matches = 1;
+      try {
+        matches = await this.elementLocator(page, ref).count();
+      } catch (_) {
+        // A stale or unknown ref: the action itself reports it.
+      }
+      if (matches === 0) reresolved = await reresolveRef(page, ref);
+    }
+    try {
+      return await this.executeActionByType(page, action, executionContext);
+    } catch (error) {
+      if (!ref || reresolved || !/not attached to the DOM/i.test(error?.message) ||
+          !(await reresolveRef(page, ref))) {
+        throw error;
+      }
+      return await this.executeActionByType(page, action, executionContext);
+    }
+  }
+
   /**
    * Execute action based on its type
    * @param {Page} page - Playwright page
@@ -1442,14 +1504,22 @@ export class ActionExecutor extends EventEmitter {
     const startedAt = Date.now();
     const timeout = this.actionTimeout(action);
 
+    // Both checks run before the page is touched, so a target refused here
+    // leaves the page where it was.
     await assertUrlAllowed(action.url, { resolveDns: true });
     const gateWarnings = await this.assertRobotsAllowed(action.url, executionContext?.browserOptions);
 
-    await this.navigateToUrl(page, action.url, {
-      waitUntil: action.waitUntil,
-      timeout,
-      browserOptions: executionContext?.browserOptions
-    });
+    const previousUrl = page.url();
+    try {
+      await this.navigateToUrl(page, action.url, {
+        waitUntil: action.waitUntil,
+        timeout,
+        browserOptions: executionContext?.browserOptions
+      });
+    } catch (error) {
+      await this.restoreAfterRefusal(page, previousUrl, error, executionContext?.browserOptions);
+      throw error;
+    }
     page.__crawlforgeGateWarnings = gateWarnings;
 
     // Consent handling gets at most what is left of the action's deadline, so
@@ -1477,6 +1547,41 @@ export class ActionExecutor extends EventEmitter {
     };
   }
 
+  /**
+   * Put the page back after a navigation refused partway.
+   *
+   * A target refused up front never leaves the page. A redirect hop to a
+   * blocked address is refused only once the browser asks for it — by the
+   * guarded proxy — and by then the page is gone: blanked by
+   * assertNavigationAllowed, or on the browser's error page when the proxy
+   * refused an https tunnel. The browser cannot undo a committed navigation, so
+   * the page that was left is loaded again (through the same gate). That keeps
+   * a session on its page and its cookies, but not on state that lived only in
+   * the page, and the error says which of the two happened (live sweep
+   * 2026-10-20, 3.3).
+   * @param {Page} page - Playwright page
+   * @param {string} previousUrl - where the page was before the navigation
+   * @param {Error} error - the navigation's failure; its message is extended
+   * @param {Object} [browserOptions] - Browser options (`respectRobots` for the reload's gate)
+   * @returns {Promise<void>}
+   */
+  async restoreAfterRefusal(page, previousUrl, error, browserOptions) {
+    const refused = gateRefusalCode(error) || /ERR_TUNNEL_CONNECTION_FAILED/.test(error?.message);
+    if (!refused || !/^https?:\/\//i.test(previousUrl)) return;
+    // A refused tunnel's error page commits after goto() has already rejected;
+    // let it land first, or it interrupts the reload.
+    if (page.url() === previousUrl) {
+      await page.waitForURL((url) => url.href !== previousUrl, { timeout: 2000 }).catch(() => {});
+    }
+    try {
+      await this.navigateToUrl(page, previousUrl, { browserOptions });
+      error.message += ` The page was returned to ${previousUrl} by loading it again; ` +
+        'state held only in the page (unsaved form input, script state) was lost.';
+    } catch (_) {
+      error.message += ` The page could not be returned to ${previousUrl} and is now ${page.url()}.`;
+    }
+  }
+
   /**
    * What the navigation that just finished reached: the page, or a wall. The
    * same verdict the chain's final document gets, run on this document with

package/src/core/AgentOrchestrator.js

@@ -27,6 +27,7 @@ import { fenceUntrusted } from '../utils/untrustedContent.js';
 import { stealthDocumentVerdict as documentVerdict } from '../utils/stealthVerdict.js';
 import { aBrowserMightPass } from '../tools/scrape/escalation.js';
 import { AGENT_MAX_ESCALATIONS } from '../tools/agent/escalation.js';
+import { siteLabel, corroboratedOrigins, updateLabelledDates } from '../tools/agent/sourceChecks.js';
 
 const DEFAULT_WALL_CLOCK_MS = 120_000;
 const DEFAULT_MAX_STEPS = 5;
@@ -280,13 +281,6 @@ export function isCurrentStateTask(prompt) {
   return CURRENT_STATE_RE.test(prompt || '');
 }
 
-/** The registrable label of a hostname: "python" for www.python.org, "bbc" for bbc.co.uk. */
-function siteLabel(hostname) {
-  const labels = hostname.toLowerCase().split('.');
-  const ccSecondLevel = labels.length >= 3 && labels.at(-1).length === 2 && labels.at(-2).length <= 3;
-  return labels[Math.max(labels.length - (ccSecondLevel ? 3 : 2), 0)];
-}
-
 /**
  * True when a search result is the entity's own site: its domain is the
  * entity's name (python.org for "Python"), or it is a site root whose title
@@ -580,15 +574,22 @@ export class AgentOrchestrator {
     // live front page, which rarely states the fact asked for: "what does
     // Southwest charge for a first checked bag" fetched southwest.com's home,
     // booking and careers pages and answered that the fee is not stated
-    // (R20, 2026-09-07). Keep the entity query first — the domain vote and
-    // live-root promotion depend on it — and add the task's own words as a
-    // second query so the page that states the fact is in the queue too.
+    // (R20, 2026-09-07). Add the task's own words as a second query so the
+    // page that states the fact is in the queue too. A one-word name is also
+    // ambiguous: "Latest stable Rust version and its release date" searched
+    // "Rust" first and got the game and the film (live sweep, 2026-10-20).
+    // Then the task's words, its noun phrase intact, are searched first. The
+    // entity query is searched either way: the domain vote and live-root
+    // promotion below read its results.
+    const entityQuery = searchQueries[0];
+    const oneWordEntity = !/\s/.test(entityQuery.trim());
     if (currentState && searchQueries.length === 1) {
       try {
         const { compactSearchTopic, clampSearchQuery } = await import('./ResearchOrchestrator.js');
         const factQuery = clampSearchQuery(compactSearchTopic(prompt));
-        if (factQuery && factQuery.toLowerCase() !== searchQueries[0].toLowerCase()) {
-          searchQueries.push(factQuery);
+        if (factQuery && factQuery.toLowerCase() !== entityQuery.toLowerCase()) {
+          if (oneWordEntity) searchQueries.unshift(factQuery);
+          else searchQueries.push(factQuery);
         }
       } catch { /* the entity query alone is the pre-R20 behaviour */ }
     }
@@ -656,7 +657,6 @@ export class AgentOrchestrator {
       const official = new Set();
       // Only the bare entity query votes: the fact query added above returns
       // guides and news sites, which must not outvote the official domain.
-      const entityQuery = searchQueries[0];
       for (const s of searchResults) {
         if (s.query !== entityQuery) continue;
         try {
@@ -665,6 +665,22 @@ export class AgentOrchestrator {
           if (isOfficialResult(entityQuery, s.url, s.title)) official.add(origin);
         } catch { /* unparsable result url */ }
       }
+      // For a one-word name, a site the task's own search does not surface is
+      // a namesake, not the subject: rust.facepunch.com, a root titled "Rust",
+      // passed as official for the Rust language and was blamed in the
+      // degraded reason (live sweep, 2026-10-20).
+      const onTopic = oneWordEntity
+        ? corroboratedOrigins(originCounts.keys(), searchResults.filter(s => s.query !== entityQuery).map(s => s.url))
+        : new Set(originCounts.keys());
+      for (const origin of official) if (!onTopic.has(origin)) official.delete(origin);
+      // The namesakes' pages are not fetched either: the game's site is not a
+      // source for the language's release.
+      for (let i = urlQueue.length - 1; i >= 0; i--) {
+        try {
+          if (!namedUrls.has(urlQueue[i]) && originCounts.has(new URL(urlQueue[i]).origin) &&
+            !onTopic.has(new URL(urlQueue[i]).origin)) urlQueue.splice(i, 1);
+        } catch { /* unparsable url stays */ }
+      }
       // The entity's own site wins over any vote count; only when no result
       // is official does the vote decide, and a tied vote decides nothing —
       // with one result per origin it would just promote the top-ranked one.
@@ -672,6 +688,7 @@ export class AgentOrchestrator {
       let bestCount = 0;
       let tied = false;
       for (const [origin, count] of originCounts) {
+        if (!onTopic.has(origin)) continue;
         if (official.size > 0 && !official.has(origin)) continue;
         // Strict > keeps the earliest-seen (top-ranked) origin on ties.
         if (count > bestCount) { bestOrigin = origin; bestCount = count; tied = false; }
@@ -896,6 +913,7 @@ export class AgentOrchestrator {
     let unverified = [];
     let unverifiedLinks = [];
     let forumOnlyVersions = [];
+    let updateDates = [];
     // Distinct from `degraded`: a run can be degraded because its version
     // claims are unsupported and still have had its provenance checked.
     let synthesisFailed = false;
@@ -926,6 +944,7 @@ export class AgentOrchestrator {
         `- Answer ONLY from the provided sources; do not use outside knowledge.\n` +
         `- Read the sources carefully before concluding anything is missing from them.\n` +
         `- Cite the exact URL(s) you used: a source URL, or the URL an item has under "Links on this page".\n` +
+        `- For a date, use the one the sources give for the event itself (e.g. "released", "release date"), never a page's "last updated" or "modified" date, and quote the sentence or table row it came from.\n` +
         `- If, after careful reading, the sources genuinely do not contain the answer, say so explicitly.\n` +
         `- NEVER invent or guess a URL; cite only URLs that appear in the sources above.\n\n` +
         `Provide a clear, concise answer.`;
@@ -933,6 +952,29 @@ export class AgentOrchestrator {
       const { text } = await this._getSamplingClient().complete(synthesisPrompt, { maxTokens: 1024, timeoutMs: remaining() });
       answer = text;
 
+      // A page's "Last updated on 16 October 2026" is not the release date
+      // its table gives (15 Oct 2026), and the provenance check below passes
+      // it: the string is on the page (live sweep, 2026-10-20). A date the
+      // sources state only as an update date gets one corrective rewrite
+      // before the provenance check; what survives it is flagged below.
+      updateDates = updateLabelledDates(answer, evidence, prompt);
+      if (updateDates.length > 0 && remaining() >= REWRITE_MIN_REMAINING_MS) {
+        const dateRetryPrompt =
+          `${synthesisPrompt}\n\nYour previous answer was:\n${fenceUntrusted(answer, 'previous answer')}\n` +
+          `It gives ${updateDates.map(d => `"${d.date}"`).join(', ')}, which the sources state only as the date a page was last updated or modified. ` +
+          `Rewrite the answer using the date the sources give for the event itself (e.g. "released", "release date", the date in the row of the version asked about), ` +
+          `and quote the sentence or table row it came from. If the sources give no such date, say so.`;
+        try {
+          const { text: rewritten } = await this._getSamplingClient().complete(dateRetryPrompt, { maxTokens: 1024, timeoutMs: remaining() });
+          if (rewritten && rewritten.trim()) {
+            answer = rewritten;
+            updateDates = updateLabelledDates(answer, evidence, prompt);
+          }
+        } catch {
+          // Keep the first answer; it is flagged below.
+        }
+      }
+
       // A small model fills gaps from memory: "Swift 5.9.3", "commander 8.6.2
       // published January 18, 2024" — none of it on any fetched page (R17,
       // 2026-09-04). Versions, dates and counts in the answer must appear in
@@ -1012,6 +1054,20 @@ export class AgentOrchestrator {
         }
       }
 
+      // A provenance or version rewrite above may have changed the dates.
+      updateDates = updateLabelledDates(answer, evidence, prompt);
+      if (updateDates.length > 0) {
+        answer +=
+          `\n\nDate warning: ${updateDates.map(d => `"${d.date}"`).join(', ')} ` +
+          `${updateDates.length === 1 ? 'is' : 'are'} the date a source page was last updated, not the date of the event asked about.`;
+        if (!degraded) {
+          degraded = true;
+          degradedReason =
+            `The answer gives a page-update date as the date asked for: ` +
+            updateDates.map(d => `"${d.date}" is labelled as an update date on ${d.url}`).join('; ') + '.';
+        }
+      }
+
       // A promoted live page is only worth its +1000 if the answer came from
       // it. R25 (2026-10-19) answered "not stated" over a promoted Wikipedia
       // Main Page, and the HN question from thehackernews.com, both with
@@ -1069,7 +1125,8 @@ export class AgentOrchestrator {
         checked: !synthesisFailed,
         unverified,
         ...(unverifiedLinks.length > 0 ? { unverified_urls: unverifiedLinks } : {}),
-        ...(forumOnlyVersions.length > 0 ? { unsupported_versions: forumOnlyVersions } : {})
+        ...(forumOnlyVersions.length > 0 ? { unsupported_versions: forumOnlyVersions } : {}),
+        ...(updateDates.length > 0 ? { update_dates: updateDates } : {})
       }
     };
   }

package/src/core/AuthManager.js

@@ -769,7 +769,7 @@ class AuthManager {
           : 'Browser operation. configure/get_stats/cleanup cost 1 credit each.';
         break;
       case 'browser_session':
-        note = `Priced per operation: open 3, read 2, snapshot/act/screenshot/close/list 1. This call bills ${projected}.`;
+        note = `Priced per operation: open 3, read 2, snapshot/act/screenshot/close/list 1. This call's list price is ${projected}.`;
         break;
       case 'serp_rank':
         note = projected === 0
@@ -811,6 +811,12 @@ class AuthManager {
       note += ` redact_pii mode:"model" adds ${REDACT_PII_MODEL_CREDITS} once for the PERSON/LOCATION pass; the charge drops back when no LLM route answered.`;
     }
 
+    // Creator mode bills nothing, so withAuth reports _cost.projected and
+    // actual as 0; the note must not read as a charge next to them (6.10).
+    if (this.isCreatorMode()) {
+      note += ` This install is not billed (creator mode): the figures above are list prices, and _cost.projected and actual are 0.`;
+    }
+
     return { projected, note };
   }
 

package/src/core/ChangeTracker.js

@@ -28,6 +28,7 @@ const ChangeTrackingSchema = z.object({
     trackLinks: z.boolean().default(true),
     ignoreWhitespace: z.boolean().default(true),
     ignoreCase: z.boolean().default(false),
+    includeMarkupChanges: z.boolean().default(false),
     customSelectors: z.array(z.string()).optional(),
     excludeSelectors: z.array(z.string()).optional().default([
       'script', 'style', 'noscript', '.advertisement', '.ad'
@@ -344,10 +345,12 @@ export class ChangeTracker extends EventEmitter {
       const currentAnalysis = await this.analyzeContent(currentContent, baseline.options);
       
       // Perform comprehensive change detection
+      // includeMarkupChanges only shapes the response, so unlike the analysis
+      // options it may be asked for at compare time.
       const changeAnalysis = await this.detectChanges(
         baseline.analysis,
         currentAnalysis,
-        baseline.options
+        { ...baseline.options, includeMarkupChanges: options.includeMarkupChanges ?? baseline.options.includeMarkupChanges }
       );
       
       // Calculate change significance
@@ -410,7 +413,16 @@ export class ChangeTracker extends EventEmitter {
         metrics: changeRecord.metrics,
         recommendations: this.generateChangeRecommendations(changeRecord),
         ...(alerts.length ? { alerts } : {}),
+        // ignoredOptions is the structured form of the warning: each entry
+        // names what the caller asked for and what the baseline applied, so
+        // a compare at granularity "text" against a "section" baseline says
+        // so in a field, not only in prose (6.12).
         ...(ignoredOptions.length ? {
+          ignoredOptions: ignoredOptions.map(option => ({
+            option,
+            requested: options[option],
+            applied: baseline.options[option] ?? null
+          })),
           warnings: [
             `${ignoredOptions.join(', ')} passed to this compare ${ignoredOptions.length === 1 ? 'was' : 'were'} ignored — ` +
             `the baseline's options are applied to both sides of the diff. Recreate the baseline to change them.`
@@ -615,13 +627,23 @@ export class ChangeTracker extends EventEmitter {
       changes.modifiedElements.push(...elementChanges.modified);
     }
     
-    // Detect text changes
+    // Detect text changes. Diffed over the visible text, not the raw HTML,
+    // so a class rename or icon markup no longer reads as a text change
+    // (live sweep 4.1). The raw markup diff is kept apart in markupChanges
+    // and only computed when the caller asks for it.
     if (options.trackText) {
       changes.textChanges = await this.detectTextChanges(
-        baseline.originalContent,
-        current.originalContent,
+        this.visibleText(baseline.originalContent),
+        this.visibleText(current.originalContent),
         options
       );
+      if (options.includeMarkupChanges) {
+        changes.markupChanges = await this.detectTextChanges(
+          baseline.originalContent,
+          current.originalContent,
+          options
+        );
+      }
     }
     
     // Detect monetary value changes. Scored by magnitude rather than by how
@@ -1038,6 +1060,32 @@ export class ChangeTracker extends EventEmitter {
     return changes;
   }
   
+  /**
+   * The text a reader sees: text nodes in document order, one per line, with
+   * script, style, noscript and template content dropped. Input with no
+   * markup (a plain-text content param) is returned unchanged.
+   * @param {string} content - HTML or plain text
+   * @returns {string} - Visible text
+   */
+  visibleText(content = '') {
+    if (!/<[a-z!/]/i.test(content)) return content;
+    const $ = load(content);
+    $('script, style, noscript, template').remove();
+    const parts = [];
+    const walk = nodes => {
+      for (const node of nodes) {
+        if (node.type === 'text') {
+          const text = node.data.trim();
+          if (text) parts.push(text);
+        } else if (node.children) {
+          walk(node.children);
+        }
+      }
+    };
+    walk($.root().contents().toArray());
+    return parts.join('\n');
+  }
+
   async detectTextChanges(baselineContent, currentContent, options = {}) {
     const textChanges = [];