crawlforge-mcp-server 6.23.2 → 6.24.0

package/src/server/withAuth.js

@@ -7,7 +7,11 @@
  *   - resolves toolCost once per call; every tool is metered (no free tier),
  *     so a valid API key is required for every invocation
  *   - try/finally guarantees a single `tool invocation` log line per call
- *   - log payload: { toolName, paramHash, durationMs, outcome, creditCost, creatorMode }
+ *   - log payload: { toolName, paramHash, durationMs, outcome, creditCost, creatorMode,
+ *     internal, runId, client, source, urlKey, cache } — runId is per server process
+ *     (CRAWLFORGE_RUN_ID overrides), client is "<name>/<version>" from initialize,
+ *     source ∈ { 'client' | 'test' | 'harness' | 'eval' }, urlKey is a hash of the
+ *     canonical URL (never the URL itself), cache ∈ { 'hit' | 'miss' | 'bypass' | null }
  *   - outcome ∈ { 'success' | 'error' | 'insufficient_credits' | 'input_required' }
  *   - an `input_required` return (Phase 4.4) is a round trip, not an answer: the
  *     handler did no work, so it is billed NOTHING and reports no usage. The
@@ -15,21 +19,59 @@
  *     once, so a confirmation costs exactly what the call always cost (G4).
  *   - a failed call (an isError result, a JSON body with success:false, or a
  *     throw) is charged NOTHING and reports no usage: failures are free
+ *   - a call whose every page fetch was served from the page cache
+ *     (src/core/pageCache.js) costs at most 1 credit (D1), says so in `cache`
+ *     and `_cost.projection_note`, and logs `cache: 'hit'`
  *   - error results get a "Next step:" hint naming the tool to try next
  *     (src/server/fallbackHints.js) so a failure is not followed by a blind retry
  *   - emits an OTel span via src/observability/tracing.js (no-op if disabled)
  *   - increments Prometheus counters via src/observability/metrics.js (if registry passed)
  */
 
-import { createHash } from 'node:crypto';
+import { createHash, randomUUID } from 'node:crypto';
 import { isInputRequiredResult } from '@modelcontextprotocol/server';
 import { recordToolInvocation } from '../observability/tracing.js';
-import { isInternalRequest, reportedActualCost, requestContext } from './requestContext.js';
+import { isInternalRequest, pageCacheSummary, reportedActualCost, requestContext, servingServer } from './requestContext.js';
 import { appendFallbackHint } from './fallbackHints.js';
 import { failure, normalizeErrorResult } from './errorText.js';
-import { INLINE_THRESHOLD_TOOLS, applyInlineThreshold } from './inlineThreshold.js';
+import { CACHE_BLOCK_RESERVE, COST_BLOCK_RESERVE, INLINE_THRESHOLD_TOOLS, applyInlineThreshold } from './inlineThreshold.js';
 import { REDACTION_TOOLS, redactionSurcharge, runRedactionStage } from './redaction.js';
 import { getResultStore } from '../core/ResultStore.js';
+import { canonicalUrl } from '../utils/canonicalUrl.js';
+
+// One id per server process; over stdio that is one client session.
+const PROCESS_RUN_ID = randomUUID().slice(0, 8);
+const LOG_SOURCES = new Set(['harness', 'eval']);
+// A cache hit costs at most this much (D1). agent is left out: its price pays
+// for searches and model steps the page cache does not cover.
+const CACHE_HIT_CREDITS = 1;
+const CACHE_HIT_EXCLUDED = new Set(['agent']);
+
+/** Who made this call: a test run, a named harness, or a real client. */
+function logSource() {
+  if (process.env.NODE_ENV === 'test') return 'test';
+  const declared = process.env.CRAWLFORGE_LOG_SOURCE;
+  return LOG_SOURCES.has(declared) ? declared : 'client';
+}
+
+/** First 12 hex chars of sha256(canonical URL); null when there is no single URL. */
+export function urlKey(url) {
+  const canonical = canonicalUrl(url);
+  return canonical ? createHash('sha256').update(canonical).digest('hex').slice(0, 12) : null;
+}
+
+/**
+ * "<name>/<version>" from the client's initialize. Over HTTP each session (or
+ * 2026-era request) is served by its own clone, stamped on the request context.
+ */
+function clientLabel(mcpServer) {
+  try {
+    const info = (servingServer() ?? mcpServer)?.server?.getClientVersion?.();
+    return info?.name ? `${info.name}/${info.version ?? ''}` : null;
+  } catch {
+    return null;
+  }
+}
 
 /**
  * Replace an over-threshold JSON text result (and its structuredContent, if
@@ -39,7 +81,9 @@ function shapeLargeResult(toolName, result, params) {
   if (!result || !Array.isArray(result.content) || result.content[0]?.type !== 'text') return;
   let parsed;
   try { parsed = JSON.parse(result.content[0].text); } catch { return; }
-  const { result: shaped, stored } = applyInlineThreshold(toolName, parsed, params, { store: getResultStore() });
+  // A call that went through the page cache also gets a `cache` block after shaping.
+  const reserve = COST_BLOCK_RESERVE + (pageCacheSummary() ? CACHE_BLOCK_RESERVE : 0);
+  const { result: shaped, stored } = applyInlineThreshold(toolName, parsed, params, { store: getResultStore(), reserve });
   if (!stored) return;
   result.content[0].text = JSON.stringify(shaped, null, 2);
   if (result.structuredContent) result.structuredContent = shaped;
@@ -63,6 +107,11 @@ async function redactResult(toolName, result, params, mcpServer) {
   return modelRan;
 }
 
+/** The one line a cache hit adds to the result (plan 1.6). */
+export function cacheHitLine(ageMs) {
+  return `Served from a copy of this page fetched ${Math.round((ageMs ?? 0) / 1000)} s ago in this session (pass maxAge: 0 for a fresh fetch).`;
+}
+
 export function hashParams(params) {
   try {
     return createHash('sha256').update(JSON.stringify(params ?? {})).digest('hex').slice(0, 12);
@@ -101,9 +150,16 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
       // A handler may report a lower spend (setActualCost). The projection is
       // the ceiling the caller saw before the call, so a report can only ever
       // lower the charge, never raise it.
+      // A page served from the cache costs CACHE_HIT_CREDITS in place of the
+      // tool's base price (D1). Only the base price is discounted: surcharges
+      // for work that still ran on the cached page (scrape's query and model
+      // formats) are still charged.
+      let cacheHitPrice = false;
+      const cacheHitDiscount = Math.max(0, authManager.getToolCost(toolName, {}) - CACHE_HIT_CREDITS);
       const billable = () => {
         const reported = reportedActualCost();
-        const spent = reported == null ? toolCost : Math.min(reported, toolCost);
+        let spent = reported == null ? toolCost : Math.min(reported, toolCost);
+        if (cacheHitPrice) spent = Math.max(Math.min(spent, CACHE_HIT_CREDITS), spent - cacheHitDiscount);
         return spent + (redactionModelRan ? redactionCost : 0);
       };
 
@@ -184,6 +240,9 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
           try { shapeLargeResult(toolName, result, params); } catch { /* shaping must never break the request path */ }
         }
 
+        const cache = isErrorResult ? null : pageCacheSummary();
+        cacheHitPrice = cache?.state === 'hit' && !CACHE_HIT_EXCLUDED.has(toolName);
+
         // Failures are free, as the website promises. A pre-flight refusal that
         // sank the call is one of them; a multi-URL tool that skipped one
         // disallowed URL and still returned a result is a success and bills.
@@ -202,7 +261,14 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
             projected: creditCost,
             actual: creatorMode ? 0 : charge,
             remaining_credits: remainingCredits,
-            projection_note: projection.note
+            projection_note: cacheHitPrice && charge > 0
+              ? `${projection.note} Served from this session's page cache: charged ${charge} instead of ${creditCost}.`
+              : projection.note
+          };
+          // 1.6: say where the page came from, and on a hit that a fresh fetch is one parameter away.
+          const cacheMeta = cache && {
+            ...cache,
+            ...(cache.state === 'hit' ? { note: cacheHitLine(cache.ageMs) } : {})
           };
 
           // Inject _cost into the first text content item if it's JSON
@@ -210,14 +276,17 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
             try {
               const parsed = JSON.parse(result.content[0].text);
               parsed._cost = costMeta;
+              if (cacheMeta) parsed.cache = cacheMeta;
               result.content[0].text = JSON.stringify(parsed, null, 2);
             } catch {
-              // Not JSON — skip injection silently
+              // Not JSON: the hit line goes on the end of the text instead.
+              if (cacheMeta?.note) result.content[0].text += `\n${cacheMeta.note}`;
             }
           }
           // ...and into structuredContent, which is all some clients show (R25 0.2).
           if (result?.structuredContent && typeof result.structuredContent === 'object') {
             result.structuredContent._cost = costMeta;
+            if (cacheMeta) result.structuredContent.cache = cacheMeta;
           }
         } catch {
           // Cost injection must never break the request path
@@ -255,7 +324,12 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
           outcome,
           creditCost,
           creatorMode,
-          internal
+          internal,
+          runId: process.env.CRAWLFORGE_RUN_ID || PROCESS_RUN_ID,
+          client: clientLabel(mcpServer),
+          source: logSource(),
+          urlKey: urlKey(params?.url),
+          cache: pageCacheSummary()?.state ?? null
         });
 
         // Prometheus (no-op unless registry was supplied)
@@ -304,6 +378,7 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
         ...(requestContext.getStore() ?? {}),
         preflightRefusal: null,
         actualCost: null,
+        pageCache: null,
         servingRequestId: ctx?.mcpReq?.id
       },
       () => invoke(params, ctx)

package/src/skills/agent-skills/crawlforge-batch-automation/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-batch-automation
 description: "Automates large scraping jobs and browser interactions with CrawlForge's batch_scrape, get_batch_results, scrape_with_actions, and generate_llms_txt tools. Use when the user wants to scrape many URLs at once, batch-scrape a list of pages, collect dozens of product, news, or competitor pages, run browser actions (click, type, scroll, wait) before scraping, log in or fill a form before extracting, or generate an llms.txt for a site. Use sync mode for up to about 25 URLs and async with a webhook for large batches; retrieve paginated output with get_batch_results."
 metadata:
-  version: 6.23.2
+  version: 6.24.0
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-browser-sessions/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-browser-sessions
 description: "Keeps one browser page alive across several tool calls with CrawlForge's browser_session tool. Use when the user needs to log in and then read pages behind that login, work through a multi-step flow (search, filter, paginate, fill a wizard), or look at a page before deciding what to click. The loop is: open a session on a URL, snapshot it to list the interactive elements with stable refs (@e1, @e2), act on those refs, read the content, close. Unlike scrape_with_actions, which is one-shot and closes its browser when the call returns, a session survives between calls, so one login is paid for once and a wrong selector costs one call instead of a whole chain."
 metadata:
-  version: 6.23.2
+  version: 6.24.0
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-change-tracking/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-change-tracking
 description: "Monitors web pages for changes over time with CrawlForge's track_changes tool. Use when the user wants to track changes to a page, watch a URL, monitor competitor pricing, detect when content updates, get notified of regulation or product-availability changes, or diff a page against a saved baseline. Workflow: create a baseline with operation create_baseline, then periodically compare with operation compare to get a change percentage and a diff; supports CSS-selector scoping, webhooks, and scheduled monitoring."
 metadata:
-  version: 6.23.2
+  version: 6.24.0
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-deep-research/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-deep-research
 description: "Runs multi-source web research and autonomous question-answering with CrawlForge's deep_research, agent, search_web, and reddit_search tools. Use when the user wants to research a topic, do a deep dive, compare competitors, gather facts with citations, answer a question from the web, search the web, or get a synthesized report from many sources, or search Reddit posts, comments, and threads. deep_research expands queries, fetches and dedupes sources, then synthesizes; agent autonomously plans searches and navigation from a plain-English prompt with no URLs needed; search_web returns ranked results; reddit_search searches Reddit via community archives (reddit.com blocks scrapers). Caps costs via max_urls and confirms before expensive runs."
 metadata:
-  version: 6.23.2
+  version: 6.24.0
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-getting-started/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-getting-started
 description: "Orientation and tool-selection guide for the CrawlForge MCP server's 31 web tools. Use when the user is getting started with CrawlForge, asks which CrawlForge tool to use, how to set up the API key, how skills or the CLI work, what a tool costs in credits, or when one tool fails and a fallback is needed. Routes requests to the right specialized skill (web scraping, deep research, stealth, structured extraction, change tracking, batch automation, browser sessions), and explains MCP-tools-vs-CLI, the Ollama-first LLM fallback chain, and per-tool credit costs."
 metadata:
-  version: 6.23.2
+  version: 6.24.0
   source: crawlforge-mcp-server
 ---
 
@@ -108,3 +108,6 @@ get_batch_results...) up to 10+ for `deep_research`. Full table:
 [credits](references/credits.md). Prefer the cheapest tool that does the job, and
 cap dynamic tools (`deep_research`, `agent`, `crawl_deep`) with their max
 parameters.
+In a local session, asking again for the same page with different formats, paths or
+selectors reuses this session's copy for 10 minutes at 1 credit; pass `maxAge: 0` to
+force a fresh fetch.

package/src/skills/agent-skills/crawlforge-getting-started/references/credits.md

@@ -81,6 +81,10 @@ metered; there is no free tier. Tools marked "scales" cost more as work grows.
   when the whole interaction can be written down in advance.
 - A result over `max_inline_chars` costs nothing extra; read the stored copy with
   `read_result` (1) instead of fetching again.
+- In a local (stdio) session, asking again for a page read in the last 10 minutes
+  (`fetch_url`, `scrape`, `extract_*`, `scrape_structured`, `extract_structured`,
+  `extract_with_llm`) with different formats, paths or selectors reuses the copy and
+  costs 1 credit; `maxAge: 0` forces a fresh fetch at the full price.
 - On a site that blocks, one `scrape` with `escalate:true` costs at most 7 and
   only 2 when the plain fetch works; `scrape` then `stealth_mode` costs 7 either
   way. Do not set it on a site that reads fine — it raises the projection your

package/src/skills/agent-skills/crawlforge-stealth-browsing/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-stealth-browsing
 description: "Bypasses bot detection with CrawlForge's stealth_mode tool and looks up a country's locale settings with localization. Use when a site returns 403 or 429, CAPTCHAs, 'please enable JavaScript', or empty content, or is protected by Cloudflare, DataDome, or PerimeterX, or when the user needs a page in a specific locale, language, or timezone. stealth_mode runs a stealth browser (engine auto by default: camoufox/Firefox for advanced fingerprinting, playwright/Chromium when pinned for speed) and can screenshot; localization returns a country's Accept-Language, timezone and currency for you to pass to fetch_url or stealth_mode - it applies nothing itself and uses no proxy. Explains when to escalate from a normal scrape to stealth, and why a hard block needs a residential proxy."
 metadata:
-  version: 6.23.2
+  version: 6.24.0
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-structured-extraction/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-structured-extraction
 description: "Extracts structured JSON and analyzes content with CrawlForge's extract_structured, extract_with_llm, extract_embedded_state, scrape_structured, scrape_template, process_document, analyze_content, summarize_content, and list_ollama_models tools. Use when the user wants to extract specific fields, pull data into a JSON schema, extract by natural-language prompt, scrape with CSS selectors, read a page's embedded JavaScript state (__NEXT_DATA__, React Server Components, Nuxt, Apollo, Redux), get product, profile, or repo data from known sites (Amazon, LinkedIn, GitHub, YouTube, Reddit, and more), parse a PDF or DOCX, summarize a page, or analyze sentiment, entities, or keywords. Defaults to local Ollama for LLM extraction; OpenAI and Anthropic optional."
 metadata:
-  version: 6.23.2
+  version: 6.24.0
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-web-scraping/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-web-scraping
 description: "Scrapes web pages and returns clean Markdown, HTML, plain text, links, or metadata using CrawlForge's scrape, fetch_url, extract_content, extract_text, extract_links, extract_metadata, map_site, and crawl_deep tools. Use when the user wants to scrape a URL, fetch a page, get the markdown or text of a website, extract links or metadata, read an article, map a site's URLs, generate a sitemap, or crawl a whole website or documentation site. Prefer the unified scrape tool to get multiple formats in one call; use map_site to discover URLs and crawl_deep to walk an entire site."
 metadata:
-  version: 6.23.2
+  version: 6.24.0
   source: crawlforge-mcp-server
 ---
 

package/src/tools/basic/_fetch.js

@@ -10,6 +10,7 @@ import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
 import { preflightFetch } from '../../utils/robotsGate.js';
 import { fetchResigned } from '../../utils/resignedFetch.js';
 import { withFetchCause } from '../../utils/fetchError.js';
+import { pageFromResponse, throughPageCache } from '../../core/pageCache.js';
 
 /**
  * Fetch a URL with a configurable timeout and body-size cap.
@@ -19,12 +20,39 @@ import { withFetchCause } from '../../utils/fetchError.js';
  * configurable cap from config.fetch.maxBodySize (env MAX_FETCH_BODY_SIZE,
  * default 25 MB).
  *
+ * The page goes through the process page cache (src/core/pageCache.js):
+ * `maxAge` (ms, default 10 min, 0 = fetch fresh) and `noCache` steer it. A
+ * copy served from it is a Response-like object with the same fields.
+ *
  * @param {string} url
  * @param {{ timeout?: number, headers?: Record<string,string>, userAgent?: string,
- *   respectRobots?: boolean, tool?: string, apiKey?: string }} [options]
+ *   respectRobots?: boolean, tool?: string, apiKey?: string, maxAge?: number,
+ *   noCache?: boolean }} [options]
  * @returns {Promise<Response & { _body: string, _warnings: string[] }>}
  */
-export async function fetchWithTimeout(url, options = {}) {
+export function fetchWithTimeout(url, options = {}) {
+  return throughPageCache(url, options, 'plain', () => fetchFresh(url, options), responseFromPage);
+}
+
+/** A stored page as the Response-like object fetchFresh returns. */
+function responseFromPage(page, warnings) {
+  return {
+    ok: page.status >= 200 && page.status <= 299,
+    status: page.status,
+    statusText: page.statusText,
+    url: page.url,
+    redirected: page.redirected,
+    headers: new Headers(page.headers),
+    text: () => Promise.resolve(page.body),
+    json: () => Promise.resolve(JSON.parse(page.body)),
+    _body: page.body,
+    _responseTime: page.responseTime,
+    _warnings: warnings
+  };
+}
+
+/** The network fetch; resolves to { result: the Response, page: what the cache may keep }. */
+async function fetchFresh(url, options) {
   const { timeout = 10000, headers = {}, userAgent, respectRobots, tool, apiKey } = options;
   const maxBodySize = config.fetch.maxBodySize;
 
@@ -91,13 +119,16 @@ export async function fetchWithTimeout(url, options = {}) {
     }
 
     // Attach the pre-read text so callers can call .text() on the result.
-    return Object.assign(response, {
+    const responseTime = Date.now() - startedAt;
+    const page = { ...pageFromResponse(response, bodyText), responseTime };
+    const result = Object.assign(response, {
       text: () => Promise.resolve(bodyText),
       json: () => Promise.resolve(JSON.parse(bodyText)),
       _body: bodyText,
-      _responseTime: Date.now() - startedAt,
+      _responseTime: responseTime,
       _warnings: gate.warnings
     });
+    return { result, page };
   } finally {
     clearTimeout(timeoutId);
   }

package/src/tools/basic/extractLinks.js

@@ -11,16 +11,17 @@ import { fetchLadder, ladderErrorResult, isJsonType } from '../../utils/fetchLad
 
 /**
  * @param {{ url: string, filter_external?: boolean, base_url?: string,
- *   user_agent?: string, respect_robots?: boolean,
+ *   user_agent?: string, respect_robots?: boolean, maxAge?: number,
  *   escalate?: boolean, escalate_engine?: string }} params
  * @param {Function} [escalateFetch] the stealth escalation stage
  */
-async function extractLinks({ url, filter_external, base_url, user_agent, respect_robots, escalate, escalate_engine }, escalateFetch) {
+async function extractLinks({ url, filter_external, base_url, user_agent, respect_robots, maxAge, escalate, escalate_engine }, escalateFetch) {
   try {
     const ladder = await fetchLadder(url, {
       tool: 'extract_links',
       userAgent: user_agent,
       respectRobots: respect_robots,
+      maxAge,
       escalate: escalate === true,
       escalateEngine: escalate_engine,
       escalateFetch

package/src/tools/basic/extractMetadata.js

@@ -64,13 +64,14 @@ function redirectFields(requested, final) {
 
 /**
  * @param {{ url: string, user_agent?: string, respect_robots?: boolean,
- *   json_ld_types?: string[] }} params
+ *   json_ld_types?: string[], maxAge?: number }} params
  */
-export async function extractMetadataHandler({ url, user_agent, respect_robots, json_ld_types }) {
+export async function extractMetadataHandler({ url, user_agent, respect_robots, json_ld_types, maxAge }) {
   try {
     const response = await fetchWithTimeout(url, {
       userAgent: user_agent,
       respectRobots: respect_robots,
+      maxAge,
       tool: 'extract_metadata'
     });
     if (!response.ok) {

package/src/tools/basic/extractText.js

@@ -105,16 +105,17 @@ function isCut(text, maxLength) {
 /**
  * @param {{ url: string, remove_scripts?: boolean, remove_styles?: boolean,
  *   output_format?: "text"|"markdown", selector?: string, max_length?: number,
- *   user_agent?: string, respect_robots?: boolean,
+ *   user_agent?: string, respect_robots?: boolean, maxAge?: number,
  *   escalate?: boolean, escalate_engine?: string }} params
  * @param {Function} [escalateFetch] the stealth escalation stage
  */
-async function extractText({ url, remove_scripts, remove_styles, output_format, selector, max_length, user_agent, respect_robots, escalate, escalate_engine }, escalateFetch) {
+async function extractText({ url, remove_scripts, remove_styles, output_format, selector, max_length, user_agent, respect_robots, maxAge, escalate, escalate_engine }, escalateFetch) {
   try {
     const ladder = await fetchLadder(url, {
       tool: 'extract_text',
       userAgent: user_agent,
       respectRobots: respect_robots,
+      maxAge,
       escalate: escalate === true,
       escalateEngine: escalate_engine,
       escalateFetch

package/src/tools/basic/fetchUrl.js

@@ -7,15 +7,16 @@ import { fetchWithTimeout } from './_fetch.js';
 
 /**
  * @param {{ url: string, headers?: Record<string,string>, timeout?: number,
- *   user_agent?: string, respect_robots?: boolean }} params
+ *   user_agent?: string, respect_robots?: boolean, maxAge?: number }} params
  */
-export async function fetchUrlHandler({ url, headers, timeout, user_agent, respect_robots }) {
+export async function fetchUrlHandler({ url, headers, timeout, user_agent, respect_robots, maxAge }) {
   try {
     const response = await fetchWithTimeout(url, {
       timeout: timeout || 10000,
       headers: headers || {},
       userAgent: user_agent,
       respectRobots: respect_robots,
+      maxAge,
       tool: 'fetch_url'
     });
 

package/src/tools/basic/scrapeStructured.js

@@ -139,13 +139,14 @@ function extractRows($, { row_selector, selectors, max_results, baseUrl }) {
 
 /**
  * @param {{ url: string, selectors: Record<string, string|{ selector: string, multiple?: boolean }>, row_selector?: string,
- *   max_results?: number, user_agent?: string, respect_robots?: boolean }} params
+ *   max_results?: number, user_agent?: string, respect_robots?: boolean, maxAge?: number }} params
  */
-export async function scrapeStructuredHandler({ url, selectors, row_selector, max_results, user_agent, respect_robots }) {
+export async function scrapeStructuredHandler({ url, selectors, row_selector, max_results, user_agent, respect_robots, maxAge }) {
   try {
     const response = await fetchWithTimeout(url, {
       userAgent: user_agent,
       respectRobots: respect_robots,
+      maxAge,
       tool: 'scrape_structured'
     });
     if (!response.ok) {

package/src/tools/extract/_fetchAndParse.js

@@ -16,6 +16,7 @@ import { safeFetch } from '../../utils/ssrfGuard.js';
 import { config } from '../../constants/config.js';
 import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
 import { preflightFetch } from '../../utils/robotsGate.js';
+import { pageFromResponse, pageHeader, throughPageCache } from '../../core/pageCache.js';
 
 const DEFAULT_TIMEOUT_MS = 15000;
 
@@ -108,16 +109,60 @@ export { flattenText as flattenBodyText };
  * @param {number}   [options.timeoutMs]
  * @param {string[]} [options.stripTags]   — additional tags to strip (default: script, style, noscript, iframe, svg)
  * @param {boolean}  [options.errorDocuments] — return a non-2xx HTML/text body with its status instead of throwing
+ * @param {number}   [options.maxAge]   — page cache: reuse a copy at most this many ms old (default 10 min; 0 = fetch fresh)
+ * @param {boolean}  [options.noCache]  — page cache: neither read nor store
  * @returns {Promise<{ html: string, $: import('cheerio').CheerioAPI, textContent: string, finalUrl: string, warnings: string[], status: number }>}
  */
 export async function fetchAndParse(url, options = {}) {
+  const { stripTags = ['script', 'style', 'noscript', 'iframe', 'svg'] } = options;
+
+  // The page itself may come from the process page cache (src/core/pageCache.js);
+  // the parse below is always this call's own, so no two calls share a `$`.
+  const page = await throughPageCache(
+    url,
+    options,
+    'html',
+    async () => {
+      const fetched = await fetchPage(url, options);
+      return { result: fetched, page: fetched };
+    },
+    (entry, warnings) => ({ ...entry, warnings })
+  );
+
+  const html = page.body;
+  const { warnings, status } = page;
+  const finalUrl = page.url;
+
+  // text/plain and application/json aren't markup: running them through the
+  // HTML parser risks misinterpreting substrings (e.g. a "<script>" value
+  // inside a JSON string) as real tags and stripping/mangling content.
+  if (classifyContentType(pageHeader(page, 'content-type')) === 'text') {
+    return { html, $: load(''), textContent: html.trim(), finalUrl, warnings, status };
+  }
+
+  const $ = load(html);
+
+  if (stripTags.length > 0) {
+    $(stripTags.join(', ')).remove();
+  }
+
+  const textContent = flattenText($);
+
+  return { html, $, textContent, finalUrl, warnings, status };
+}
+
+/**
+ * The network fetch behind fetchAndParse: the gate, the request, the status
+ * and content-type checks and the capped body read.
+ * @returns {Promise<object>} the page as pageFromResponse gives it, plus `warnings`
+ */
+async function fetchPage(url, options) {
   const {
     userAgent,
     respectRobots,
     tool,
     apiKey,
     timeoutMs = DEFAULT_TIMEOUT_MS,
-    stripTags = ['script', 'style', 'noscript', 'iframe', 'svg'],
     errorDocuments = false
   } = options;
 
@@ -157,21 +202,5 @@ export async function fetchAndParse(url, options = {}) {
   }
 
   const html = await readTextWithSizeCap(response);
-
-  // text/plain and application/json aren't markup: running them through the
-  // HTML parser risks misinterpreting substrings (e.g. a "<script>" value
-  // inside a JSON string) as real tags and stripping/mangling content.
-  if (classification === 'text') {
-    return { html, $: load(''), textContent: html.trim(), finalUrl: response.url, warnings, status: response.status };
-  }
-
-  const $ = load(html);
-
-  if (stripTags.length > 0) {
-    $(stripTags.join(', ')).remove();
-  }
-
-  const textContent = flattenText($);
-
-  return { html, $, textContent, finalUrl: response.url, warnings, status: response.status };
+  return { ...pageFromResponse(response, html), warnings };
 }