crawlforge-mcp-server 6.23.1 → 6.24.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "6.23.1",
+  "version": "6.24.0",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 31 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",

package/server.js

@@ -60,6 +60,7 @@ import { scrapeStructuredHandler } from "./src/tools/basic/scrapeStructured.js";
 import { createExtractEmbeddedStateHandler, EMBEDDED_STATE_INPUT_SHAPE } from "./src/tools/extract/extractEmbeddedState.js";
 import { READ_RESULT_INPUT_SHAPE, readResultHandler } from "./src/tools/result/readResult.js"; // Phase 2
 import { MAX_INLINE_CHARS_PARAM } from "./src/server/inlineThreshold.js"; // Phase 2
+import { MAX_AGE_PARAM, PAGE_CACHE_SENTENCE } from "./src/utils/maxAgeParam.js"; // repeat-call reduction 1.5
 import { REDACT_PII_PARAM } from "./src/server/redaction.js"; // Phase 5 (5.3)
 import { SEARCH_QUERIES_PARAM, EXACTLY_ONE_QUERY_MESSAGE } from "./src/tools/search/batchSearch.js"; // Phase 5 (5.1)
 import { markPreflightRefusal, preflightRefusal, internalOwnerToken } from "./src/server/requestContext.js";
@@ -113,7 +114,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "6.23.1",
+  version: "6.24.0",
   description: "Production-ready MCP server with 31 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, stateful browser sessions with element refs, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -427,20 +428,21 @@ const refusedByGate = (result) => result?.success === false && preflightRefusal(
 
 // Tool: fetch_url
 registerToolIfEnabled("fetch_url", {
-  description: "Use this for a raw HTTP body - JSON, XML, plain text, an API response - or for the status code, headers and response time. Returns the body unprocessed. Not for HTML you intend to read: scrape returns markdown from one fetch, so fetch_url followed by extract_* is a double fetch. Not for JS-rendered or bot-protected pages (scrape, then stealth_mode). Supports custom headers (e.g. auth tokens) and a timeout; the result reports redirected and final_url. Cost: 1 credit. Example: fetch_url({url: \"https://api.example.com/v1/items\", timeout: 15000})",
+  description: "Use this for a raw HTTP body - JSON, XML, plain text, an API response - or for the status code, headers and response time. Returns the body unprocessed. Not for HTML you intend to read: scrape returns markdown from one fetch, so fetch_url followed by extract_* is a double fetch. Not for JS-rendered or bot-protected pages (scrape, then stealth_mode). Supports custom headers (e.g. auth tokens) and a timeout; the result reports redirected and final_url." + PAGE_CACHE_SENTENCE + " Cost: 1 credit. Example: fetch_url({url: \"https://api.example.com/v1/items\", timeout: 15000})",
   annotations: { title: "Fetch URL", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The URL to fetch content from"),
     headers: z.record(z.string()).optional().describe("Custom HTTP headers to include in the request"),
     timeout: z.number().min(1000).max(30000).optional().default(10000).describe("Request timeout in milliseconds (1000-30000)"),
     ...COMPLIANCE_PARAMS,
+    ...MAX_AGE_PARAM,
     ...MAX_INLINE_CHARS_PARAM
   }
 }, withAuth("fetch_url", fetchUrlHandler));
 
 // Tool: extract_text
 registerToolIfEnabled("extract_text", {
-  description: "Use this for a page's plain text or markdown with tags, scripts and styles removed - the cheapest read of a static HTML page. Use output_format:\"markdown\" for RAG; selector reads only the matched elements, max_length caps the returned text. Not for article pages (extract_content strips nav and boilerplate), JS-rendered pages (scrape), or when you also want links or metadata (scrape with several formats, one fetch). A PDF or other binary is refused (process_document reads documents); an empty client-rendered shell succeeds with rendered:false and a warning. Set escalate:true when the site is known to block: the plain fetch still runs first, and only if it is walled does the stealth browser re-read the page - projected at 6, charged 1 when the plain fetch worked. Cost: 1 credit. Example: extract_text({url: \"https://example.com/article\", output_format:\"markdown\"})",
+  description: "Use this for a page's plain text or markdown with tags, scripts and styles removed - the cheapest read of a static HTML page. Use output_format:\"markdown\" for RAG; selector reads only the matched elements, max_length caps the returned text. Not for article pages (extract_content strips nav and boilerplate), JS-rendered pages (scrape), or when you also want links or metadata (scrape with several formats, one fetch). A PDF or other binary is refused (process_document reads documents); an empty client-rendered shell succeeds with rendered:false and a warning. Set escalate:true when the site is known to block: the plain fetch still runs first, and only if it is walled does the stealth browser re-read the page - projected at 6, charged 1 when the plain fetch worked." + PAGE_CACHE_SENTENCE + " Cost: 1 credit. Example: extract_text({url: \"https://example.com/article\", output_format:\"markdown\"})",
   annotations: { title: "Extract Text", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The URL to extract text from"),
@@ -451,6 +453,7 @@ registerToolIfEnabled("extract_text", {
     max_length: z.number().int().min(1).max(1000000).optional().describe("Maximum characters of text or markdown to return; a longer result is cut, ends with \"...\" and carries truncated:true"),
     ...EXTRACT_ESCALATION_SHAPE,
     ...COMPLIANCE_PARAMS,
+    ...MAX_AGE_PARAM,
     ...REDACT_PII_PARAM
   }
 }, withAuth("extract_text", createExtractTextHandler({
@@ -459,7 +462,7 @@ registerToolIfEnabled("extract_text", {
 
 // Tool: extract_links
 registerToolIfEnabled("extract_links", {
-  description: "Use this to list the hyperlinks on one page - a crawl seed list, a broken-link audit, related resources. filter_external:true drops internal links; mailto:, tel: and javascript: links are type \"other\". Not for a whole site (map_site), and not alongside a scrape of the same URL: scrape formats:[\"markdown\",\"links\"] returns both in one fetch. A PDF or other binary is refused (process_document reads documents); an empty client-rendered shell succeeds with rendered:false and a warning. Set escalate:true when the site is known to block: the plain fetch still runs first, and only if it is walled does the stealth browser re-read the page - projected at 6, charged 1 when the plain fetch worked. Cost: 1 credit. Example: extract_links({url: \"https://example.com\", filter_external: true})",
+  description: "Use this to list the hyperlinks on one page - a crawl seed list, a broken-link audit, related resources. filter_external:true drops internal links; mailto:, tel: and javascript: links are type \"other\". Not for a whole site (map_site), and not alongside a scrape of the same URL: scrape formats:[\"markdown\",\"links\"] returns both in one fetch. A PDF or other binary is refused (process_document reads documents); an empty client-rendered shell succeeds with rendered:false and a warning. Set escalate:true when the site is known to block: the plain fetch still runs first, and only if it is walled does the stealth browser re-read the page - projected at 6, charged 1 when the plain fetch worked." + PAGE_CACHE_SENTENCE + " Cost: 1 credit. Example: extract_links({url: \"https://example.com\", filter_external: true})",
   annotations: { title: "Extract Links", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The URL to extract links from"),
@@ -467,6 +470,7 @@ registerToolIfEnabled("extract_links", {
     base_url: z.string().url().optional().describe("Base URL for resolving relative links"),
     ...EXTRACT_ESCALATION_SHAPE,
     ...COMPLIANCE_PARAMS,
+    ...MAX_AGE_PARAM,
     ...MAX_INLINE_CHARS_PARAM
   }
 }, withAuth("extract_links", createExtractLinksHandler({
@@ -475,23 +479,25 @@ registerToolIfEnabled("extract_links", {
 
 // Tool: extract_metadata
 registerToolIfEnabled("extract_metadata", {
-  description: "Use this for a page's SEO metadata only: title, meta description, Open Graph tags, canonical URL, schema.org data. `url` is the final URL; redirected:true (with requested_url) says a redirect moved the request to another page. Not alongside a scrape of the same URL: scrape formats:[\"markdown\",\"metadata\"] returns both in one fetch. Cost: 1 credit. Example: extract_metadata({url: \"https://example.com\"})",
+  description: "Use this for a page's SEO metadata only: title, meta description, Open Graph tags, canonical URL, schema.org data. `url` is the final URL; redirected:true (with requested_url) says a redirect moved the request to another page. Not alongside a scrape of the same URL: scrape formats:[\"markdown\",\"metadata\"] returns both in one fetch." + PAGE_CACHE_SENTENCE + " Cost: 1 credit. Example: extract_metadata({url: \"https://example.com\"})",
   annotations: { title: "Extract Metadata", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The URL to extract metadata from"),
     json_ld_types: z.array(z.string()).optional().describe("Filter the returned JSON-LD to nodes of these schema.org types, e.g. [\"Product\",\"Offer\"]. Subtypes match their parent: \"Event\" returns MusicEvent, \"Offer\" returns AggregateOffer, \"ItemList\" returns BreadcrumbList. Nodes are found at any depth, including inside @graph and nested inside a parent node; a match nested inside another returned node comes back inside it, not again on its own. When set, json_ld carries only the matching nodes instead of the raw dump, and json_ld_type_counts reports how many matched per requested type, nested ones included. Documented types: ItemList, Product, Offer, Event, JobPosting, RealEstateListing — any other schema.org type is matched exactly."),
     ...COMPLIANCE_PARAMS,
+    ...MAX_AGE_PARAM,
     ...MAX_INLINE_CHARS_PARAM
   }
 }, withAuth("extract_metadata", extractMetadataHandler));
 
 // Tool: extract_embedded_state
 registerToolIfEnabled("extract_embedded_state", {
-  description: "Use this when a page's data lives in its embedded JavaScript state rather than its rendered HTML - Next.js (__NEXT_DATA__, and React Server Component payloads with references between rows resolved plus a data_rows index of the rows carrying data), Nuxt (Nuxt 3's __NUXT_DATA__ decoded), SvelteKit, Apollo, Redux (__INITIAL_STATE__, __PRELOADED_STATE__), ytInitialData and ytInitialPlayerResponse, Inertia, Shopify, JSON data-* attributes and <script type=\"application/json\"> blocks. One fetch, exact values, no LLM in the extraction path, so nothing can be fabricated. Payloads are routinely over a megabyte - pass `path` to return one subtree, keys_only:true to see the first two levels of keys before choosing one, or find:\"<key>\" to get every path where a field of that name lives, each ready to pass back as `path`. A result over max_inline_chars comes back as a preview (whole lines of the JSON), data_keys and a result_handle; read the rest with read_result json_path, e.g. \"data.next_data.props\". Set escalate:true when the site is known to block: the plain fetch still runs first, and only if it is walled does the stealth browser re-read the page, run the same parser and also read the globals off window (window_state: __NEXT_DATA__, __NUXT__, ytInitialData and others, read after JavaScript ran) - projected at 7, charged 2 when the plain fetch worked. Not for the rendered text of a page (scrape) or for sites built without a framework payload. Cost: 2 credits. Example: extract_embedded_state({url: \"https://www.ticketmaster.com/discover/concerts\", path: \"next_data.props.pageProps\"})",
+  description: "Use this when a page's data lives in its embedded JavaScript state rather than its rendered HTML - Next.js (__NEXT_DATA__, and React Server Component payloads with references between rows resolved plus a data_rows index of the rows carrying data), Nuxt (Nuxt 3's __NUXT_DATA__ decoded), SvelteKit, Apollo, Redux (__INITIAL_STATE__, __PRELOADED_STATE__), ytInitialData and ytInitialPlayerResponse, Inertia, Shopify, JSON data-* attributes and <script type=\"application/json\"> blocks. One fetch, exact values, no LLM in the extraction path, so nothing can be fabricated. Payloads are routinely over a megabyte - pass `path` to return one subtree, keys_only:true to see the first two levels of keys before choosing one, or find:\"<key>\" to get every path where a field of that name lives, each ready to pass back as `path`. A result over max_inline_chars comes back as a preview (whole lines of the JSON), data_keys and a result_handle; read the rest with read_result json_path, e.g. \"data.next_data.props\". Set escalate:true when the site is known to block: the plain fetch still runs first, and only if it is walled does the stealth browser re-read the page, run the same parser and also read the globals off window (window_state: __NEXT_DATA__, __NUXT__, ytInitialData and others, read after JavaScript ran) - projected at 7, charged 2 when the plain fetch worked. Not for the rendered text of a page (scrape) or for sites built without a framework payload." + PAGE_CACHE_SENTENCE + " Cost: 2 credits. Example: extract_embedded_state({url: \"https://www.ticketmaster.com/discover/concerts\", path: \"next_data.props.pageProps\"})",
   annotations: { title: "Extract Embedded State", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     ...EMBEDDED_STATE_INPUT_SHAPE,
     ...COMPLIANCE_PARAMS,
+    ...MAX_AGE_PARAM,
     ...MAX_INLINE_CHARS_PARAM
   }
 }, withAuth("extract_embedded_state", createExtractEmbeddedStateHandler({
@@ -500,7 +506,7 @@ registerToolIfEnabled("extract_embedded_state", {
 
 // Tool: scrape_structured
 registerToolIfEnabled("scrape_structured", {
-  description: "Use this when you know the exact CSS selectors for the data you want - e.g. a pricing table or product list with consistent markup. More reliable than LLM extraction for well-structured pages. By default each selector is matched independently across the whole page, so the returned arrays are NOT row-aligned: data.price[0] need not belong to the same row as data.name[0]. Pass row_selector to get aligned records instead - one object per row, null for a field the row lacks. Not for pages whose markup varies or where you cannot name the selectors (extract_structured, LLM-driven). Cost: 2 credits. Example: scrape_structured({url: \"https://shop.com/products\", row_selector: \".product-card\", selectors: {price: \".price\", name: \".product-title\"}})",
+  description: "Use this when you know the exact CSS selectors for the data you want - e.g. a pricing table or product list with consistent markup. More reliable than LLM extraction for well-structured pages. By default each selector is matched independently across the whole page, so the returned arrays are NOT row-aligned: data.price[0] need not belong to the same row as data.name[0]. Pass row_selector to get aligned records instead - one object per row, null for a field the row lacks. Not for pages whose markup varies or where you cannot name the selectors (extract_structured, LLM-driven)." + PAGE_CACHE_SENTENCE + " Cost: 2 credits. Example: scrape_structured({url: \"https://shop.com/products\", row_selector: \".product-card\", selectors: {price: \".price\", name: \".product-title\"}})",
   annotations: { title: "Scrape Structured Data", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The URL to scrape"),
@@ -510,7 +516,8 @@ registerToolIfEnabled("scrape_structured", {
     ])).describe("CSS selectors mapping field names to selectors. Append @attr to extract an attribute instead of text (e.g. \"a.link@href\", \"img@src\"); @href and @src values are resolved to absolute URLs. A field may also be {selector, multiple: true} to return every match as an array - inside a row_selector row a field otherwise keeps only the first match"),
     row_selector: z.string().optional().describe("CSS selector for the repeating row/container element. When set, each field in selectors is matched inside each row and data is an array of row-aligned records ({field: value|null}) instead of parallel arrays"),
     max_results: z.number().int().min(1).optional().describe("Maximum number of matches to return per field when a selector matches multiple elements, or the maximum number of rows when row_selector is set"),
-    ...COMPLIANCE_PARAMS
+    ...COMPLIANCE_PARAMS,
+    ...MAX_AGE_PARAM
   }
 }, withAuth("scrape_structured", scrapeStructuredHandler));
 
@@ -823,7 +830,7 @@ registerToolIfEnabled("analyze_content", {
 
 // Tool: extract_structured
 registerToolIfEnabled("extract_structured", {
-  description: "Use this to get a specific data shape from a page using a JSON schema when you can describe the fields but not their selectors - product details, job listings, event data. Uses an LLM by default; falls back to CSS selectors when no LLM is configured. Not for pages with stable markup whose selectors you know (scrape_structured, no LLM, cheaper). Cost: 3 credits. Example: extract_structured({url: \"https://jobs.example.com/post/123\", schema: {properties: {title: {type:\"string\"}, salary: {type:\"string\"}}, required:[\"title\"]}})",
+  description: "Use this to get a specific data shape from a page using a JSON schema when you can describe the fields but not their selectors - product details, job listings, event data. Uses an LLM by default; falls back to CSS selectors when no LLM is configured. Not for pages with stable markup whose selectors you know (scrape_structured, no LLM, cheaper)." + PAGE_CACHE_SENTENCE + " Cost: 3 credits. Example: extract_structured({url: \"https://jobs.example.com/post/123\", schema: {properties: {title: {type:\"string\"}, salary: {type:\"string\"}}, required:[\"title\"]}})",
   annotations: { title: "Extract Structured Data", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The URL to extract structured data from"),
@@ -840,6 +847,7 @@ registerToolIfEnabled("extract_structured", {
     fallbackToSelectors: z.boolean().optional().default(true).describe("Fall back to CSS selector extraction if LLM is unavailable"),
     selectorHints: z.record(z.string()).optional().describe("CSS selector hints to guide extraction"),
     ...COMPLIANCE_PARAMS,
+    ...MAX_AGE_PARAM,
     ...VERIFY_NUMBERS_PARAM
   },
   outputSchema: OUTPUT_SCHEMAS.extract_structured
@@ -858,7 +866,7 @@ registerToolIfEnabled("extract_structured", {
 
 // Tool: extract_with_llm
 registerToolIfEnabled("extract_with_llm", {
-  description: "Extract data from a URL or text using a natural-language prompt. Defaults to a local Ollama model (http://localhost:11434, no API key required) and picks an installed model itself; pass `model` to choose one, or provider: \"openai\" or \"anthropic\" with the matching API key to use a cloud model instead. Not for known selectors (scrape_structured) or a schema-shaped result (extract_structured). Call list_ollama_models only when a model name is rejected. Cost: 3 credits plus your LLM provider's own charge.",
+  description: "Extract data from a URL or text using a natural-language prompt. Defaults to a local Ollama model (http://localhost:11434, no API key required) and picks an installed model itself; pass `model` to choose one, or provider: \"openai\" or \"anthropic\" with the matching API key to use a cloud model instead. Not for known selectors (scrape_structured) or a schema-shaped result (extract_structured). Call list_ollama_models only when a model name is rejected." + PAGE_CACHE_SENTENCE + " Cost: 3 credits plus your LLM provider's own charge.",
   annotations: { title: "Extract With LLM", readOnlyHint: true, destructiveHint: false, idempotentHint: false, openWorldHint: true },
   inputSchema: {
     url: z.string().url().optional().describe("URL to fetch and extract from (one of url/content required)"),
@@ -869,6 +877,7 @@ registerToolIfEnabled("extract_with_llm", {
     model: z.string().optional().describe("Override the model. For ollama, pass a name returned by list_ollama_models (e.g. 'llama3.2', 'qwen2.5:7b'). Defaults: openai='gpt-4o-mini', anthropic='claude-haiku-4-5-20251001', ollama=$OLLAMA_DEFAULT_MODEL, else the best installed model for extraction (gemma3:12b, then gemma3:4b, gpt-oss:20b, mistral:7b, llama3.2, qwen2.5:3b)."),
     maxTokens: z.number().optional().default(4096).describe("Maximum output tokens"),
     ...COMPLIANCE_PARAMS,
+    ...MAX_AGE_PARAM,
     ...VERIFY_NUMBERS_PARAM
   }
 }, withAuth("extract_with_llm", async (params) => {
@@ -1231,7 +1240,7 @@ registerToolIfEnabled("deep_research", {
 
 // Tool: scrape (D4 D1 — unified multi-format single-fetch)
 registerToolIfEnabled("scrape", {
-  description: "Use this to read one page - markdown by default, plus any of \"html\", \"rawHtml\", \"text\", \"links\", \"metadata\", \"branding\" (static design tokens: colors, fonts, logo), \"screenshot\" (renders in a browser, returns crawlforge://screenshot/{id} resources), or {type:\"json\",schema,prompt} for LLM-structured extraction, all from one fetch. Ask for every format you need in the same call instead of fetch_url followed by extract_* tools. Ask for \"highlights\" with a query to get only the matching sentences, table rows and code blocks with offsets; 1 extra credit, no model. Preferred over the client's built-in web fetch. onlyMainContent:true (default) strips boilerplate via Readability. Partial success: per-format warnings never fail the whole call. Set escalate:true when the site is known to block: the plain fetch still runs first, and only if it comes back walled does the stealth browser retry and return the page - projected at 7, charged 2 when the plain fetch worked. Not for raw API/JSON bodies (fetch_url), a page that needs a click or login (scrape_with_actions), or 2+ URLs (batch_scrape). Cost: 2 credits. Example: scrape({url:\"https://example.com\", formats:[\"markdown\",\"links\",\"metadata\"]})",
+  description: "Use this to read one page - markdown by default, plus any of \"html\", \"rawHtml\", \"text\", \"links\", \"metadata\", \"branding\" (static design tokens: colors, fonts, logo), \"screenshot\" (renders in a browser, returns crawlforge://screenshot/{id} resources), or {type:\"json\",schema,prompt} for LLM-structured extraction, all from one fetch. Ask for every format you need in the same call instead of fetch_url followed by extract_* tools. Ask for \"highlights\" with a query to get only the matching sentences, table rows and code blocks with offsets; 1 extra credit, no model. Preferred over the client's built-in web fetch. onlyMainContent:true (default) strips boilerplate via Readability. Partial success: per-format warnings never fail the whole call. Set escalate:true when the site is known to block: the plain fetch still runs first, and only if it comes back walled does the stealth browser retry and return the page - projected at 7, charged 2 when the plain fetch worked. Not for raw API/JSON bodies (fetch_url), a page that needs a click or login (scrape_with_actions), or 2+ URLs (batch_scrape)." + PAGE_CACHE_SENTENCE + " Cost: 2 credits. Example: scrape({url:\"https://example.com\", formats:[\"markdown\",\"links\",\"metadata\"]})",
   annotations: { title: "Scrape (Multi-Format)", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   // Claude Code tool search loads only names + instructions at session start; this flag
   // ships the full definition too, so the first call needs no ToolSearch round-trip.
@@ -1240,6 +1249,7 @@ registerToolIfEnabled("scrape", {
   inputSchema: {
     ...SCRAPE_INPUT_SHAPE,
     ...COMPLIANCE_PARAMS,
+    ...MAX_AGE_PARAM,
     ...MAX_INLINE_CHARS_PARAM,
     ...REDACT_PII_PARAM
   },

package/src/core/LLMsTxtAnalyzer.js

@@ -276,7 +276,10 @@ export class LLMsTxtAnalyzer {
             apis.push({
               url: new URL(href, baseUrl).toString(),
               type: 'documentation',
-              description: text.trim()
+              description: text.trim(),
+              // Link text as written, the label when the page itself was not
+              // classified (live sweep 7.4: it was labelled by its slug).
+              title: $(element).text().replace(/\s+/g, ' ').trim()
             });
           }
         });

package/src/core/pageCache.js

@@ -0,0 +1,249 @@
+/**
+ * pageCache — one copy of each fetched page per process, shared by the two
+ * fetch seams (REPEAT_CALL_REDUCTION_PLAN Phase 1).
+ *
+ * fetchWithTimeout (fetch_url, extract_metadata, scrape_structured,
+ * extract_text, extract_links) and fetchAndParse (scrape,
+ * extract_embedded_state, extract_structured, extract_with_llm, agent) both go
+ * through throughPageCache, so asking for the same page again with other
+ * formats, paths or selectors reuses the first fetch. What is kept is the
+ * response (status, final URL, headers, body), never a tool's output.
+ *
+ * Bounded by bytes, least recently used first out (CRAWLFORGE_PAGE_CACHE_MB,
+ * default 64; 0 turns it off). Memory only. A copy is served while it is
+ * younger than `maxAge` (default 10 min, D2); `maxAge: 0` fetches fresh and
+ * stores the result, `noCache: true` neither reads nor writes.
+ *
+ * Off on the hosted HTTP server and for internal (REST-proxy) requests until
+ * tenant isolation lands (D3), and off under NODE_ENV=test unless a test turns
+ * it on with _resetPageCache({ enabled: true }) (likewise inside node --test).
+ */
+
+import { createHash } from 'node:crypto';
+import { canonicalUrl } from '../utils/canonicalUrl.js';
+import { config } from '../constants/config.js';
+import { preflightDecision } from '../utils/robotsGate.js';
+import { plainVerdict } from '../tools/basic/_targetFailure.js';
+import { isInternalRequest, recordPageCache } from '../server/requestContext.js';
+
+export const DEFAULT_MAX_AGE_MS = 10 * 60 * 1000;
+const DEFAULT_CACHE_MB = 64;
+// A bot wall is tens of KB. Larger bodies skip the verdict's second parse.
+const VERDICT_MAX_BYTES = 1024 * 1024;
+// Requests that carry a credential are never served from or stored in the cache.
+const CREDENTIAL_HEADERS = new Set(['authorization', 'proxy-authorization', 'cookie']);
+
+/** key -> entry, least recently used first */
+const entries = new Map();
+/** in-flight key -> Promise<{ result, page, entry }> */
+const inflight = new Map();
+let totalBytes = 0;
+let testOverride = null;
+
+function maxBytes() {
+  if (testOverride?.maxBytes != null) return testOverride.maxBytes;
+  const raw = process.env.CRAWLFORGE_PAGE_CACHE_MB;
+  const mb = raw === undefined || raw === '' ? DEFAULT_CACHE_MB : Number(raw);
+  return (Number.isFinite(mb) && mb >= 0 ? mb : DEFAULT_CACHE_MB) * 1024 * 1024;
+}
+
+/** Whether this process and this request may use the cache at all. */
+export function pageCacheActive() {
+  // node --test marks its test processes with NODE_TEST_CONTEXT.
+  const underTest = process.env.NODE_ENV === 'test' || process.env.NODE_TEST_CONTEXT !== undefined;
+  if (underTest && !testOverride?.enabled) return false;
+  if (process.argv.includes('--http') || process.env.MCP_HTTP === 'true') return false;
+  if (isInternalRequest()) return false;
+  return maxBytes() > 0;
+}
+
+/**
+ * The fetch, canonicalised: URL, method, robots setting, identity, the
+ * caller's headers and API key, and which seam asked (the two send different
+ * Accept headers). Hashed, so no header value or key is kept in memory as a
+ * key. Null when the request must not touch the cache.
+ */
+export function pageCacheKey(url, options, variant) {
+  const canonical = canonicalUrl(url);
+  if (!canonical) return null;
+  if (options.method && String(options.method).toUpperCase() !== 'GET') return null;
+  const headers = Object.entries(options.headers || {})
+    .map(([name, value]) => [name.toLowerCase(), String(value)])
+    .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
+  if (headers.some(([name]) => CREDENTIAL_HEADERS.has(name))) return null;
+  const respect = options.respectRobots === undefined
+    ? config.crawling.respectRobots
+    : options.respectRobots !== false;
+  const userAgent = typeof options.userAgent === 'string' ? options.userAgent.trim() || null : null;
+  return createHash('sha256')
+    .update(JSON.stringify([variant, 'GET', canonical, respect, userAgent, headers, options.apiKey ?? null]))
+    .digest('hex');
+}
+
+/** The status, final URL, headers and body of a fetched response, as stored. */
+export function pageFromResponse(response, body) {
+  const headers = [];
+  if (typeof response.headers?.forEach === 'function') {
+    response.headers.forEach((value, name) => headers.push([name.toLowerCase(), value]));
+  } else {
+    // A headers object with only get() (test doubles): keep what the seams read.
+    for (const name of ['content-type', 'cache-control']) {
+      const value = response.headers?.get?.(name);
+      if (value != null) headers.push([name, value]);
+    }
+  }
+  return {
+    status: response.status,
+    statusText: response.statusText ?? '',
+    url: response.url,
+    redirected: response.redirected === true,
+    headers,
+    body,
+    fetchedAt: Date.now()
+  };
+}
+
+export function pageHeader(page, name) {
+  return page.headers.find(([key]) => key === name)?.[1] ?? null;
+}
+
+/** 2xx, storable, and not a bot wall (the same verdict extract_text applies). */
+function cacheable(page) {
+  if (!(page.status >= 200 && page.status <= 299) || typeof page.body !== 'string') return false;
+  if (/\bno-store\b/i.test(pageHeader(page, 'cache-control') ?? '')) return false;
+  if (Buffer.byteLength(page.body) <= VERDICT_MAX_BYTES) {
+    const verdict = plainVerdict(
+      { status: page.status, url: page.url, _body: page.body, headers: new Headers(page.headers) },
+      page.url
+    );
+    if (verdict?.blocked) return false;
+  }
+  return true;
+}
+
+function store(key, page) {
+  if (!cacheable(page)) return null;
+  const entry = {
+    status: page.status,
+    statusText: page.statusText,
+    url: page.url,
+    redirected: page.redirected,
+    headers: page.headers,
+    body: page.body,
+    responseTime: page.responseTime,
+    fetchedAt: page.fetchedAt
+  };
+  entry.bytes = Buffer.byteLength(entry.body) + Buffer.byteLength(JSON.stringify(entry.headers)) + (entry.url?.length ?? 0);
+  const cap = maxBytes();
+  if (entry.bytes > cap) return null;
+  drop(key);
+  entries.set(key, entry);
+  totalBytes += entry.bytes;
+  for (const [oldKey] of entries) {
+    if (totalBytes <= cap) break;
+    drop(oldKey);
+  }
+  return entry;
+}
+
+function drop(key) {
+  const entry = entries.get(key);
+  if (!entry) return;
+  entries.delete(key);
+  totalBytes -= entry.bytes;
+}
+
+function lookup(key, maxAge) {
+  const entry = entries.get(key);
+  if (!entry || Date.now() - entry.fetchedAt > maxAge) return null;
+  entries.delete(key);
+  entries.set(key, entry); // most recently used
+  return entry;
+}
+
+/**
+ * fetchAndParse asks for HTML (Accept: text/html…); fetchWithTimeout sends no
+ * Accept. A server that answered "any type" with HTML has an HTML variant, so
+ * a request preferring HTML gets the same: that copy serves fetchAndParse. The
+ * other direction is not safe (an API may answer `*\/*` with JSON).
+ */
+function lookupHtmlFromPlain(url, options, maxAge) {
+  const entry = lookup(pageCacheKey(url, options, 'plain'), maxAge);
+  const type = (entry && pageHeader(entry, 'content-type')) || '';
+  return /^\s*(?:text\/html|application\/xhtml\+xml)\b/i.test(type) ? entry : null;
+}
+
+async function serveHit(url, options, entry, fromEntry) {
+  // The blocklist and robots.txt are asked again; only the request is skipped.
+  const decision = await preflightDecision(url, options);
+  if (entry.redirected && entry.url) await preflightDecision(entry.url, options, { redirectedFrom: url });
+  recordPageCache('hit', entry.fetchedAt);
+  return fromEntry(entry, decision.warnings);
+}
+
+/**
+ * Serve a page from the cache, or fetch it once and keep it.
+ *
+ * @param {string} url
+ * @param {{ maxAge?: number, noCache?: boolean, headers?: object, userAgent?: string,
+ *   respectRobots?: boolean, apiKey?: string, method?: string, errorDocuments?: boolean }} options
+ * @param {'plain'|'html'} variant the seam asking
+ * @param {() => Promise<{ result: any, page: object }>} fetchFresh the seam's
+ *   own fetch: its usual return value plus the page as pageFromResponse gives it
+ * @param {(entry: object, warnings: string[]) => any} fromEntry the seam's
+ *   return value built from a stored page
+ */
+export async function throughPageCache(url, options, variant, fetchFresh, fromEntry) {
+  if (!pageCacheActive()) return (await fetchFresh()).result;
+
+  const key = options.noCache === true ? null : pageCacheKey(url, options, variant);
+  if (key === null) {
+    const { result, page } = await fetchFresh();
+    recordPageCache('bypass', page?.fetchedAt);
+    return result;
+  }
+
+  const maxAge = Number.isFinite(options.maxAge) && options.maxAge >= 0 ? options.maxAge : DEFAULT_MAX_AGE_MS;
+  // A caller that reads error bodies (errorDocuments) must not inherit the
+  // throw of one that does not, so the two never share an in-flight fetch.
+  const flightKey = options.errorDocuments ? `${key}:errors` : key;
+
+  if (maxAge > 0) {
+    const entry = lookup(key, maxAge) ?? (variant === 'html' ? lookupHtmlFromPlain(url, options, maxAge) : null);
+    if (entry) return serveHit(url, options, entry, fromEntry);
+    const pending = inflight.get(flightKey);
+    if (pending) {
+      // A failed fetch rejects every caller waiting on it; nothing is stored.
+      const shared = await pending;
+      if (shared.entry) return serveHit(url, options, shared.entry, fromEntry);
+      // Not storable (non-2xx, a wall): this call makes its own request.
+    }
+  }
+
+  const run = fetchFresh().then(({ result, page }) => ({ result, page, entry: page ? store(key, page) : null }));
+  if (!inflight.has(flightKey)) inflight.set(flightKey, run);
+  try {
+    const { result, page } = await run;
+    recordPageCache(maxAge > 0 ? 'miss' : 'bypass', page?.fetchedAt);
+    return result;
+  } finally {
+    if (inflight.get(flightKey) === run) inflight.delete(flightKey);
+  }
+}
+
+/** Bytes currently held (for tests and diagnostics). */
+export function pageCacheBytes() {
+  return totalBytes;
+}
+
+/**
+ * Empty the cache. Tests pass `{ enabled: true }` to turn it on under
+ * NODE_ENV=test or node --test, optionally with a small `maxBytes`; no argument restores the
+ * defaults.
+ */
+export function _resetPageCache(override = null) {
+  entries.clear();
+  inflight.clear();
+  totalBytes = 0;
+  testOverride = override;
+}

package/src/schemas/toolOutputSchemas.js

@@ -26,6 +26,15 @@ const costShape = z.object({
   projection_note: z.string().optional().describe('Human-readable note about how the cost was projected')
 }).passthrough().optional().describe('Cost-transparency metadata (D3.5): what this call was projected to cost and what it was charged');
 
+// `cache` is injected the same way when the call fetched through the page
+// cache (REPEAT_CALL_REDUCTION_PLAN 1.6).
+const cacheShape = z.object({
+  state: z.enum(['hit', 'miss', 'bypass']).optional().describe('hit: every page came from this session\'s copy; bypass: fetched fresh on request (maxAge 0) or not cacheable; miss: fetched'),
+  fetchedAt: z.string().nullable().optional().describe('When the oldest page used was fetched (ISO 8601)'),
+  ageMs: z.number().nullable().optional().describe('Age of that copy in ms'),
+  note: z.string().optional().describe('Hits only: how old the copy is and how to force a fresh fetch')
+}).passthrough().optional().describe('Page cache: whether this call reused a copy of the page fetched earlier in the session');
+
 const nextStepShape = z.string().optional().describe('Failed calls only: the tool or fix to try next');
 
 // ── result handles (Phase 2) ────────────────────────────────────────────────
@@ -144,6 +153,7 @@ const scrapeShape = {
   redaction: redactionShape,
   ...resultHandleShape,
   _cost: costShape,
+  cache: cacheShape,
   next_step: nextStepShape
 };
 
@@ -188,6 +198,7 @@ const mapSiteShape = {
   warnings: z.array(z.string()).optional().describe('Present when the map is partial, e.g. the start page could not be read and the URLs come from the sitemap only, or when the result is over max_inline_chars'),
   ...resultHandleShape,
   _cost: costShape,
+  cache: cacheShape,
   next_step: nextStepShape
 };
 
@@ -223,6 +234,7 @@ const serpRankShape = {
   cost: z.number().optional().describe('USD charged by DataForSEO for this lookup (separate from CrawlForge credits)'),
   checkedAt: z.string().optional(),
   _cost: costShape,
+  cache: cacheShape,
   next_step: nextStepShape
 };
 
@@ -277,6 +289,7 @@ const redditSearchShape = {
   notes: z.array(z.string()).optional().describe('Data-provenance caveats (archive freshness, coverage gaps)'),
   checkedAt: z.string().optional(),
   _cost: costShape,
+  cache: cacheShape,
   next_step: nextStepShape
 };
 
@@ -339,6 +352,7 @@ const searchWebShape = {
   warnings: z.array(z.string()).optional().describe('Notes on this result; over max_inline_chars, where the full result is kept and how to read it'),
   ...resultHandleShape,
   _cost: costShape,
+  cache: cacheShape,
   next_step: nextStepShape
 };
 
@@ -373,6 +387,7 @@ const extractStructuredShape = {
     skipped: z.string().optional().describe('"empty_source" when there was nothing to check against')
   }).passthrough().optional(),
   _cost: costShape,
+  cache: cacheShape,
   next_step: nextStepShape
 };
 
@@ -423,8 +438,10 @@ const crawlDeepShape = {
   crawled_at: z.string().optional().describe('When the pages were actually fetched (ISO 8601)'),
   cached: z.boolean().optional().describe('True when this response was replayed from an earlier crawl rather than crawled now; crawled_at gives its age'),
   redaction: redactionShape,
+  warnings: z.array(z.string()).optional().describe('Notes on this result; over max_inline_chars, where the full result is kept and how to read it'),
   ...resultHandleShape,
   _cost: costShape,
+  cache: cacheShape,
   next_step: nextStepShape
 };
 
@@ -464,6 +481,7 @@ const readResultShape = {
   truncated: z.boolean().optional().describe('search: more matches than returned; json_path: value replaced by a preview'),
   warnings: z.array(z.string()).optional(),
   _cost: costShape,
+  cache: cacheShape,
   next_step: nextStepShape
 };
 

package/src/server/inlineThreshold.js

@@ -222,10 +222,10 @@ function warningsOf(resultObject) {
  * @param {string} toolName
  * @param {object} resultObject — the parsed JSON result
  * @param {object} params — the tool params (max_inline_chars, url, operation)
- * @param {{ store: import('../core/ResultStore.js').ResultStore, env?: object }} deps
+ * @param {{ store: import('../core/ResultStore.js').ResultStore, env?: object, reserve?: number }} deps
  * @returns {{ result: object, stored: boolean }}
  */
-export function applyInlineThreshold(toolName, resultObject, params, { store, env = process.env }) {
+export function applyInlineThreshold(toolName, resultObject, params, { store, env = process.env, reserve = COST_BLOCK_RESERVE }) {
   const unchanged = { result: resultObject, stored: false };
   const config = INLINE_THRESHOLD_TOOLS[toolName];
   if (!config || (config.when && !config.when(params))) return unchanged;
@@ -254,7 +254,7 @@ export function applyInlineThreshold(toolName, resultObject, params, { store, en
   const readWith = 'read it with read_result (1 credit) - operation "search" (query), "slice" (offset, length), "lines" or "json_path" (path) - and do not fetch the page again';
 
   if (toolName === 'extract_embedded_state') {
-    return { result: shapeEmbeddedState(resultObject, { json, text, maxInline, handle, expires_at, readWith }), stored: true };
+    return { result: shapeEmbeddedState(resultObject, { json, text, maxInline, handle, expires_at, readWith, reserve }), stored: true };
   }
 
   const preview = text.slice(0, maxInline);
@@ -316,6 +316,13 @@ export function applyInlineThreshold(toolName, resultObject, params, { store, en
  */
 export const COST_BLOCK_RESERVE = 400;
 
+/**
+ * Extra room when the call went through the page cache: the `cache` block
+ * with its hit note (~250 chars) and the longer projection note (~70 chars).
+ * withAuth passes it only then, so other calls keep their preview budget.
+ */
+export const CACHE_BLOCK_RESERVE = 350;
+
 /** Object.keys(data) for a plain object, "array(<n>)" for an array, null otherwise. */
 export function embeddedDataKeys(data) {
   if (Array.isArray(data)) return `array(${data.length})`;
@@ -359,7 +366,7 @@ export function wholeLinePreview(text, budget) {
  * under the handle for read_result json_path. The REST route shapes it the
  * same way, measured as the compact JSON it sends; keep the two in step.
  */
-function shapeEmbeddedState(resultObject, { json, text, maxInline, handle, expires_at, readWith }) {
+function shapeEmbeddedState(resultObject, { json, text, maxInline, handle, expires_at, readWith, reserve = COST_BLOCK_RESERVE }) {
   const dataKeys = embeddedDataKeys(resultObject.data);
   const firstKey = Array.isArray(dataKeys) && dataKeys.length > 0 ? dataKeys[0] : null;
   const example = firstKey && /^[A-Za-z_$][\w$]*$/.test(firstKey) ? ` Read one payload with operation "json_path", path "data.${firstKey}".` : '';
@@ -386,7 +393,7 @@ function shapeEmbeddedState(resultObject, { json, text, maxInline, handle, expir
   };
   // The preview's value is one line of the pretty-printed text and escapes
   // the same way there as in compact JSON; `""` is its empty value's quotes.
-  const envelope = JSON.stringify({ ...shaped, preview: '', ...tail }, null, 2).length + COST_BLOCK_RESERVE;
+  const envelope = JSON.stringify({ ...shaped, preview: '', ...tail }, null, 2).length + reserve;
   const preview = wholeLinePreview(text, Math.max(0, maxInline - envelope + 2));
   return { ...shaped, preview, ...tail };
 }

package/src/server/requestContext.js

@@ -140,3 +140,43 @@ export function servingEra() {
 export function servingRequestId() {
   return requestContext.getStore()?.servingRequestId ?? null;
 }
+
+/**
+ * Record one page-cache outcome for this invocation (src/core/pageCache.js):
+ * 'hit' served a stored copy, 'miss' fetched and may have stored, 'bypass'
+ * fetched fresh because the caller asked (maxAge 0, noCache) or the request
+ * may not be cached. A tool can fetch several pages, so outcomes are counted,
+ * with the oldest copy's fetch time.
+ *
+ * @param {'hit'|'miss'|'bypass'} state
+ * @param {number} [fetchedAt] ms since epoch
+ */
+export function recordPageCache(state, fetchedAt) {
+  const store = requestContext.getStore();
+  if (!store) return;
+  const seen = store.pageCache ??= { hit: 0, miss: 0, bypass: 0, fetchedAt: null };
+  seen[state] += 1;
+  if (Number.isFinite(fetchedAt) && (seen.fetchedAt === null || fetchedAt < seen.fetchedAt)) {
+    seen.fetchedAt = fetchedAt;
+  }
+}
+
+/**
+ * This invocation's page-cache summary, or null when no fetch went through the
+ * cache. 'hit' only when every fetch was one; 'bypass' when every fetch was
+ * one; otherwise 'miss'.
+ *
+ * @returns {{ state: 'hit'|'miss'|'bypass', fetchedAt: string|null, ageMs: number|null } | null}
+ */
+export function pageCacheSummary(now = Date.now()) {
+  const seen = requestContext.getStore()?.pageCache;
+  if (!seen) return null;
+  const state = seen.miss === 0 && seen.bypass === 0 ? 'hit'
+    : seen.hit === 0 && seen.miss === 0 ? 'bypass'
+      : 'miss';
+  return {
+    state,
+    fetchedAt: seen.fetchedAt === null ? null : new Date(seen.fetchedAt).toISOString(),
+    ageMs: seen.fetchedAt === null ? null : Math.max(0, now - seen.fetchedAt)
+  };
+}