crawlforge-mcp-server 6.23.2 → 6.25.0

package/src/server/requestContext.js

@@ -140,3 +140,43 @@ export function servingEra() {
 export function servingRequestId() {
   return requestContext.getStore()?.servingRequestId ?? null;
 }
+
+/**
+ * Record one page-cache outcome for this invocation (src/core/pageCache.js):
+ * 'hit' served a stored copy, 'miss' fetched and may have stored, 'bypass'
+ * fetched fresh because the caller asked (maxAge 0, noCache) or the request
+ * may not be cached. A tool can fetch several pages, so outcomes are counted,
+ * with the oldest copy's fetch time.
+ *
+ * @param {'hit'|'miss'|'bypass'} state
+ * @param {number} [fetchedAt] ms since epoch
+ */
+export function recordPageCache(state, fetchedAt) {
+  const store = requestContext.getStore();
+  if (!store) return;
+  const seen = store.pageCache ??= { hit: 0, miss: 0, bypass: 0, fetchedAt: null };
+  seen[state] += 1;
+  if (Number.isFinite(fetchedAt) && (seen.fetchedAt === null || fetchedAt < seen.fetchedAt)) {
+    seen.fetchedAt = fetchedAt;
+  }
+}
+
+/**
+ * This invocation's page-cache summary, or null when no fetch went through the
+ * cache. 'hit' only when every fetch was one; 'bypass' when every fetch was
+ * one; otherwise 'miss'.
+ *
+ * @returns {{ state: 'hit'|'miss'|'bypass', fetchedAt: string|null, ageMs: number|null } | null}
+ */
+export function pageCacheSummary(now = Date.now()) {
+  const seen = requestContext.getStore()?.pageCache;
+  if (!seen) return null;
+  const state = seen.miss === 0 && seen.bypass === 0 ? 'hit'
+    : seen.hit === 0 && seen.miss === 0 ? 'bypass'
+      : 'miss';
+  return {
+    state,
+    fetchedAt: seen.fetchedAt === null ? null : new Date(seen.fetchedAt).toISOString(),
+    ageMs: seen.fetchedAt === null ? null : Math.max(0, now - seen.fetchedAt)
+  };
+}

package/src/server/withAuth.js

@@ -7,7 +7,11 @@
  *   - resolves toolCost once per call; every tool is metered (no free tier),
  *     so a valid API key is required for every invocation
  *   - try/finally guarantees a single `tool invocation` log line per call
- *   - log payload: { toolName, paramHash, durationMs, outcome, creditCost, creatorMode }
+ *   - log payload: { toolName, paramHash, durationMs, outcome, creditCost, creatorMode,
+ *     internal, runId, client, source, urlKey, cache } — runId is per server process
+ *     (CRAWLFORGE_RUN_ID overrides), client is "<name>/<version>" from initialize,
+ *     source ∈ { 'client' | 'test' | 'harness' | 'eval' }, urlKey is a hash of the
+ *     canonical URL (never the URL itself), cache ∈ { 'hit' | 'miss' | 'bypass' | null }
  *   - outcome ∈ { 'success' | 'error' | 'insufficient_credits' | 'input_required' }
  *   - an `input_required` return (Phase 4.4) is a round trip, not an answer: the
  *     handler did no work, so it is billed NOTHING and reports no usage. The
@@ -15,21 +19,59 @@
  *     once, so a confirmation costs exactly what the call always cost (G4).
  *   - a failed call (an isError result, a JSON body with success:false, or a
  *     throw) is charged NOTHING and reports no usage: failures are free
+ *   - a call whose every page fetch was served from the page cache
+ *     (src/core/pageCache.js) costs at most 1 credit (D1), says so in `cache`
+ *     and `_cost.projection_note`, and logs `cache: 'hit'`
  *   - error results get a "Next step:" hint naming the tool to try next
  *     (src/server/fallbackHints.js) so a failure is not followed by a blind retry
  *   - emits an OTel span via src/observability/tracing.js (no-op if disabled)
  *   - increments Prometheus counters via src/observability/metrics.js (if registry passed)
  */
 
-import { createHash } from 'node:crypto';
+import { createHash, randomUUID } from 'node:crypto';
 import { isInputRequiredResult } from '@modelcontextprotocol/server';
 import { recordToolInvocation } from '../observability/tracing.js';
-import { isInternalRequest, reportedActualCost, requestContext } from './requestContext.js';
+import { isInternalRequest, pageCacheSummary, reportedActualCost, requestContext, servingServer } from './requestContext.js';
 import { appendFallbackHint } from './fallbackHints.js';
 import { failure, normalizeErrorResult } from './errorText.js';
-import { INLINE_THRESHOLD_TOOLS, applyInlineThreshold } from './inlineThreshold.js';
+import { CACHE_BLOCK_RESERVE, COST_BLOCK_RESERVE, INLINE_THRESHOLD_TOOLS, applyInlineThreshold } from './inlineThreshold.js';
 import { REDACTION_TOOLS, redactionSurcharge, runRedactionStage } from './redaction.js';
 import { getResultStore } from '../core/ResultStore.js';
+import { canonicalUrl } from '../utils/canonicalUrl.js';
+
+// One id per server process; over stdio that is one client session.
+const PROCESS_RUN_ID = randomUUID().slice(0, 8);
+const LOG_SOURCES = new Set(['harness', 'eval']);
+// A cache hit costs at most this much (D1). agent is left out: its price pays
+// for searches and model steps the page cache does not cover.
+const CACHE_HIT_CREDITS = 1;
+const CACHE_HIT_EXCLUDED = new Set(['agent']);
+
+/** Who made this call: a test run, a named harness, or a real client. */
+function logSource() {
+  if (process.env.NODE_ENV === 'test') return 'test';
+  const declared = process.env.CRAWLFORGE_LOG_SOURCE;
+  return LOG_SOURCES.has(declared) ? declared : 'client';
+}
+
+/** First 12 hex chars of sha256(canonical URL); null when there is no single URL. */
+export function urlKey(url) {
+  const canonical = canonicalUrl(url);
+  return canonical ? createHash('sha256').update(canonical).digest('hex').slice(0, 12) : null;
+}
+
+/**
+ * "<name>/<version>" from the client's initialize. Over HTTP each session (or
+ * 2026-era request) is served by its own clone, stamped on the request context.
+ */
+function clientLabel(mcpServer) {
+  try {
+    const info = (servingServer() ?? mcpServer)?.server?.getClientVersion?.();
+    return info?.name ? `${info.name}/${info.version ?? ''}` : null;
+  } catch {
+    return null;
+  }
+}
 
 /**
  * Replace an over-threshold JSON text result (and its structuredContent, if
@@ -39,7 +81,9 @@ function shapeLargeResult(toolName, result, params) {
   if (!result || !Array.isArray(result.content) || result.content[0]?.type !== 'text') return;
   let parsed;
   try { parsed = JSON.parse(result.content[0].text); } catch { return; }
-  const { result: shaped, stored } = applyInlineThreshold(toolName, parsed, params, { store: getResultStore() });
+  // A call that went through the page cache also gets a `cache` block after shaping.
+  const reserve = COST_BLOCK_RESERVE + (pageCacheSummary() ? CACHE_BLOCK_RESERVE : 0);
+  const { result: shaped, stored } = applyInlineThreshold(toolName, parsed, params, { store: getResultStore(), reserve });
   if (!stored) return;
   result.content[0].text = JSON.stringify(shaped, null, 2);
   if (result.structuredContent) result.structuredContent = shaped;
@@ -63,6 +107,11 @@ async function redactResult(toolName, result, params, mcpServer) {
   return modelRan;
 }
 
+/** The one line a cache hit adds to the result (plan 1.6). */
+export function cacheHitLine(ageMs) {
+  return `Served from a copy of this page fetched ${Math.round((ageMs ?? 0) / 1000)} s ago in this session (pass maxAge: 0 for a fresh fetch).`;
+}
+
 export function hashParams(params) {
   try {
     return createHash('sha256').update(JSON.stringify(params ?? {})).digest('hex').slice(0, 12);
@@ -101,9 +150,16 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
       // A handler may report a lower spend (setActualCost). The projection is
       // the ceiling the caller saw before the call, so a report can only ever
       // lower the charge, never raise it.
+      // A page served from the cache costs CACHE_HIT_CREDITS in place of the
+      // tool's base price (D1). Only the base price is discounted: surcharges
+      // for work that still ran on the cached page (scrape's query and model
+      // formats) are still charged.
+      let cacheHitPrice = false;
+      const cacheHitDiscount = Math.max(0, authManager.getToolCost(toolName, {}) - CACHE_HIT_CREDITS);
       const billable = () => {
         const reported = reportedActualCost();
-        const spent = reported == null ? toolCost : Math.min(reported, toolCost);
+        let spent = reported == null ? toolCost : Math.min(reported, toolCost);
+        if (cacheHitPrice) spent = Math.max(Math.min(spent, CACHE_HIT_CREDITS), spent - cacheHitDiscount);
         return spent + (redactionModelRan ? redactionCost : 0);
       };
 
@@ -184,6 +240,9 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
           try { shapeLargeResult(toolName, result, params); } catch { /* shaping must never break the request path */ }
         }
 
+        const cache = isErrorResult ? null : pageCacheSummary();
+        cacheHitPrice = cache?.state === 'hit' && !CACHE_HIT_EXCLUDED.has(toolName);
+
         // Failures are free, as the website promises. A pre-flight refusal that
         // sank the call is one of them; a multi-URL tool that skipped one
         // disallowed URL and still returned a result is a success and bills.
@@ -202,7 +261,14 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
             projected: creditCost,
             actual: creatorMode ? 0 : charge,
             remaining_credits: remainingCredits,
-            projection_note: projection.note
+            projection_note: cacheHitPrice && charge > 0
+              ? `${projection.note} Served from this session's page cache: charged ${charge} instead of ${creditCost}.`
+              : projection.note
+          };
+          // 1.6: say where the page came from, and on a hit that a fresh fetch is one parameter away.
+          const cacheMeta = cache && {
+            ...cache,
+            ...(cache.state === 'hit' ? { note: cacheHitLine(cache.ageMs) } : {})
           };
 
           // Inject _cost into the first text content item if it's JSON
@@ -210,14 +276,17 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
             try {
               const parsed = JSON.parse(result.content[0].text);
               parsed._cost = costMeta;
+              if (cacheMeta) parsed.cache = cacheMeta;
               result.content[0].text = JSON.stringify(parsed, null, 2);
             } catch {
-              // Not JSON — skip injection silently
+              // Not JSON: the hit line goes on the end of the text instead.
+              if (cacheMeta?.note) result.content[0].text += `\n${cacheMeta.note}`;
             }
           }
           // ...and into structuredContent, which is all some clients show (R25 0.2).
           if (result?.structuredContent && typeof result.structuredContent === 'object') {
             result.structuredContent._cost = costMeta;
+            if (cacheMeta) result.structuredContent.cache = cacheMeta;
           }
         } catch {
           // Cost injection must never break the request path
@@ -255,7 +324,12 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
           outcome,
           creditCost,
           creatorMode,
-          internal
+          internal,
+          runId: process.env.CRAWLFORGE_RUN_ID || PROCESS_RUN_ID,
+          client: clientLabel(mcpServer),
+          source: logSource(),
+          urlKey: urlKey(params?.url),
+          cache: pageCacheSummary()?.state ?? null
         });
 
         // Prometheus (no-op unless registry was supplied)
@@ -304,6 +378,7 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
         ...(requestContext.getStore() ?? {}),
         preflightRefusal: null,
         actualCost: null,
+        pageCache: null,
         servingRequestId: ctx?.mcpReq?.id
       },
       () => invoke(params, ctx)

package/src/skills/agent-skills/crawlforge-batch-automation/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-batch-automation
 description: "Automates large scraping jobs and browser interactions with CrawlForge's batch_scrape, get_batch_results, scrape_with_actions, and generate_llms_txt tools. Use when the user wants to scrape many URLs at once, batch-scrape a list of pages, collect dozens of product, news, or competitor pages, run browser actions (click, type, scroll, wait) before scraping, log in or fill a form before extracting, or generate an llms.txt for a site. Use sync mode for up to about 25 URLs and async with a webhook for large batches; retrieve paginated output with get_batch_results."
 metadata:
-  version: 6.23.2
+  version: 6.25.0
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-brand-style-guide/SKILL.md

@@ -0,0 +1,107 @@
+---
+name: crawlforge-brand-style-guide
+description: "Builds a brand style guide or a design-token file from a live website with the branding format of CrawlForge's scrape tool. Use when the user wants a company's brand colours, fonts, logo, button styles or design tokens, asks to extract a website's style, design system or brand kit, wants a style guide made from a URL, wants to match a site's look, or wants tokens for Figma, Tokens Studio or Style Dictionary (W3C DTCG JSON). One scrape call returns colour roles, the primary and secondary buttons and type read from the rendered page, the logo and favicons, font families and grouped CSS tokens, each value labelled with where it came from."
+metadata:
+  version: 6.25.0
+  source: crawlforge-mcp-server
+---
+
+# CrawlForge Brand Style Guide
+
+Turn a website into a style guide or a design-token file. CrawlForge collects
+the brand data in one `scrape` call; you write the guide from it.
+
+## When to use
+
+- "Make a style guide from stripe.com" / "build a brand kit for this site"
+- "What colours and fonts does this company use?"
+- "Grab this site's logo and button styles"
+- "Match the look of this website"
+- "Export this site's design tokens for Figma / Style Dictionary"
+
+## The one call
+
+```json
+{
+  "tool": "scrape",
+  "params": {
+    "url": "https://stripe.com",
+    "formats": ["branding", "screenshot"],
+    "brandingOptions": { "dtcg": true }
+  }
+}
+```
+
+Cost: 2 credits. `branding` and `screenshot` share one browser render. Add
+`"metadata"` if you need the site name and description too. Leave out `dtcg`
+unless the user wants a token file: it can double the size of the result.
+
+If the result is `truncated: true`, do not scrape again. Read the parts with
+`read_result` and a `json_path`, such as `content.branding.roles` or
+`content.branding.dtcg`.
+
+## What comes back in `content.branding`
+
+| Key | What it holds |
+|-----|---------------|
+| `colorScheme` | `light` or `dark`, from the page background |
+| `roles` | `background`, `text`, `heading`, `link`, `primary`, `onPrimary`: each `{ value, source, evidence }` |
+| `components` | `buttonPrimary` and `buttonSecondary`: fill, label colour, border, radius, padding, font, size and weight |
+| `typography` | `body`, `h1`, `h2`: family, stack, size, weight, line height, letter spacing, colour |
+| `logo` | `inlineHeaderSvg` (the header logo as SVG), `favicons`, `ogImage`, image `candidates` |
+| `fonts`, `fontFaces` | font families ranked by use, and their `@font-face` sources |
+| `tokens` | the site's CSS custom properties, grouped: `colors.ramps`, `colors.semantic`, `typography`, `spacing`, `radius`, `shadow`, plus `stats` on what was dropped |
+| `colors` | every colour found, ranked by frequency |
+| `dtcg` | with `dtcg: true`, the tokens and roles as W3C Design Tokens (2025.10) JSON |
+| `provenance` | what each `source` value means |
+
+### Reading `source`
+
+- `computed`: read from an element on the rendered page. The `evidence` says
+  which, for example `button "Get started"`. Trust these first.
+- `css-variable`, `css-var`, `declaration`, `theme-color`: found in the
+  site's CSS or HTML.
+- `inferred`: a guess from CSS variable names, used when no element showed
+  the role. Say in the guide that the value is a guess, or confirm it on the
+  screenshot first.
+
+`sources.rendered` tells you whether the page was rendered. When it is
+`false`, `warnings` says why, and `components` and `typography` are empty.
+
+## Writing the guide
+
+Use the screenshot to check the roles before you write. Then cover:
+
+1. **Logo**: the header SVG on light and dark grounds, plus the favicon and
+   share-image links.
+2. **Colour roles**: one swatch per role, with its hex, its token name when a
+   ramp step matches, and what it is used for.
+3. **Palette**: the ramps from `tokens.colors.ramps`, in step order.
+4. **Typography**: the families, the type scale from `tokens.typography`, and
+   the body and heading styles from `typography`.
+5. **Spacing and radius**, from `tokens.spacing` and `tokens.radius`.
+6. **Buttons**: the primary and secondary buttons rebuilt from `components`.
+7. **Sources**: label each value as read from the page, found in CSS, or
+   inferred, so a reader can check it.
+
+For a page or artifact, show the swatches and buttons visually. For a token
+file, save `dtcg` as `tokens.json`.
+
+## Limits to tell the user about
+
+- The guide is unofficial when the site is not the user's own. Say so on the
+  page, and point to the company's own brand resources for anything that will
+  ship.
+- Many brand fonts are licensed. The `@font-face` URL shows which font a site
+  uses, not permission to use it, so render specimens in a fallback face.
+- CSS holds no usage rules: no logo clear space, minimum sizes or do's and
+  don'ts. Those live in brand documents.
+- `dtcg` follows the DTCG 2025.10 format, where colours and dimensions are
+  objects rather than strings. Support for 2025.10 varies by tool and version
+  (Style Dictionary lists it as in progress for v5), so tell the user to check
+  their tool before relying on the file.
+- `buttonSecondary` is the outline button next to the primary. A site with no
+  partner button returns none.
+- A blocked site comes back with a warning: retry with `escalate: true`. A
+  page that needed the stealth browser is not re-rendered for computed
+  styles, so its roles are `inferred`.

package/src/skills/agent-skills/crawlforge-browser-sessions/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-browser-sessions
 description: "Keeps one browser page alive across several tool calls with CrawlForge's browser_session tool. Use when the user needs to log in and then read pages behind that login, work through a multi-step flow (search, filter, paginate, fill a wizard), or look at a page before deciding what to click. The loop is: open a session on a URL, snapshot it to list the interactive elements with stable refs (@e1, @e2), act on those refs, read the content, close. Unlike scrape_with_actions, which is one-shot and closes its browser when the call returns, a session survives between calls, so one login is paid for once and a wrong selector costs one call instead of a whole chain."
 metadata:
-  version: 6.23.2
+  version: 6.25.0
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-change-tracking/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-change-tracking
 description: "Monitors web pages for changes over time with CrawlForge's track_changes tool. Use when the user wants to track changes to a page, watch a URL, monitor competitor pricing, detect when content updates, get notified of regulation or product-availability changes, or diff a page against a saved baseline. Workflow: create a baseline with operation create_baseline, then periodically compare with operation compare to get a change percentage and a diff; supports CSS-selector scoping, webhooks, and scheduled monitoring."
 metadata:
-  version: 6.23.2
+  version: 6.25.0
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-deep-research/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-deep-research
 description: "Runs multi-source web research and autonomous question-answering with CrawlForge's deep_research, agent, search_web, and reddit_search tools. Use when the user wants to research a topic, do a deep dive, compare competitors, gather facts with citations, answer a question from the web, search the web, or get a synthesized report from many sources, or search Reddit posts, comments, and threads. deep_research expands queries, fetches and dedupes sources, then synthesizes; agent autonomously plans searches and navigation from a plain-English prompt with no URLs needed; search_web returns ranked results; reddit_search searches Reddit via community archives (reddit.com blocks scrapers). Caps costs via max_urls and confirms before expensive runs."
 metadata:
-  version: 6.23.2
+  version: 6.25.0
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-getting-started/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-getting-started
 description: "Orientation and tool-selection guide for the CrawlForge MCP server's 31 web tools. Use when the user is getting started with CrawlForge, asks which CrawlForge tool to use, how to set up the API key, how skills or the CLI work, what a tool costs in credits, or when one tool fails and a fallback is needed. Routes requests to the right specialized skill (web scraping, deep research, stealth, structured extraction, change tracking, batch automation, browser sessions), and explains MCP-tools-vs-CLI, the Ollama-first LLM fallback chain, and per-tool credit costs."
 metadata:
-  version: 6.23.2
+  version: 6.25.0
   source: crawlforge-mcp-server
 ---
 
@@ -52,6 +52,7 @@ stored at `~/.crawlforge/config.json`.
 | Watch a page for changes / monitor pricing | **crawlforge-change-tracking** |
 | Scrape many URLs, run browser actions, generate llms.txt | **crawlforge-batch-automation** |
 | Keep a browser open across calls: log in then read, click through a multi-step flow | **crawlforge-browser-sessions** |
+| Make a style guide or design-token file from a website's brand | **crawlforge-brand-style-guide** |
 
 ## The 31 tools at a glance
 
@@ -108,3 +109,6 @@ get_batch_results...) up to 10+ for `deep_research`. Full table:
 [credits](references/credits.md). Prefer the cheapest tool that does the job, and
 cap dynamic tools (`deep_research`, `agent`, `crawl_deep`) with their max
 parameters.
+In a local session, asking again for the same page with different formats, paths or
+selectors reuses this session's copy for 10 minutes at 1 credit; pass `maxAge: 0` to
+force a fresh fetch.

package/src/skills/agent-skills/crawlforge-getting-started/references/credits.md

@@ -81,6 +81,10 @@ metered; there is no free tier. Tools marked "scales" cost more as work grows.
   when the whole interaction can be written down in advance.
 - A result over `max_inline_chars` costs nothing extra; read the stored copy with
   `read_result` (1) instead of fetching again.
+- In a local (stdio) session, asking again for a page read in the last 10 minutes
+  (`fetch_url`, `scrape`, `extract_*`, `scrape_structured`, `extract_structured`,
+  `extract_with_llm`) with different formats, paths or selectors reuses the copy and
+  costs 1 credit; `maxAge: 0` forces a fresh fetch at the full price.
 - On a site that blocks, one `scrape` with `escalate:true` costs at most 7 and
   only 2 when the plain fetch works; `scrape` then `stealth_mode` costs 7 either
   way. Do not set it on a site that reads fine — it raises the projection your

package/src/skills/agent-skills/crawlforge-stealth-browsing/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-stealth-browsing
 description: "Bypasses bot detection with CrawlForge's stealth_mode tool and looks up a country's locale settings with localization. Use when a site returns 403 or 429, CAPTCHAs, 'please enable JavaScript', or empty content, or is protected by Cloudflare, DataDome, or PerimeterX, or when the user needs a page in a specific locale, language, or timezone. stealth_mode runs a stealth browser (engine auto by default: camoufox/Firefox for advanced fingerprinting, playwright/Chromium when pinned for speed) and can screenshot; localization returns a country's Accept-Language, timezone and currency for you to pass to fetch_url or stealth_mode - it applies nothing itself and uses no proxy. Explains when to escalate from a normal scrape to stealth, and why a hard block needs a residential proxy."
 metadata:
-  version: 6.23.2
+  version: 6.25.0
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-structured-extraction/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-structured-extraction
 description: "Extracts structured JSON and analyzes content with CrawlForge's extract_structured, extract_with_llm, extract_embedded_state, scrape_structured, scrape_template, process_document, analyze_content, summarize_content, and list_ollama_models tools. Use when the user wants to extract specific fields, pull data into a JSON schema, extract by natural-language prompt, scrape with CSS selectors, read a page's embedded JavaScript state (__NEXT_DATA__, React Server Components, Nuxt, Apollo, Redux), get product, profile, or repo data from known sites (Amazon, LinkedIn, GitHub, YouTube, Reddit, and more), parse a PDF or DOCX, summarize a page, or analyze sentiment, entities, or keywords. Defaults to local Ollama for LLM extraction; OpenAI and Anthropic optional."
 metadata:
-  version: 6.23.2
+  version: 6.25.0
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-web-scraping/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-web-scraping
 description: "Scrapes web pages and returns clean Markdown, HTML, plain text, links, or metadata using CrawlForge's scrape, fetch_url, extract_content, extract_text, extract_links, extract_metadata, map_site, and crawl_deep tools. Use when the user wants to scrape a URL, fetch a page, get the markdown or text of a website, extract links or metadata, read an article, map a site's URLs, generate a sitemap, or crawl a whole website or documentation site. Prefer the unified scrape tool to get multiple formats in one call; use map_site to discover URLs and crawl_deep to walk an entire site."
 metadata:
-  version: 6.23.2
+  version: 6.25.0
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-web-scraping/references/tool-reference.md

@@ -10,6 +10,7 @@ CrawlForge credits.
 | `url` | string (URL) | — | Required. |
 | `formats` | array | `["markdown"]` | Any of `markdown`, `html`, `rawHtml`, `text`, `links`, `metadata`, `branding`, `screenshot`, `{type:"json", schema?, prompt?}`, `{type:"highlights", query, max_highlights?, mode?}` (the matching sentences, table rows and code blocks, verbatim, with offsets into the markdown; +1 credit once per call) or `{type:"question", question, mode?}` (an answer assembled from the evidence units; same +1). `mode:"model"` adds 3 credits and needs an LLM route. |
 | `onlyMainContent` | boolean | `true` | Strip boilerplate via Readability. |
+| `brandingOptions` | object | — | For `branding`: `computeStyles` (default `true`: render the page and read colour roles, the primary and secondary buttons and type from computed styles; `false` stays static and marks roles `inferred`), `rawVariables` (default `false`: add every CSS custom property as `cssVariables`), `dtcg` (default `false`: add the tokens as W3C Design Tokens 2025.10 JSON as `dtcg`, which can double the result; read it back with `read_result` and `json_path: "content.branding.dtcg"`), `fetchLinkedCss`, `maxStylesheets`. Every role and component carries `source` (`computed`, `css-variable`, `inferred`…); `provenance` in the result defines each. |
 | `timeoutMs` | number | `15000` | 1000–60000. |
 | `max_inline_chars` | number | `40000` | 1,000–10,000,000. A larger result returns `preview` + `result_handle` for `read_result` (below). |
 

package/src/tools/basic/_fetch.js

@@ -10,6 +10,7 @@ import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
 import { preflightFetch } from '../../utils/robotsGate.js';
 import { fetchResigned } from '../../utils/resignedFetch.js';
 import { withFetchCause } from '../../utils/fetchError.js';
+import { pageFromResponse, throughPageCache } from '../../core/pageCache.js';
 
 /**
  * Fetch a URL with a configurable timeout and body-size cap.
@@ -19,12 +20,39 @@ import { withFetchCause } from '../../utils/fetchError.js';
  * configurable cap from config.fetch.maxBodySize (env MAX_FETCH_BODY_SIZE,
  * default 25 MB).
  *
+ * The page goes through the process page cache (src/core/pageCache.js):
+ * `maxAge` (ms, default 10 min, 0 = fetch fresh) and `noCache` steer it. A
+ * copy served from it is a Response-like object with the same fields.
+ *
  * @param {string} url
  * @param {{ timeout?: number, headers?: Record<string,string>, userAgent?: string,
- *   respectRobots?: boolean, tool?: string, apiKey?: string }} [options]
+ *   respectRobots?: boolean, tool?: string, apiKey?: string, maxAge?: number,
+ *   noCache?: boolean }} [options]
  * @returns {Promise<Response & { _body: string, _warnings: string[] }>}
  */
-export async function fetchWithTimeout(url, options = {}) {
+export function fetchWithTimeout(url, options = {}) {
+  return throughPageCache(url, options, 'plain', () => fetchFresh(url, options), responseFromPage);
+}
+
+/** A stored page as the Response-like object fetchFresh returns. */
+function responseFromPage(page, warnings) {
+  return {
+    ok: page.status >= 200 && page.status <= 299,
+    status: page.status,
+    statusText: page.statusText,
+    url: page.url,
+    redirected: page.redirected,
+    headers: new Headers(page.headers),
+    text: () => Promise.resolve(page.body),
+    json: () => Promise.resolve(JSON.parse(page.body)),
+    _body: page.body,
+    _responseTime: page.responseTime,
+    _warnings: warnings
+  };
+}
+
+/** The network fetch; resolves to { result: the Response, page: what the cache may keep }. */
+async function fetchFresh(url, options) {
   const { timeout = 10000, headers = {}, userAgent, respectRobots, tool, apiKey } = options;
   const maxBodySize = config.fetch.maxBodySize;
 
@@ -91,13 +119,16 @@ export async function fetchWithTimeout(url, options = {}) {
     }
 
     // Attach the pre-read text so callers can call .text() on the result.
-    return Object.assign(response, {
+    const responseTime = Date.now() - startedAt;
+    const page = { ...pageFromResponse(response, bodyText), responseTime };
+    const result = Object.assign(response, {
       text: () => Promise.resolve(bodyText),
       json: () => Promise.resolve(JSON.parse(bodyText)),
       _body: bodyText,
-      _responseTime: Date.now() - startedAt,
+      _responseTime: responseTime,
       _warnings: gate.warnings
     });
+    return { result, page };
   } finally {
     clearTimeout(timeoutId);
   }

package/src/tools/basic/extractLinks.js

@@ -11,16 +11,17 @@ import { fetchLadder, ladderErrorResult, isJsonType } from '../../utils/fetchLad
 
 /**
  * @param {{ url: string, filter_external?: boolean, base_url?: string,
- *   user_agent?: string, respect_robots?: boolean,
+ *   user_agent?: string, respect_robots?: boolean, maxAge?: number,
  *   escalate?: boolean, escalate_engine?: string }} params
  * @param {Function} [escalateFetch] the stealth escalation stage
  */
-async function extractLinks({ url, filter_external, base_url, user_agent, respect_robots, escalate, escalate_engine }, escalateFetch) {
+async function extractLinks({ url, filter_external, base_url, user_agent, respect_robots, maxAge, escalate, escalate_engine }, escalateFetch) {
   try {
     const ladder = await fetchLadder(url, {
       tool: 'extract_links',
       userAgent: user_agent,
       respectRobots: respect_robots,
+      maxAge,
       escalate: escalate === true,
       escalateEngine: escalate_engine,
       escalateFetch

package/src/tools/basic/extractMetadata.js

@@ -64,13 +64,14 @@ function redirectFields(requested, final) {
 
 /**
  * @param {{ url: string, user_agent?: string, respect_robots?: boolean,
- *   json_ld_types?: string[] }} params
+ *   json_ld_types?: string[], maxAge?: number }} params
  */
-export async function extractMetadataHandler({ url, user_agent, respect_robots, json_ld_types }) {
+export async function extractMetadataHandler({ url, user_agent, respect_robots, json_ld_types, maxAge }) {
   try {
     const response = await fetchWithTimeout(url, {
       userAgent: user_agent,
       respectRobots: respect_robots,
+      maxAge,
       tool: 'extract_metadata'
     });
     if (!response.ok) {

package/src/tools/basic/extractText.js

@@ -105,16 +105,17 @@ function isCut(text, maxLength) {
 /**
  * @param {{ url: string, remove_scripts?: boolean, remove_styles?: boolean,
  *   output_format?: "text"|"markdown", selector?: string, max_length?: number,
- *   user_agent?: string, respect_robots?: boolean,
+ *   user_agent?: string, respect_robots?: boolean, maxAge?: number,
  *   escalate?: boolean, escalate_engine?: string }} params
  * @param {Function} [escalateFetch] the stealth escalation stage
  */
-async function extractText({ url, remove_scripts, remove_styles, output_format, selector, max_length, user_agent, respect_robots, escalate, escalate_engine }, escalateFetch) {
+async function extractText({ url, remove_scripts, remove_styles, output_format, selector, max_length, user_agent, respect_robots, maxAge, escalate, escalate_engine }, escalateFetch) {
   try {
     const ladder = await fetchLadder(url, {
       tool: 'extract_text',
       userAgent: user_agent,
       respectRobots: respect_robots,
+      maxAge,
       escalate: escalate === true,
       escalateEngine: escalate_engine,
       escalateFetch

package/src/tools/basic/fetchUrl.js

@@ -7,15 +7,16 @@ import { fetchWithTimeout } from './_fetch.js';
 
 /**
  * @param {{ url: string, headers?: Record<string,string>, timeout?: number,
- *   user_agent?: string, respect_robots?: boolean }} params
+ *   user_agent?: string, respect_robots?: boolean, maxAge?: number }} params
  */
-export async function fetchUrlHandler({ url, headers, timeout, user_agent, respect_robots }) {
+export async function fetchUrlHandler({ url, headers, timeout, user_agent, respect_robots, maxAge }) {
   try {
     const response = await fetchWithTimeout(url, {
       timeout: timeout || 10000,
       headers: headers || {},
       userAgent: user_agent,
       respectRobots: respect_robots,
+      maxAge,
       tool: 'fetch_url'
     });
 

package/src/tools/basic/scrapeStructured.js

@@ -139,13 +139,14 @@ function extractRows($, { row_selector, selectors, max_results, baseUrl }) {
 
 /**
  * @param {{ url: string, selectors: Record<string, string|{ selector: string, multiple?: boolean }>, row_selector?: string,
- *   max_results?: number, user_agent?: string, respect_robots?: boolean }} params
+ *   max_results?: number, user_agent?: string, respect_robots?: boolean, maxAge?: number }} params
  */
-export async function scrapeStructuredHandler({ url, selectors, row_selector, max_results, user_agent, respect_robots }) {
+export async function scrapeStructuredHandler({ url, selectors, row_selector, max_results, user_agent, respect_robots, maxAge }) {
   try {
     const response = await fetchWithTimeout(url, {
       userAgent: user_agent,
       respectRobots: respect_robots,
+      maxAge,
       tool: 'scrape_structured'
     });
     if (!response.ok) {