crawlforge-mcp-server 5.6.5 → 5.6.7

package/CLAUDE.md

@@ -62,7 +62,7 @@ These guidelines are working if: fewer unnecessary changes in diffs, fewer rewri
 
 CrawlForge MCP Server - A professional MCP (Model Context Protocol) server providing 29 web scraping, crawling, and content processing tools (6 inline + 23 advanced).
 
-**Current Version:** 5.6.0
+**Current Version:** 5.6.6
 
 ## Development Commands
 
@@ -307,8 +307,8 @@ try {
 
 ## Project Management Rules
 
-- Always have the project manager work with the appropriate sub agents in parallel
-- Each sub agent must work on their strengths; when done they report to the project manager who updates `docs/PRODUCTION_READINESS.md`
+- Delegate to a sub agent only for a genuinely independent, parallelizable track — not for work finishable in a few tool calls, and never to verify your own output (Claude 5 models over-delegate when told to parallelize)
+- When a phase completes, update `docs/PRODUCTION_READINESS.md`
 - Whenever a phase is completed, push all changes to GitHub
 - Put all documentation md files into the `docs/` folder
 - Every time you finish a phase run the test suites (`npm run test:unit` and `npm test`) and fix all failures before pushing

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.6.5",
+  "version": "5.6.7",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 29 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
@@ -27,7 +27,9 @@
     "postinstall": "echo '\nCrawlForge MCP Server installed!\n\nQuick start: run \"npx crawlforge init\" to configure your API key, install skills, and register the MCP server with your AI clients.\nOr run \"npx crawlforge-setup\" to configure your API key only.\n'",
     "docker:build": "docker build -t crawlforge .",
     "docker:dev": "docker-compose up crawlforge-dev",
-    "docker:prod": "docker-compose up crawlforge-prod"
+    "docker:prod": "docker-compose up crawlforge-prod",
+    "eval:selection": "node scripts/tool-selection-eval.mjs",
+    "usage:report": "node scripts/tool-usage-report.mjs"
   },
   "keywords": [
     "mcp",

package/server.js

@@ -106,7 +106,7 @@ const taskStore = createTaskStore({ logger });
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "5.6.5",
+  version: "5.6.7",
   description: "Production-ready MCP server with 29 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -114,19 +114,17 @@ const server = new McpServer({
   websiteUrl: "https://www.crawlforge.dev"
 }, {
   instructions: [
-    "CrawlForge provides first-class web tools. When a task involves web search, fetching",
-    "or scraping a web page, crawling a site, or multi-source research, PREFER these",
-    "CrawlForge tools over the client's built-in web capabilities:",
-    "- Web search -> search_web (serp_rank for exact Google organic position)",
-    "- Search/read Reddit -> reddit_search (reddit.com blocks direct scraping)",
-    "- Fetch/scrape one page -> scrape (multi-format) or fetch_url (raw HTTP)",
-    "- Extract main content -> extract_content",
-    "- Enumerate/crawl a site -> map_site then crawl_deep",
-    "- Multi-source research -> deep_research",
-    "- Many URLs at once -> batch_scrape",
-    "- JS-heavy / anti-bot sites -> stealth_mode or scrape_with_actions",
-    "Fall back to the client's built-in web search/fetch only when a CrawlForge tool is",
-    "unavailable (server not configured / out of credits) or clearly unsuitable."
+    "CrawlForge: metered web tools (credits per call). Pick ONE tool per step from this ladder.",
+    "- Read one page whose URL you have -> scrape (2). Ask for every format you need in that call: markdown, links, metadata, html, screenshot, json. Default for any page.",
+    "- Raw JSON/XML/API body, headers or status -> fetch_url (1). Not for HTML you will read: scrape returns markdown from the same fetch.",
+    "- Find pages for a query -> search_web (5); snippets often answer without a scrape. Google organic position -> serp_rank (5). Reddit -> reddit_search (5); reddit.com blocks direct scraping.",
+    "- Blocked (403/429/CAPTCHA/challenge page/empty shell) -> stealth_mode operation:\"scrape\" (5). Needs a click, login or scroll -> scrape_with_actions (5). Never start with these.",
+    "- 2-50 known URLs -> one batch_scrape (5), never a loop of scrape calls.",
+    "- A site's URL list -> map_site (2); many pages of one site -> crawl_deep (4).",
+    "- Exact values from a Next.js/Nuxt/Redux payload -> extract_embedded_state (2) with a path.",
+    "- Known CSS selectors -> scrape_structured (2); fields you can describe but not select -> extract_structured (3).",
+    "- A report from several sources -> ONE deep_research call (10 + ~1 per 5 sources); it replaces a search_web + scrape fan-out that costs 5 per search and 2 per page. Open question with no URLs -> agent (8).",
+    "Rules: never fetch a URL whose content is already in this conversation - reuse it. One call per page: scrape with several formats replaces fetch_url + extract_* pairs. Error results end with \"Next step:\" naming the tool to try; follow it instead of retrying the same call. Use the client's built-in web search/fetch only when CrawlForge is unavailable or out of credits."
   ].join("\n"),
   taskStore
 });
@@ -143,25 +141,42 @@ server.registerPrompt("getting-started", {
       role: "user",
       content: {
         type: "text",
-        text: "You have access to CrawlForge MCP with 29 web scraping tools. Key tools:\n\n" +
-          "- fetch_url: Fetch raw HTML/content from any URL\n" +
-          "- extract_text: Extract clean text from a webpage\n" +
-          "- extract_content: Smart content extraction with readability\n" +
-          "- search_web: Search the web and get structured results\n" +
-          "- serp_rank: Check where a domain ranks in Google's real organic SERP for a keyword\n" +
-          "- reddit_search: Search Reddit posts/comments or read a full thread (reddit.com blocks direct scraping)\n" +
-          "- crawl_deep: Crawl a website following links to a specified depth\n" +
-          "- map_site: Discover all pages on a website\n" +
-          "- batch_scrape: Scrape multiple URLs in parallel\n" +
-          "- scrape_with_actions: Automate browser actions then scrape\n" +
-          "- deep_research: Multi-source research on any topic\n" +
-          "- stealth_mode: Anti-detection browsing for protected sites\n" +
-          "- extract_structured: LLM-powered structured data extraction\n" +
-          "- extract_with_llm: Natural-language extraction — defaults to local Ollama (no API key); openai/anthropic available with key\n" +
-          "- list_ollama_models: List installed Ollama models so you can pick one for extract_with_llm\n" +
-          "- track_changes: Monitor website changes over time\n" +
-          "- generate_llms_txt: Generate llms.txt for any website\n\n" +
-          "Workflow: search_web -> fetch_url -> extract_content -> analyze_content\n\n" +
+        text: "You have access to CrawlForge MCP: 29 metered web tools (credits per call, shown in brackets). Pick one tool per step from this ladder, and reuse content already in the conversation instead of fetching it again.\n" +
+          "\n" +
+          "Read a page\n" +
+          "- scrape (2): one URL, every format you need in one call - markdown, links, metadata, html, screenshot, json. Default for any page whose URL you have.\n" +
+          "- fetch_url (1): raw JSON/XML/API bodies, headers, status. Not for HTML you will read.\n" +
+          "- extract_text (1), extract_content (2), extract_links (1), extract_metadata (1): single-format reads; scrape with several formats replaces any pair of them.\n" +
+          "- extract_embedded_state (2): exact values from a Next.js/Nuxt/Redux payload - pass a path.\n" +
+          "- process_document (2): PDF text, sections and metadata.\n" +
+          "\n" +
+          "Find pages\n" +
+          "- search_web (5): titles, URLs, snippets; language, date and site filters. Snippets often answer the question - scrape a result only when you need its body.\n" +
+          "- serp_rank (5): a domain's real Google organic position for a keyword.\n" +
+          "- reddit_search (5): Reddit posts, comments, or a full thread (reddit.com blocks direct scraping).\n" +
+          "\n" +
+          "Many pages\n" +
+          "- batch_scrape (5): 2-50 known URLs in one call; get_batch_results (1) reads an async job.\n" +
+          "- map_site (2): a site's URL list. crawl_deep (4): follow links and fetch pages.\n" +
+          "\n" +
+          "Hard pages\n" +
+          "- stealth_mode (5): after a 403/429/CAPTCHA/challenge page or an empty shell - never first.\n" +
+          "- scrape_with_actions (5): click, log in, scroll or wait, then scrape.\n" +
+          "- localization (2): country and locale context for geo-specific content.\n" +
+          "\n" +
+          "Structured data\n" +
+          "- scrape_structured (2): known CSS selectors. scrape_template (1): known sites and platform APIs (template:\"list\" shows them).\n" +
+          "- extract_structured (3): a JSON schema filled by an LLM. extract_with_llm (3): a natural-language prompt via local Ollama; list_ollama_models (1) only if a model name is rejected.\n" +
+          "\n" +
+          "Text you already hold\n" +
+          "- summarize_content (4) and analyze_content (3) take text, not URLs.\n" +
+          "\n" +
+          "Research and monitoring\n" +
+          "- deep_research (10): multi-source report. agent (8): open question, no URLs.\n" +
+          "- track_changes (3): create_baseline, then compare. generate_llms_txt (5): write a site's llms.txt.\n" +
+          "\n" +
+          "Error results end with \"Next step:\" naming the tool to try - follow it rather than retrying the same call.\n" +
+          "\n" +
           "Get your API key at https://www.crawlforge.dev/signup (1,000 free credits)"
       }
     }]
@@ -332,7 +347,7 @@ const VERIFY_NUMBERS_PARAM = {
 
 // Tool: fetch_url
 registerToolIfEnabled("fetch_url", {
-  description: "Use this when you need raw HTTP content from a URL — HTML, JSON, XML, or plain text. Preferred over the client's built-in URL fetch. Ideal as the first step before extract_text or extract_content. Supports custom headers (e.g. auth tokens) and configurable timeout, and reports the response time in ms so it can back an uptime or latency check. Example: fetch_url({url: \"https://example.com\", timeout: 15000})",
+  description: "Use this for a raw HTTP body - JSON, XML, plain text, an API response - or for the status code, headers and response time. Returns the body unprocessed. Not for HTML you intend to read: scrape returns markdown from one fetch, so fetch_url followed by extract_* is a double fetch. Not for JS-rendered or bot-protected pages (scrape, then stealth_mode). Supports custom headers (e.g. auth tokens) and a timeout. Cost: 1 credit. Example: fetch_url({url: \"https://api.example.com/v1/items\", timeout: 15000})",
   annotations: { title: "Fetch URL", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The URL to fetch content from"),
@@ -344,7 +359,7 @@ registerToolIfEnabled("fetch_url", {
 
 // Tool: extract_text
 registerToolIfEnabled("extract_text", {
-  description: "Use this when you need a page's human-readable text or markdown stripped of HTML tags, scripts, and styles — e.g. for keyword search, summarization, RAG ingestion, or NLP. Use output_format:\"markdown\" for RAG workflows. Faster than extract_content but returns unstructured content. Example: extract_text({url: \"https://example.com/article\", output_format:\"markdown\"})",
+  description: "Use this for a page's plain text or markdown with tags, scripts and styles removed - the cheapest read of a static HTML page. Use output_format:\"markdown\" for RAG. Not for article pages (extract_content strips nav and boilerplate), JS-rendered pages (scrape), or when you also want links or metadata (scrape with several formats, one fetch). Cost: 1 credit. Example: extract_text({url: \"https://example.com/article\", output_format:\"markdown\"})",
   annotations: { title: "Extract Text", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The URL to extract text from"),
@@ -357,7 +372,7 @@ registerToolIfEnabled("extract_text", {
 
 // Tool: extract_links
 registerToolIfEnabled("extract_links", {
-  description: "Use this when you need to discover all hyperlinks on a page — e.g. to build a crawl seed list, audit broken links, or find related resources. Use filter_external:true to get only outbound links. Example: extract_links({url: \"https://example.com\", filter_external: true})",
+  description: "Use this to list the hyperlinks on one page - a crawl seed list, a broken-link audit, related resources. filter_external:true returns only outbound links. Not for a whole site (map_site), and not alongside a scrape of the same URL: scrape formats:[\"markdown\",\"links\"] returns both in one fetch. Cost: 1 credit. Example: extract_links({url: \"https://example.com\", filter_external: true})",
   annotations: { title: "Extract Links", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The URL to extract links from"),
@@ -369,7 +384,7 @@ registerToolIfEnabled("extract_links", {
 
 // Tool: extract_metadata
 registerToolIfEnabled("extract_metadata", {
-  description: "Use this when you need a page's SEO metadata: title, meta description, Open Graph tags, canonical URL, schema.org data. Ideal for site audits and competitive SEO analysis. Example: extract_metadata({url: \"https://example.com\"})",
+  description: "Use this for a page's SEO metadata only: title, meta description, Open Graph tags, canonical URL, schema.org data. Not alongside a scrape of the same URL: scrape formats:[\"markdown\",\"metadata\"] returns both in one fetch. Cost: 1 credit. Example: extract_metadata({url: \"https://example.com\"})",
   annotations: { title: "Extract Metadata", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The URL to extract metadata from"),
@@ -380,7 +395,7 @@ registerToolIfEnabled("extract_metadata", {
 
 // Tool: extract_embedded_state
 registerToolIfEnabled("extract_embedded_state", {
-  description: "Use this when a page's data lives in its embedded JavaScript state rather than its rendered HTML — Next.js (__NEXT_DATA__ and React Server Component payloads), Nuxt, Apollo, Redux (__INITIAL_STATE__, __PRELOADED_STATE__), and <script type=\"application/json\"> blocks. One fetch, exact values, no LLM in the extraction path, so nothing can be fabricated. Payloads are routinely over a megabyte — pass `path` to return one subtree instead of the whole blob. Example: extract_embedded_state({url: \"https://www.ticketmaster.com/discover/concerts\", path: \"next_data.props.pageProps\"})",
+  description: "Use this when a page's data lives in its embedded JavaScript state rather than its rendered HTML - Next.js (__NEXT_DATA__ and React Server Component payloads), Nuxt, Apollo, Redux (__INITIAL_STATE__, __PRELOADED_STATE__), and <script type=\"application/json\"> blocks. One fetch, exact values, no LLM in the extraction path, so nothing can be fabricated. Payloads are routinely over a megabyte - pass `path` to return one subtree instead of the whole blob. Not for the rendered text of a page (scrape) or for sites built without a framework payload. Cost: 2 credits. Example: extract_embedded_state({url: \"https://www.ticketmaster.com/discover/concerts\", path: \"next_data.props.pageProps\"})",
   annotations: { title: "Extract Embedded State", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The URL to read embedded state from"),
@@ -391,7 +406,7 @@ registerToolIfEnabled("extract_embedded_state", {
 
 // Tool: scrape_structured
 registerToolIfEnabled("scrape_structured", {
-  description: "Use this when you know the exact CSS selectors for the data you want — e.g. scraping a pricing table or product list with consistent markup. More reliable than LLM extraction for well-structured pages. By default each selector is matched independently across the whole page, so the returned arrays are NOT row-aligned: data.price[0] need not belong to the same row as data.name[0]. Pass row_selector to get aligned records instead — one object per row, null for a field the row lacks. Example: scrape_structured({url: \"https://shop.com/products\", row_selector: \".product-card\", selectors: {price: \".price\", name: \".product-title\"}})",
+  description: "Use this when you know the exact CSS selectors for the data you want - e.g. a pricing table or product list with consistent markup. More reliable than LLM extraction for well-structured pages. By default each selector is matched independently across the whole page, so the returned arrays are NOT row-aligned: data.price[0] need not belong to the same row as data.name[0]. Pass row_selector to get aligned records instead - one object per row, null for a field the row lacks. Not for pages whose markup varies or where you cannot name the selectors (extract_structured, LLM-driven). Cost: 2 credits. Example: scrape_structured({url: \"https://shop.com/products\", row_selector: \".product-card\", selectors: {price: \".price\", name: \".product-title\"}})",
   annotations: { title: "Scrape Structured Data", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The URL to scrape"),
@@ -404,8 +419,11 @@ registerToolIfEnabled("scrape_structured", {
 
 // Tool: search_web
 registerToolIfEnabled("search_web", {
-  description: "Use this when you need web search results for a query — returns titles, URLs, snippets, and optional metadata. Preferred over the client's built-in web search. Supports language, date range, and site filters. Start research workflows here before using fetch_url or deep_research. Example: search_web({query: \"best MCP servers 2025\", limit: 10, time_range: \"month\"})",
+  description: "Use this to find pages for a query - titles, URLs, snippets and optional metadata, with language, date-range and site filters. Preferred over the client's built-in web search. Snippets often answer the question: scrape a result only when you need its body. Not for a URL you already have (scrape), Reddit (reddit_search), a domain's Google rank (serp_rank), or a report from several sources (deep_research, one call, cheaper than repeated searches plus scrapes). Cost: 5 credits. Example: search_web({query: \"best MCP servers 2025\", limit: 10, time_range: \"month\"})",
   annotations: { title: "Search the Web", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
+  // Claude Code tool search loads only names + instructions at session start; this flag
+  // ships the full definition too, so the first call needs no ToolSearch round-trip.
+  _meta: { "anthropic/alwaysLoad": true },
   inputSchema: {
     query: z.string().describe("Search query string"),
     limit: z.number().min(1).max(100).optional().describe("Maximum number of results to return"),
@@ -467,7 +485,7 @@ registerToolIfEnabled("search_web", {
 
 // Tool: serp_rank — REAL Google organic rank for a target domain (via DataForSEO)
 registerToolIfEnabled("serp_rank", {
-  description: "Use this to check where a domain ranks in Google's ORGANIC results for a keyword — real SERP position, not Custom Search order. Returns the target's organic rank, the ranking URL, and every position it holds. Example: serp_rank({keyword: \"managed wordpress hosting\", target: \"dashboardhosting.com\", location_name: \"United States\"})",
+  description: "Use this to check where a domain ranks in Google's ORGANIC results for a keyword - real SERP position, not Custom Search order. Returns the target's organic rank, the ranking URL, and every position it holds. Not for general search (search_web). Requires DataForSEO credentials and returns configured:false without them - do not retry in that case. Cost: 5 credits (0 when unconfigured). Example: serp_rank({keyword: \"managed wordpress hosting\", target: \"dashboardhosting.com\", location_name: \"United States\"})",
   annotations: { title: "SERP Rank Check", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     keyword: z.string().describe("The search query to check ranking for"),
@@ -493,7 +511,7 @@ registerToolIfEnabled("serp_rank", {
 
 // Tool: reddit_search — search Reddit posts/comments or read a full thread (via community archives)
 registerToolIfEnabled("reddit_search", {
-  description: "Use this to search Reddit posts or comments, or read a full comment thread — reddit.com blocks direct scraping, so this reads the Arctic Shift community archive instead (free, no Reddit credentials). Modes: 'posts' (default) and 'comments' search; 'thread' returns a post plus its nested comment tree by link_id. A subreddit/author-scoped search queries the archive directly. A keyword search across ALL of Reddit finds posts with a site-restricted web search and then reads those posts from the archive, because Arctic Shift can only keyword-search within a scope; results come back as real archive rows, ordered by search relevance. An unscoped COMMENT search discovers posts the same way and then searches each post's comments for the keywords. A scoped comment search Arctic Shift times out on is retried over narrower windows (7d, 3d, 1d) and reports window_applied. Example: reddit_search({query: \"best mechanical keyboard\", subreddit: \"MechanicalKeyboards\", limit: 10})",
+  description: "Use this to search Reddit posts or comments, or read a full comment thread - reddit.com blocks direct scraping, so this reads the Arctic Shift community archive instead (free, no Reddit credentials). Modes: 'posts' (default) and 'comments' search; 'thread' returns a post plus its nested comment tree by link_id. A subreddit/author-scoped search queries the archive directly. A keyword search across ALL of Reddit finds posts with a site-restricted web search and then reads those posts from the archive, because Arctic Shift can only keyword-search within a scope; results come back as real archive rows, ordered by search relevance. An unscoped COMMENT search discovers posts the same way and then searches each post's comments for the keywords. A scoped comment search Arctic Shift times out on is retried over narrower windows (7d, 3d, 1d) and reports window_applied. Not for reddit.com URLs via scrape or fetch_url (blocked) - use mode:'thread' with the post's link_id. Cost: 5 credits. Example: reddit_search({query: \"best mechanical keyboard\", subreddit: \"MechanicalKeyboards\", limit: 10})",
   annotations: { title: "Reddit Search", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     query: z.string().optional().describe("Keyword search. Posts: matches title+selftext; comments: matches body. Supports \"quoted phrases\", OR, -exclusion"),
@@ -520,7 +538,7 @@ registerToolIfEnabled("reddit_search", {
 // Tool: crawl_deep (async task pattern — Phase 6; taskSupport:'optional' keeps sync callers working)
 if (toolFilter.isEnabled("crawl_deep")) {
   server.experimental.tasks.registerToolTask("crawl_deep", {
-    description: "Use this when you need to discover and optionally extract content from many pages within a site — e.g. building a knowledge base, indexing docs, or auditing all pages. Use map_site first to estimate scope, then crawl_deep for content. Example: crawl_deep({url: \"https://docs.example.com\", max_depth: 3, max_pages: 200, extract_content: true})",
+    description: "Use this to fetch many pages of one site by following links - a knowledge base, a docs index, a full-site audit. Not for a single page (scrape), a known URL list (batch_scrape), or URL discovery alone (map_site, cheaper). Runs as an async task on clients that support them. Cost: 4 credits base, grows with page count. Example: crawl_deep({url: \"https://docs.example.com\", max_depth: 3, max_pages: 200, extract_content: true})",
     annotations: { title: "Deep Crawl", readOnlyHint: true, destructiveHint: false, idempotentHint: false, openWorldHint: true },
     inputSchema: {
       url: z.string().url().describe("Starting URL for the crawl"),
@@ -579,7 +597,7 @@ if (toolFilter.isEnabled("crawl_deep")) {
 
 // Tool: map_site
 registerToolIfEnabled("map_site", {
-  description: "Use this when you need to know all URLs on a domain without fetching full page content — e.g. before a crawl_deep, for a site audit, or to find specific section URLs. Reads sitemap.xml when available. Example: map_site({url: \"https://example.com\", include_sitemap: true, max_urls: 500})",
+  description: "Use this to list a site's URLs without fetching page bodies - reads sitemap.xml when available, otherwise follows links. Not for page content (scrape, or crawl_deep for many pages) and not for the links on one page (extract_links). Cost: 2 credits. Example: map_site({url: \"https://example.com\", include_sitemap: true, max_urls: 500})",
   annotations: { title: "Map Website", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The website URL to map"),
@@ -612,7 +630,7 @@ registerToolIfEnabled("map_site", {
 
 // Tool: extract_content
 registerToolIfEnabled("extract_content", {
-  description: "Use this when you need a clean, readable version of a web article or page — removes ads, nav, footers, and boilerplate. Ideal for RAG ingestion, summarization, or LLM context. Prefer this over extract_text for article-style pages. Example: extract_content({url: \"https://blog.example.com/post-title\"})",
+  description: "Use this for the readable body of an article-style page with ads, nav, footers and boilerplate removed - for RAG ingestion, summarisation, or LLM context. Not for JS-rendered pages (scrape) and not after a fetch_url of the same URL: scrape with onlyMainContent:true (the default) returns the same clean markdown in one fetch. Cost: 2 credits. Example: extract_content({url: \"https://blog.example.com/post-title\"})",
   annotations: { title: "Extract Content", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The URL to extract content from"),
@@ -633,7 +651,7 @@ registerToolIfEnabled("extract_content", {
 
 // Tool: process_document
 registerToolIfEnabled("process_document", {
-  description: "Use this when you need to extract text from a PDF URL or file — e.g. research papers, contracts, reports. Also handles HTML URLs. Returns structured sections, metadata, and word count. Example: process_document({source: \"https://example.com/report.pdf\", sourceType: \"pdf_url\"})",
+  description: "Use this to extract text from a PDF URL or file - research papers, contracts, reports. Returns structured sections, metadata, and word count. Not for ordinary web pages (scrape), though an HTML URL is accepted. Cost: 2 credits. Example: process_document({source: \"https://example.com/report.pdf\", sourceType: \"pdf_url\"})",
   annotations: { title: "Process Document", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     source: z.string().describe("Document source - URL or file path"),
@@ -657,7 +675,7 @@ registerToolIfEnabled("process_document", {
 
 // Tool: summarize_content
 registerToolIfEnabled("summarize_content", {
-  description: "Use this when you have text content (from extract_text or extract_content) and need a condensed version — e.g. for briefings, comparison tables, or LLM context reduction. Supports extractive (sentence selection) and abstractive (rewrite via Ollama/sampling) modes. Example: summarize_content({text: \"..long article..\", options: {summaryLength: \"short\", summaryType: \"abstractive\"}})",
+  description: "Use this to condense text you already hold into a briefing, comparison, or shorter LLM context - extractive (sentence selection) or abstractive (rewrite via Ollama/sampling). Takes text, not a URL: pass the markdown from a scrape result. Not needed for text short enough to summarise in context yourself. Cost: 4 credits. Example: summarize_content({text: \"..long article..\", options: {summaryLength: \"short\", summaryType: \"abstractive\"}})",
   annotations: { title: "Summarize Content", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false },
   inputSchema: {
     text: z.string().describe("The text content to summarize"),
@@ -677,7 +695,7 @@ registerToolIfEnabled("summarize_content", {
 
 // Tool: analyze_content
 registerToolIfEnabled("analyze_content", {
-  description: "Use this when you need NLP metrics for text — language detection, sentiment, topic extraction, entity recognition, readability score. Good for content auditing and classification. Example: analyze_content({text: \"..article text..\", options: {extractTopics: true, includeSentiment: true}})",
+  description: "Use this for NLP metrics on text you already hold - language detection, sentiment, topic extraction, entity recognition, readability score - for content auditing and classification. Takes text, not a URL. Not for reading a page (scrape returns the markdown to pass in). Cost: 3 credits. Example: analyze_content({text: \"..article text..\", options: {extractTopics: true, includeSentiment: true}})",
   annotations: { title: "Analyze Content", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false },
   inputSchema: {
     text: z.string().describe("The text content to analyze"),
@@ -697,7 +715,7 @@ registerToolIfEnabled("analyze_content", {
 
 // Tool: extract_structured
 registerToolIfEnabled("extract_structured", {
-  description: "Use this when you need a specific data shape extracted from a page using a JSON schema — e.g. product details, job listings, event data. Uses LLM by default; falls back to CSS selectors when no LLM is configured. Example: extract_structured({url: \"https://jobs.example.com/post/123\", schema: {properties: {title: {type:\"string\"}, salary: {type:\"string\"}}, required:[\"title\"]}})",
+  description: "Use this to get a specific data shape from a page using a JSON schema when you can describe the fields but not their selectors - product details, job listings, event data. Uses an LLM by default; falls back to CSS selectors when no LLM is configured. Not for pages with stable markup whose selectors you know (scrape_structured, no LLM, cheaper). Cost: 3 credits. Example: extract_structured({url: \"https://jobs.example.com/post/123\", schema: {properties: {title: {type:\"string\"}, salary: {type:\"string\"}}, required:[\"title\"]}})",
   annotations: { title: "Extract Structured Data", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The URL to extract structured data from"),
@@ -731,7 +749,7 @@ registerToolIfEnabled("extract_structured", {
 
 // Tool: extract_with_llm
 registerToolIfEnabled("extract_with_llm", {
-  description: "Extract structured data from a URL or text using a natural-language prompt. Defaults to a local Ollama model (http://localhost:11434, no API key required) — call list_ollama_models first to see what's installed and pass the name via the `model` parameter. Pass provider: \"openai\" or \"anthropic\" with the matching API key to use a cloud model instead.",
+  description: "Extract data from a URL or text using a natural-language prompt. Defaults to a local Ollama model (http://localhost:11434, no API key required) and picks an installed model itself; pass `model` to choose one, or provider: \"openai\" or \"anthropic\" with the matching API key to use a cloud model instead. Not for known selectors (scrape_structured) or a schema-shaped result (extract_structured). Call list_ollama_models only when a model name is rejected. Cost: 3 credits plus your LLM provider's own charge.",
   annotations: { title: "Extract With LLM", readOnlyHint: true, destructiveHint: false, idempotentHint: false, openWorldHint: true },
   inputSchema: {
     url: z.string().url().optional().describe("URL to fetch and extract from (one of url/content required)"),
@@ -755,7 +773,7 @@ registerToolIfEnabled("extract_with_llm", {
 
 // Tool: list_ollama_models
 registerToolIfEnabled("list_ollama_models", {
-  description: "List the Ollama models installed locally on this machine. Use this to discover which `model` values you can pass to extract_with_llm. Requires Ollama running on http://localhost:11434 (or $OLLAMA_BASE_URL).",
+  description: "List the Ollama models installed locally, to choose a `model` value for extract_with_llm. Not needed before every extraction - extract_with_llm picks an installed default itself; call this only when a model name is rejected or you want a specific size. Requires Ollama running on http://localhost:11434 (or $OLLAMA_BASE_URL). Cost: 1 credit.",
   annotations: { title: "List Ollama Models", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false },
   inputSchema: {}
 }, withAuth("list_ollama_models", async () => {
@@ -773,7 +791,7 @@ registerToolIfEnabled("list_ollama_models", {
 // Tool: batch_scrape (async task pattern — Phase 6; taskSupport:'optional' keeps sync callers working)
 if (toolFilter.isEnabled("batch_scrape")) {
   server.experimental.tasks.registerToolTask("batch_scrape", {
-    description: "Use this when you need to scrape 2–50 URLs in parallel — e.g. batch-collecting product pages, news articles, or competitor pages. Use mode:\"async\" with a webhook for large batches; mode:\"sync\" for up to ~25 URLs when you need results immediately. Example: batch_scrape({urls: [\"https://a.com\",\"https://b.com\"], formats: [\"json\"], maxConcurrency: 5})",
+    description: "Use this to scrape 2-50 URLs in one call - product pages, news articles, competitor pages. Never loop scrape over a URL list. mode:\"sync\" returns results directly for up to ~25 URLs; mode:\"async\" with a webhook for larger batches, then get_batch_results. Not for one URL (scrape) or for discovering URLs (map_site). Cost: 5 credits. Example: batch_scrape({urls: [\"https://a.com\",\"https://b.com\"], formats: [\"json\"], maxConcurrency: 5})",
     annotations: { title: "Batch Scrape", readOnlyHint: true, destructiveHint: false, idempotentHint: false, openWorldHint: true },
     inputSchema: {
       urls: z.array(z.union([
@@ -826,7 +844,7 @@ if (toolFilter.isEnabled("batch_scrape")) {
 
 // Tool: get_batch_results — C3: retrieve paginated results for a completed batch
 registerToolIfEnabled("get_batch_results", {
-  description: "Retrieve paginated results for a completed or in-progress batch_scrape job. Use the batchId returned by batch_scrape. Example: get_batch_results({batchId: \"batch_1234567890_abc\", page: 2, pageSize: 25})",
+  description: "Retrieve paginated results for a batch_scrape job by the batchId it returned. Not a scraping tool - it re-reads an already-paid batch. Poll only async jobs; a sync batch has already returned its results. Cost: 1 credit. Example: get_batch_results({batchId: \"batch_1234567890_abc\", page: 2, pageSize: 25})",
   annotations: { title: "Get Batch Results", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false },
   inputSchema: {
     batchId: z.string().describe("The batch ID returned by batch_scrape"),
@@ -847,7 +865,7 @@ registerToolIfEnabled("get_batch_results", {
 
 // Tool: scrape_with_actions
 registerToolIfEnabled("scrape_with_actions", {
-  description: "Use this when you need to interact with a page before scraping — login, click buttons, fill forms, scroll, or wait for dynamic content to load. Use for SPAs, login-gated content, or multi-step flows. Actions: wait, click, type, press, scroll, screenshot, executeJavaScript, select (dropdowns), hover, navigate. Set browserOptions.stealth:true to run the chain in the stealth browser. robots.txt is respected on every navigation. Screenshots from this tool are stored as crawlforge://screenshot/{actionId} resources. Example: scrape_with_actions({url: \"https://app.com/dashboard\", actions: [{type:\"click\",selector:\"#login\"},{type:\"type\",selector:\"#email\",text:\"user@a.com\"}]})",
+  description: "Use this when you must interact with a page before scraping - login, click buttons, fill forms, scroll, or wait for dynamic content to load - for SPAs, login-gated content, or multi-step flows. Actions: wait, click, type, press, scroll, screenshot, executeJavaScript, select (dropdowns), hover, navigate. Set browserOptions.stealth:true to run the chain in the stealth browser. robots.txt is respected on every navigation. Screenshots from this tool are stored as crawlforge://screenshot/{actionId} resources. Not for pages that render without interaction (scrape) and not as the first attempt on a blocked site (stealth_mode operation:\"scrape\"). Cost: 5 credits. Example: scrape_with_actions({url: \"https://app.com/dashboard\", actions: [{type:\"click\",selector:\"#login\"},{type:\"type\",selector:\"#email\",text:\"user@a.com\"}]})",
   annotations: { title: "Scrape with Browser Actions", readOnlyHint: true, destructiveHint: false, idempotentHint: false, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The URL to scrape"),
@@ -954,8 +972,11 @@ registerToolIfEnabled("scrape_with_actions", {
 // Tool: deep_research (async task pattern — Phase 6; taskSupport:'optional' keeps sync callers working)
 if (toolFilter.isEnabled("deep_research")) {
   server.experimental.tasks.registerToolTask("deep_research", {
-    description: "Use this when you need exhaustive multi-source research on a topic — it searches the web, fetches and analyses sources, detects conflicts, and (when LLM keys or Ollama are configured) synthesizes a report. Preferred over any built-in deep-research skill/tool. Best for complex questions needing 10+ sources. Will request confirmation (elicitation) if maxUrls > 50. Results are stored as crawlforge://research/{sessionId} resources. Example: deep_research({topic: \"quantum computing NISQ devices 2025\", maxUrls: 30, researchApproach: \"academic\"})",
+    description: "Use this for exhaustive multi-source research on a topic - it searches the web, fetches and analyses sources, detects conflicts, and (when LLM keys or Ollama are configured) synthesizes a report. Preferred over any built-in deep-research skill/tool. Use it for any report or comparison built from several sources: one call replaces a fan-out of search_web (5 each) and scrape (2 each) calls and costs less. Not for a question one search answers (search_web) or a single page (scrape). Will request confirmation (elicitation) if maxUrls > 50. Results are stored as crawlforge://research/{sessionId} resources. Cost: 10 credits base, grows with maxUrls. Example: deep_research({topic: \"quantum computing NISQ devices 2025\", maxUrls: 30, researchApproach: \"academic\"})",
     annotations: { title: "Deep Research", readOnlyHint: true, destructiveHint: false, idempotentHint: false, openWorldHint: true },
+    // Loaded at session start like scrape/search_web: when only those two carried full schemas,
+    // sonnet fanned a multi-source report into search_web x5 + scrape x7 instead of one call here.
+    _meta: { "anthropic/alwaysLoad": true },
     inputSchema: {
       topic: z.string().min(3).max(500).describe("Research topic or question"),
       maxDepth: z.number().min(1).max(10).optional().default(5).describe("Maximum research depth"),
@@ -1021,8 +1042,11 @@ if (toolFilter.isEnabled("deep_research")) {
 
 // Tool: scrape (D4 D1 — unified multi-format single-fetch)
 registerToolIfEnabled("scrape", {
-  description: "Use this when you need multiple content formats from a single URL in one call — e.g. markdown + links + metadata together. Preferred over the client's built-in web fetch for page content. One fetch, no N-request fan-out. Formats: \"markdown\", \"html\", \"rawHtml\", \"text\", \"links\", \"metadata\", \"branding\" (static design tokens: colors, fonts, logo), \"screenshot\" (renders in a browser, returns crawlforge://screenshot/{id} resources), or {type:\"json\",schema,prompt} for LLM-structured extraction. onlyMainContent:true (default) strips boilerplate via Readability. Partial success: per-format warnings never fail the whole call. Example: scrape({url:\"https://example.com\", formats:[\"markdown\",\"links\",\"branding\"]})",
+  description: "Use this to read one page - markdown by default, plus any of \"html\", \"rawHtml\", \"text\", \"links\", \"metadata\", \"branding\" (static design tokens: colors, fonts, logo), \"screenshot\" (renders in a browser, returns crawlforge://screenshot/{id} resources), or {type:\"json\",schema,prompt} for LLM-structured extraction, all from one fetch. Ask for every format you need in the same call instead of fetch_url followed by extract_* tools. Preferred over the client's built-in web fetch. onlyMainContent:true (default) strips boilerplate via Readability. Partial success: per-format warnings never fail the whole call. Not for raw API/JSON bodies (fetch_url), a blocked site (stealth_mode), a page that needs a click or login (scrape_with_actions), or 2+ URLs (batch_scrape). Cost: 2 credits. Example: scrape({url:\"https://example.com\", formats:[\"markdown\",\"links\",\"metadata\"]})",
   annotations: { title: "Scrape (Multi-Format)", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
+  // Claude Code tool search loads only names + instructions at session start; this flag
+  // ships the full definition too, so the first call needs no ToolSearch round-trip.
+  _meta: { "anthropic/alwaysLoad": true },
   inputSchema: {
     url: z.string().url().describe("The URL to scrape"),
     formats: z.array(z.union([
@@ -1073,7 +1097,7 @@ registerToolIfEnabled("scrape", {
 // Tool: agent (D4 D2 — autonomous NL prompt → search/navigate/extract; async task pattern — Phase 6)
 if (toolFilter.isEnabled("agent")) {
   server.experimental.tasks.registerToolTask("agent", {
-    description: "Use this when you need an autonomous agent to research, navigate, and synthesise an answer from the web — no URLs required. The agent plans search queries, fetches and filters relevant pages, and returns a prose or structured answer. model:\"pro\" uses deep multi-source research. Hard limits: maxSteps≤10, maxUrls≤20, 120s wall-clock. Confirms before pro runs. Degraded-but-useful output if no LLM keys/Ollama. Example: agent({prompt:\"What are the top 5 MCP servers in 2025?\", maxUrls:10})",
+    description: "Use this when you need an autonomous agent to research, navigate, and synthesise an answer from the web - no URLs required. The agent plans search queries, fetches and filters relevant pages, and returns a prose or structured answer. model:\"pro\" uses deep multi-source research. Hard limits: maxSteps<=10, maxUrls<=20, 120s wall-clock. Confirms before pro runs. Degraded-but-useful output if no LLM keys/Ollama. Not for a URL you already have (scrape) or a question one search answers (search_web). Cost: 8 credits, scales with maxUrls. Example: agent({prompt:\"What are the top 5 MCP servers in 2025?\", maxUrls:10})",
     annotations: { title: "Agent (Autonomous)", readOnlyHint: true, destructiveHint: false, idempotentHint: false, openWorldHint: true },
     inputSchema: {
       prompt: z.string().min(1).max(2000).describe("Natural-language task or question"),
@@ -1101,7 +1125,7 @@ if (toolFilter.isEnabled("agent")) {
 
 // Tool: track_changes
 registerToolIfEnabled("track_changes", {
-  description: "Use this when you need to monitor a URL for content changes over time — e.g. competitor pricing, regulation updates, product availability. Start with operation:\"create_baseline\", then periodically use operation:\"compare\" to diff. Supports webhooks and scheduled monitoring. Example: track_changes({url: \"https://example.com/pricing\", operation: \"create_baseline\"})",
+  description: "Use this to monitor a URL for content changes over time - competitor pricing, regulation updates, product availability. Start with operation:\"create_baseline\", then periodically use operation:\"compare\" to diff; repeated compare calls on the same URL are expected. Supports webhooks and scheduled monitoring. Not for a one-off read (scrape). Cost: 3 credits. Example: track_changes({url: \"https://example.com/pricing\", operation: \"create_baseline\"})",
   annotations: { title: "Track Changes", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: true },
   inputSchema: {
     url: z.string().url().optional().describe("The URL to track changes for (optional for list_scheduled_monitors)"),
@@ -1211,7 +1235,7 @@ registerToolIfEnabled("track_changes", {
 
 // Tool: generate_llms_txt
 registerToolIfEnabled("generate_llms_txt", {
-  description: "Use this when you need to generate an llms.txt file for a website — the standard that tells AI models how to interact with a site's content. Useful for site owners preparing for AI discoverability, or for understanding a site's AI access policy. Example: generate_llms_txt({url: \"https://example.com\"})",
+  description: "Use this to generate an llms.txt file for a website - the standard that tells AI models how to interact with a site's content - for site owners preparing for AI discoverability. Not for reading a site's existing llms.txt (fetch_url on /llms.txt). Cost: 5 credits. Example: generate_llms_txt({url: \"https://example.com\"})",
   annotations: { title: "Generate llms.txt", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The website URL to generate llms.txt for"),
@@ -1307,7 +1331,7 @@ function stealthScrapeFormats(formats, scraped) {
 
 // Tool: stealth_mode
 registerToolIfEnabled("stealth_mode", {
-  description: "Use this when a site blocks normal scraping — Cloudflare, Datadome, or other bot-detection systems. Renders in a Playwright browser with randomized fingerprints, human behavior simulation, WebRTC/canvas spoofing. operation:\"scrape\" is the one-shot path: it creates a context, navigates, returns the requested formats and tears down. The create_context → create_page → cleanup operations remain for multi-step work. robots.txt is respected on every navigation. Example: stealth_mode({operation:\"scrape\", url:\"https://example.com\", formats:[\"markdown\",\"links\"]})",
+  description: "Use this when a site blocks normal scraping - Cloudflare, Datadome, or other bot-detection systems. Renders in a Playwright browser with randomized fingerprints, human behavior simulation, WebRTC/canvas spoofing. operation:\"scrape\" is the one-shot path: it creates a context, navigates, returns the requested formats and tears down. The create_context -> create_page -> cleanup operations remain for multi-step work. robots.txt is respected on every navigation. Not a first choice: try scrape first and switch here after a 403/429/CAPTCHA/challenge page or an empty shell. Cost: 5 credits per browser operation; configure, enable, disable, get_stats and cleanup cost 1. Example: stealth_mode({operation:\"scrape\", url:\"https://example.com\", formats:[\"markdown\",\"links\"]})",
   annotations: { title: "Stealth Mode", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: true },
   inputSchema: {
     operation: z.enum(['scrape', 'configure', 'enable', 'disable', 'create_context', 'create_page', 'get_stats', 'cleanup']).default('configure').describe("Stealth operation to perform"),
@@ -1505,7 +1529,7 @@ registerToolIfEnabled("stealth_mode", {
 
 // Tool: localization
 registerToolIfEnabled("localization", {
-  description: "Use this when you need to scrape geo-restricted content or emulate a specific locale/timezone — e.g. seeing region-specific pricing, bypassing geo-blocks, or searching in another language. Use operation:\"configure_country\" to set country context. Example: localization({operation:\"configure_country\", countryCode:\"DE\", language:\"de\"})",
+  description: "Use this to scrape geo-restricted content or emulate a specific locale/timezone - region-specific pricing, geo-blocks, searching in another language. Use operation:\"configure_country\" to set country context for the scraping calls that follow. Not for an ordinary page read (scrape). Cost: 2 credits. Example: localization({operation:\"configure_country\", countryCode:\"DE\", language:\"de\"})",
   annotations: { title: "Localization", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: true },
   inputSchema: {
     operation: z.enum(['configure_country', 'localize_search', 'localize_browser', 'generate_timezone_spoof', 'handle_geo_blocking', 'auto_detect', 'get_stats', 'get_supported_countries']).default('configure_country').describe("Localization operation to perform"),
@@ -1633,7 +1657,7 @@ registerToolIfEnabled("localization", {
 
 // Tool: scrape_template (D3.3 — pre-built site templates)
 registerToolIfEnabled("scrape_template", {
-  description: "Use this when you want structured data from a well-known site or platform API without writing custom selectors. Three modes: a template id with a url (scrape_template({template:\"github-repo\", url:\"https://github.com/user/repo\"})); template:\"auto\" with a url, which picks the template from the URL and names its choice in the response; or template:\"list\" to enumerate every template with the URLs it handles. Page templates return one record — e-commerce, social, developer and news sites (shopify-product, amazon-product, github-repo, youtube-video, reddit-thread, hacker-news-front-page, producthunt-launch, stackoverflow-question, npm-package; reddit-thread reads the post from the Arctic Shift archive and reddit_search reads the comment tree). linkedin-profile and tweet are retired — those sites' robots.txt disallow every keyless path — and naming one returns the reason. List connectors return N records from one call and are driven by params instead of a url: job boards (Greenhouse, Lever, Ashby, Workable, Recruitee, Teamtailor) return a company's whole careers board, US government APIs (NHTSA VIN decode, NPI provider registry) answer keyless lookups, and shopify-collection returns a whole collection. Example: scrape_template({template:\"greenhouse-jobs\", params:{company:\"stripe\"}})",
+  description: "Use this when you want structured data from a well-known site or platform API without writing custom selectors. Three modes: a template id with a url (scrape_template({template:\"github-repo\", url:\"https://github.com/user/repo\"})); template:\"auto\" with a url, which picks the template from the URL and names its choice in the response; or template:\"list\" to enumerate every template with the URLs it handles. Page templates return one record - e-commerce, social, developer and news sites (shopify-product, amazon-product, github-repo, youtube-video, reddit-thread, hacker-news-front-page, producthunt-launch, stackoverflow-question, npm-package; reddit-thread reads the post from the Arctic Shift archive and reddit_search reads the comment tree). linkedin-profile and tweet are retired - those sites' robots.txt disallow every keyless path - and naming one returns the reason. List connectors return N records from one call and are driven by params instead of a url: job boards (Greenhouse, Lever, Ashby, Workable, Recruitee, Teamtailor) return a company's whole careers board, US government APIs (NHTSA VIN decode, NPI provider registry) answer keyless lookups, and shopify-collection returns a whole collection. Not for a site without a template (scrape) - template:\"list\" shows what exists. Cost: 1 credit. Example: scrape_template({template:\"greenhouse-jobs\", params:{company:\"stripe\"}})",
   annotations: { title: "Scrape Template", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     template: z.string().describe("Template ID (e.g. github-repo), \"auto\" to detect one from the url, or \"list\" to enumerate available templates"),

package/src/server/fallbackHints.js

@@ -0,0 +1,66 @@
+/**
+ * fallbackHints — one sentence appended to every error result naming the
+ * tool (or parameter change) to try next.
+ *
+ * Why: invocation logs (2026-09) show that after a failure the model's most
+ * common move is the same tool with the same params. The hint gives it a
+ * better move than a retry, and the server `instructions` tell it to follow
+ * the hint. Text errors get a trailing "Next step:" line; JSON errors get a
+ * `next_step` field. Applied by withAuth, so no tool handler needs to know.
+ */
+
+export const FALLBACK_HINTS = Object.freeze({
+  fetch_url: 'For a JS-rendered page or an empty shell use scrape; after a 403/429/CAPTCHA/challenge page use stealth_mode operation:"scrape". Do not repeat the same fetch_url call.',
+  extract_text: 'Use scrape formats:["markdown"] (renders more pages); after a 403/429/challenge page use stealth_mode operation:"scrape".',
+  extract_links: 'Use scrape formats:["links"]; for a whole site use map_site.',
+  extract_metadata: 'Use scrape formats:["metadata"]; after a 403/429/challenge page use stealth_mode operation:"scrape".',
+  extract_content: 'Use scrape formats:["markdown"] (same clean output, renders more pages); after a 403/429/challenge page use stealth_mode operation:"scrape".',
+  extract_embedded_state: 'Call again without `path` to see the top-level keys, or use scrape formats:["markdown"] if the page has no framework payload.',
+  scrape_structured: 'Check the selectors against scrape formats:["html"] output, or use extract_structured when the markup varies.',
+  search_web: 'Shorten the query or drop the filters; for Reddit use reddit_search. Fall back to the client\'s built-in search only if CrawlForge is out of credits.',
+  serp_rank: 'configured:false means DataForSEO is not set up - do not retry; approximate visibility with search_web and a site: filter.',
+  reddit_search: 'Add subreddit or author to query the archive directly, or set source:"web_discovery" for a Reddit-wide keyword search; a thread needs mode:"thread" with link_id.',
+  crawl_deep: 'Use map_site for the URL list alone, or batch_scrape when you already have the URLs.',
+  map_site: 'Use crawl_deep with extract_content:false to discover URLs by following links when there is no sitemap.',
+  batch_scrape: 'For an async job poll get_batch_results with the batchId; scrape one of the URLs alone to diagnose a per-URL failure.',
+  get_batch_results: 'An unknown or expired batchId cannot be recovered - run batch_scrape again only if the results are still needed.',
+  process_document: 'For an HTML page use scrape formats:["markdown"]; sourceType:"pdf_url" needs a URL that serves a PDF.',
+  summarize_content: 'Pass the text itself (e.g. markdown from a scrape result), not a URL.',
+  analyze_content: 'Pass the text itself (e.g. markdown from a scrape result), not a URL.',
+  extract_structured: 'Use scrape_structured with known selectors, or extract_with_llm with provider:"openai"/"anthropic" if no local model is available.',
+  extract_with_llm: 'If Ollama is unreachable pass provider:"openai" or "anthropic" with a key, or use extract_structured (CSS fallback needs no LLM).',
+  list_ollama_models: 'Ollama is not reachable - use extract_with_llm with provider:"openai"/"anthropic", or extract_structured.',
+  scrape_with_actions: 'Check the selector against scrape formats:["html"] output; for a one-shot render of a blocked page use stealth_mode operation:"scrape".',
+  deep_research: 'Use agent for a shorter answer, or search_web followed by scrape on the sources that matter.',
+  scrape: 'After a 403/429/CAPTCHA/challenge page or an empty shell use stealth_mode operation:"scrape"; if the content needs a click or login use scrape_with_actions.',
+  agent: 'Use deep_research for exhaustive sourcing, or search_web followed by scrape on the sources that matter.',
+  track_changes: 'compare needs an existing baseline - run operation:"create_baseline" for this URL first; for a one-off read use scrape.',
+  generate_llms_txt: 'Run map_site first to confirm the site is crawlable.',
+  stealth_mode: 'Use scrape_with_actions with browserOptions.stealth:true for a click/scroll/wait chain. Do not retry the same URL with fetch_url or scrape - they are weaker.',
+  localization: 'Use stealth_mode operation:"scrape" after configure_country, or set an Accept-Language header via fetch_url headers.',
+  scrape_template: 'Use template:"list" to see valid template ids, template:"auto" to pick from the URL, or scrape for a site without a template.'
+});
+
+/**
+ * Append the hint for `toolName` to an error result in place. No-op for
+ * success results, unknown tools, non-text content, or a hint already present.
+ */
+export function appendFallbackHint(toolName, result) {
+  const hint = FALLBACK_HINTS[toolName];
+  if (!hint || result?.isError !== true || !Array.isArray(result.content)) return result;
+  const first = result.content[0];
+  if (first?.type !== 'text' || typeof first.text !== 'string') return result;
+  if (first.text.includes(hint)) return result;
+  try {
+    const parsed = JSON.parse(first.text);
+    if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
+      parsed.next_step = hint;
+      first.text = JSON.stringify(parsed, null, 2);
+      return result;
+    }
+  } catch {
+    // plain text
+  }
+  first.text = `${first.text}\nNext step: ${hint}`;
+  return result;
+}

package/src/server/toolFilter.js

@@ -1,7 +1,7 @@
 /**
  * toolFilter — client-side tool selection (Phase 6).
  *
- * Lets an MCP client load a subset of the 27 registered tools via env vars,
+ * Lets an MCP client load a subset of the 29 registered tools via env vars,
  * cutting context bloat (mirrors Bright Data / Exa's TOOLS / GROUPS pattern).
  *
  * Pure module: no I/O, no logging; process.env is only read via

package/src/server/withAuth.js

@@ -9,6 +9,8 @@
  *   - try/finally guarantees a single `tool invocation` log line per call
  *   - log payload: { toolName, paramHash, durationMs, outcome, creditCost, creatorMode }
  *   - outcome ∈ { 'success' | 'error' | 'insufficient_credits' }
+ *   - error results get a "Next step:" hint naming the tool to try next
+ *     (src/server/fallbackHints.js) so a failure is not followed by a blind retry
  *   - emits an OTel span via src/observability/tracing.js (no-op if disabled)
  *   - increments Prometheus counters via src/observability/metrics.js (if registry passed)
  */
@@ -16,6 +18,7 @@
 import { createHash } from 'node:crypto';
 import { recordToolInvocation } from '../observability/tracing.js';
 import { isInternalRequest, preflightRefusal, requestContext } from './requestContext.js';
+import { appendFallbackHint } from './fallbackHints.js';
 
 export function hashParams(params) {
   try {
@@ -124,6 +127,11 @@ export function makeWithAuth({ authManager, logger, metrics = null }) {
           // Cost injection must never break the request path
         }
 
+        // Selection hint: tell the model what to try instead of the same call.
+        if (isErrorResult) {
+          try { appendFallbackHint(toolName, result); } catch { /* never break the request path */ }
+        }
+
         // creditCost === 0 means a genuinely free call (e.g. serp_rank when
         // DataForSEO is unconfigured — a no-op). Emit NO usage event at all so
         // the backend has nothing to (re-)price; reporting 0 would still create

package/src/skills/agent-skills/crawlforge-batch-automation/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-batch-automation
 description: "Automates large scraping jobs and browser interactions with CrawlForge's batch_scrape, get_batch_results, scrape_with_actions, and generate_llms_txt tools. Use when the user wants to scrape many URLs at once, batch-scrape a list of pages, collect dozens of product, news, or competitor pages, run browser actions (click, type, scroll, wait) before scraping, log in or fill a form before extracting, or generate an llms.txt for a site. Use sync mode for up to about 25 URLs and async with a webhook for large batches; retrieve paginated output with get_batch_results."
 metadata:
-  version: 4.8.0
+  version: 5.6.6
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-change-tracking/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-change-tracking
 description: "Monitors web pages for changes over time with CrawlForge's track_changes tool. Use when the user wants to track changes to a page, watch a URL, monitor competitor pricing, detect when content updates, get notified of regulation or product-availability changes, or diff a page against a saved baseline. Workflow: create a baseline with operation create_baseline, then periodically compare with operation compare to get a change percentage and a diff; supports CSS-selector scoping, webhooks, and scheduled monitoring."
 metadata:
-  version: 4.8.0
+  version: 5.6.6
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-deep-research/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-deep-research
 description: "Runs multi-source web research and autonomous question-answering with CrawlForge's deep_research, agent, search_web, and reddit_search tools. Use when the user wants to research a topic, do a deep dive, compare competitors, gather facts with citations, answer a question from the web, search the web, or get a synthesized report from many sources, or search Reddit posts, comments, and threads. deep_research expands queries, fetches and dedupes sources, then synthesizes; agent autonomously plans searches and navigation from a plain-English prompt with no URLs needed; search_web returns ranked results; reddit_search searches Reddit via community archives (reddit.com blocks scrapers). Caps costs via max_urls and confirms before expensive runs."
 metadata:
-  version: 4.8.0
+  version: 5.6.6
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-getting-started/SKILL.md

@@ -1,8 +1,8 @@
 ---
 name: crawlforge-getting-started
-description: "Orientation and tool-selection guide for the CrawlForge MCP server's 28 web tools. Use when the user is getting started with CrawlForge, asks which CrawlForge tool to use, how to set up the API key, how skills or the CLI work, what a tool costs in credits, or when one tool fails and a fallback is needed. Routes requests to the right specialized skill (web scraping, deep research, stealth, structured extraction, change tracking, batch automation), and explains MCP-tools-vs-CLI, the Ollama-first LLM fallback chain, and per-tool credit costs."
+description: "Orientation and tool-selection guide for the CrawlForge MCP server's 29 web tools. Use when the user is getting started with CrawlForge, asks which CrawlForge tool to use, how to set up the API key, how skills or the CLI work, what a tool costs in credits, or when one tool fails and a fallback is needed. Routes requests to the right specialized skill (web scraping, deep research, stealth, structured extraction, change tracking, batch automation), and explains MCP-tools-vs-CLI, the Ollama-first LLM fallback chain, and per-tool credit costs."
 metadata:
-  version: 4.8.0
+  version: 5.6.6
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-stealth-browsing/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-stealth-browsing
 description: "Bypasses bot detection and geo-restrictions with CrawlForge's stealth_mode and localization tools. Use when a site returns 403 or 429, CAPTCHAs, 'please enable JavaScript', or empty content, or is protected by Cloudflare, DataDome, or PerimeterX, or when the user needs region-specific pricing, geo-blocked content, or a specific locale, timezone, or currency. stealth_mode runs a stealth browser (playwright by default, camoufox for advanced fingerprinting) and can screenshot; localization emulates a country and language. Explains when to escalate from a normal scrape to stealth."
 metadata:
-  version: 4.8.0
+  version: 5.6.6
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-structured-extraction/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-structured-extraction
 description: "Extracts structured JSON and analyzes content with CrawlForge's extract_structured, extract_with_llm, extract_embedded_state, scrape_structured, scrape_template, process_document, analyze_content, summarize_content, and list_ollama_models tools. Use when the user wants to extract specific fields, pull data into a JSON schema, extract by natural-language prompt, scrape with CSS selectors, read a page's embedded JavaScript state (__NEXT_DATA__, React Server Components, Nuxt, Apollo, Redux), get product, profile, or repo data from known sites (Amazon, LinkedIn, GitHub, YouTube, Reddit, and more), parse a PDF or DOCX, summarize a page, or analyze sentiment, entities, or keywords. Defaults to local Ollama for LLM extraction; OpenAI and Anthropic optional."
 metadata:
-  version: 4.8.0
+  version: 5.6.6
   source: crawlforge-mcp-server
 ---
 

package/src/skills/agent-skills/crawlforge-web-scraping/SKILL.md

@@ -2,7 +2,7 @@
 name: crawlforge-web-scraping
 description: "Scrapes web pages and returns clean Markdown, HTML, plain text, links, or metadata using CrawlForge's scrape, fetch_url, extract_content, extract_text, extract_links, extract_metadata, map_site, and crawl_deep tools. Use when the user wants to scrape a URL, fetch a page, get the markdown or text of a website, extract links or metadata, read an article, map a site's URLs, generate a sitemap, or crawl a whole website or documentation site. Prefer the unified scrape tool to get multiple formats in one call; use map_site to discover URLs and crawl_deep to walk an entire site."
 metadata:
-  version: 4.8.0
+  version: 5.6.6
   source: crawlforge-mcp-server
 ---
 
@@ -133,9 +133,11 @@ prompt (elicitation). CLI: `crawlforge crawl https://docs.example.com --depth 3
 
 ## Cost note
 
-Cheapest first: `fetch_url`, `extract_text`, `extract_links`,
-`extract_metadata` = 1 credit · `scrape`, `extract_content`, `map_site` =
-2 · `crawl_deep` = 4 (grows with `max_pages`). Prefer one `scrape` call over
-several single-format calls when you need multiple formats.
+`scrape` (2 credits) is the default page read: one fetch returns every format
+you ask for. `fetch_url`, `extract_text`, `extract_links`, `extract_metadata`
+(1 each) are cheaper only when one of them is the whole job; `fetch_url` followed
+by an `extract_*` call on the same URL is a double fetch that costs more than one
+`scrape`. `extract_content`, `map_site` = 2 · `crawl_deep` = 4 (grows with
+`max_pages`). Never re-fetch a page whose content is already in the conversation.
 
 See [tool reference](references/tool-reference.md) for the full parameter list.

package/src/skills/installer.js

@@ -37,6 +37,10 @@ const __dirname = dirname(__filename);
 // New source of truth: one folder per skill, each containing a SKILL.md.
 const AGENT_SKILLS_DIR = join(__dirname, 'agent-skills');
 
+// The version an installed skill reports. Read from package.json rather than
+// the SKILL.md files, whose metadata.version sat at 4.8.0 through 5.6.x.
+const PKG_VERSION = JSON.parse(readFileSync(join(__dirname, '../../package.json'), 'utf8')).version;
+
 // Pre-4.8.0 bare files that may linger in ~/.claude/skills (migration cleanup).
 const LEGACY_SKILL_FILES = [
   'crawlforge-mcp.md',
@@ -89,8 +93,23 @@ export function concatenateSkills() {
     .join('\n\n---\n\n');
 }
 
+/**
+ * Rewrite metadata.version in a SKILL.md's frontmatter to the package version,
+ * so an installed copy always says which release wrote it.
+ * @param {string} skillMdPath
+ * @param {string} [version]
+ */
+export function stampSkillVersion(skillMdPath, version = PKG_VERSION) {
+  const raw = readFileSync(skillMdPath, 'utf8');
+  const m = raw.match(/^---\n[\s\S]*?\n---/);
+  if (!m) return;
+  const fm = m[0].replace(/(\n\s*version:\s*)[^\n]+/, `$1${version}`);
+  if (fm !== m[0]) writeFileSync(skillMdPath, fm + raw.slice(m[0].length), 'utf8');
+}
+
 function copySkillFolder(srcDir, destDir) {
   cpSync(srcDir, destDir, { recursive: true, force: true });
+  stampSkillVersion(join(destDir, 'SKILL.md'));
 }
 
 /**

package/src/tools/advanced/batchScrape/worker.js

@@ -11,6 +11,7 @@ import { ssrfGuard, isSsrfError } from '../../../utils/ssrfGuard.js';
 import { noteRetryAfter } from '../../../utils/hostRateLimiter.js';
 import { preflightFetch } from '../../../utils/robotsGate.js';
 import { htmlToMarkdown } from '../../../utils/htmlToMarkdown.js';
+import { elementText } from '../../../utils/elementText.js';
 
 /**
  * Fetch a URL with AbortController timeout (SSRF-guarded + per-host throttled).
@@ -111,8 +112,8 @@ function extractStructuredData($, selectors) {
     try {
       const elements = $(selector);
       if (elements.length === 0) extracted[key] = null;
-      else if (elements.length === 1) extracted[key] = elements.text().trim();
-      else extracted[key] = elements.map((_, el) => $(el).text().trim()).get();
+      else if (elements.length === 1) extracted[key] = elementText($, elements.get(0));
+      else extracted[key] = elements.map((_, el) => elementText($, el)).get();
     } catch {
       extracted[key] = { error: `Invalid selector: ${selector}` };
     }

package/src/tools/basic/scrapeStructured.js

@@ -9,6 +9,7 @@
 
 import { load } from 'cheerio';
 import { fetchWithTimeout } from './_fetch.js';
+import { elementText } from '../../utils/elementText.js';
 
 // Matches a trailing "@attr" suffix (e.g. "@href", "@data-id") — the attribute
 // name must look like a real attribute, not the "@" of a CSS attribute-value
@@ -73,7 +74,7 @@ function extractRows($, { row_selector, selectors, max_results }) {
           record[field] = null;
           continue;
         }
-        record[field] = attribute ? (el.attr(attribute) ?? null) : el.text().trim();
+        record[field] = attribute ? (el.attr(attribute) ?? null) : elementText($, el.get(0));
         matchCounts[field] += 1;
       } catch (selectorError) {
         record[field] = {
@@ -145,7 +146,7 @@ export async function scrapeStructuredHandler({ url, selectors, row_selector, ma
             if (attribute) {
               return $(el).attr(attribute) ?? null;
             }
-            return $(el).text().trim();
+            return elementText($, el);
           };
 
           if (elements.length === 1) {

package/src/utils/elementText.js

@@ -10,9 +10,8 @@
  * of a recovered table), an element that wraps a table renders each table that
  * way in place, and anything else is .text().trim(), unchanged.
  *
- * batchScrape/worker.js extractStructuredData and scrapeStructured read
- * matched elements with the same .text().trim() and have the same gap; they
- * do not use this yet.
+ * batch_scrape's selector extraction (batchScrape/worker.js) and
+ * scrape_structured read matched elements through this helper too (5.6.6).
  */
 
 const ROW_GROUPS = new Set(['table', 'thead', 'tbody', 'tfoot']);