crawlforge-mcp-server 5.3.1 → 5.4.1

package/README.md

@@ -3,7 +3,7 @@
 </p>
 
 <p align="center">
-  <b>28 web scraping, crawling, deep-research &amp; autonomous-extraction tools for Claude, Cursor &amp; any MCP client.</b><br>
+  <b>29 web scraping, crawling, deep-research &amp; autonomous-extraction tools for Claude, Cursor &amp; any MCP client.</b><br>
   Clean Markdown &amp; structured JSON from any site. Get started with <b>1,000 free credits</b> — no credit card required.
 </p>
 
@@ -35,7 +35,7 @@
 
 ## 🎯 Why CrawlForge?
 
-- **28 MCP-native tools** — scraping, crawling, search, real Google SERP rank tracking, deep research, an autonomous `agent`, a unified multi-format `scrape`, document processing, stealth browsing, and more, callable directly from your AI assistant.
+- **29 MCP-native tools** — scraping, crawling, search, real Google SERP rank tracking, deep research, an autonomous `agent`, a unified multi-format `scrape`, document processing, stealth browsing, and more, callable directly from your AI assistant.
 - **Generous free tier** — 1,000 credits to start instantly, no credit card. The grant is one-time rather than monthly, and the credits never expire.
 - **Local-LLM by default** — `extract_with_llm` runs against a local **Ollama** model out of the box: no LLM API key, no per-token cost, and your data never leaves your machine. Cloud (OpenAI/Anthropic) is opt-in.
 - **LLM-ready output** — clean Markdown, structured JSON (schema-driven), screenshots, links, and metadata from a single fetch.
@@ -47,7 +47,7 @@
 
 | | **CrawlForge MCP** | Firecrawl | Raw scraping API |
 |---|:---:|:---:|:---:|
-| Native MCP server | ✅ 28 tools | ✅ | ❌ |
+| Native MCP server | ✅ 29 tools | ✅ | ❌ |
 | Free tier | ✅ 1,000 credits, rollover | Limited | Varies |
 | Self-hosted / local LLM extraction (Ollama) | ✅ default, $0/token | ❌ | ❌ |
 | Autonomous agent (no URLs needed) | ✅ `agent` | ✅ | ❌ |
@@ -180,6 +180,7 @@ CrawlForge requires a CrawlForge API key — **every tool is metered and consume
 | `get_batch_results` | 1 | Retrieve paginated results for a `batch_scrape` job by `batchId` |
 | `scrape` | 2 | **Unified single-fetch, multi-format extraction.** Pass a `formats` array (markdown/html/rawHtml/text/links/metadata/screenshot/json-schema) plus `onlyMainContent`; one fetch serves every requested format with per-format partial-success warnings |
 | `scrape_structured` | 2 | Extract structured data with CSS selectors |
+| `extract_embedded_state` | 2 | Read a page's embedded JavaScript state — `__NEXT_DATA__`, React Server Component payloads, Nuxt, Apollo, Redux, `<script type="application/json">` — with a `path` to scope the result. No LLM in the extraction path |
 | `extract_content` | 2 | Enhanced content extraction |
 | `map_site` | 2 | Discover and map website structure (optional `search=` ranks the discovered URLs) |
 | `process_document` | 2 | Multi-format document processing |
@@ -216,7 +217,7 @@ For the full canonical capabilities reference (all tools, CLI commands, stealth
 | **Business** ($399) | 250,000 / month | Large scale operations |
 
 **All plans include:**
-- Access to all 28 tools
+- Access to all 29 tools
 - Credits never expire; paid-plan credits roll over month to month
 - API access and webhook notifications
 

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.3.1",
+  "version": "5.4.1",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
-  "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 28 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
+  "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 29 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
   "bin": {
     "crawlforge": "src/cli/index.js",
@@ -114,7 +114,7 @@
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",
-    "crawlforge-extractors": "^1.2.3",
+    "crawlforge-extractors": "^1.4.0",
     "diff": "^9.0.0",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",

package/server.js

@@ -52,6 +52,7 @@ import { extractTextHandler } from "./src/tools/basic/extractText.js";
 import { extractLinksHandler } from "./src/tools/basic/extractLinks.js";
 import { extractMetadataHandler } from "./src/tools/basic/extractMetadata.js";
 import { scrapeStructuredHandler } from "./src/tools/basic/scrapeStructured.js";
+import { extractEmbeddedStateHandler } from "./src/tools/extract/extractEmbeddedState.js";
 // D1.1 Resources + D1.2 Prompts + D1.4 Elicitation
 import { ResourceRegistry } from "./src/resources/ResourceRegistry.js";
 import { PROMPTS, getPromptMessages } from "./src/prompts/PromptRegistry.js";
@@ -105,7 +106,7 @@ const taskStore = createTaskStore({ logger });
 const server = new McpServer({
   name: "crawlforge",
   version: "5.3.1",
-  description: "Production-ready MCP server with 28 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
+  description: "Production-ready MCP server with 29 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
   icons: [{ src: "https://www.crawlforge.dev/icon.png", mimeType: "image/png", sizes: ["any"] }],
@@ -141,7 +142,7 @@ server.registerPrompt("getting-started", {
       role: "user",
       content: {
         type: "text",
-        text: "You have access to CrawlForge MCP with 28 web scraping tools. Key tools:\n\n" +
+        text: "You have access to CrawlForge MCP with 29 web scraping tools. Key tools:\n\n" +
           "- fetch_url: Fetch raw HTML/content from any URL\n" +
           "- extract_text: Extract clean text from a webpage\n" +
           "- extract_content: Smart content extraction with readability\n" +
@@ -322,6 +323,11 @@ const COMPLIANCE_PARAMS = {
   user_agent: z.string().optional().describe("Override the outbound User-Agent. CrawlForge identifies itself honestly by default; use this only for targets you have your own agreement with.")
 };
 
+// 3.4: the two tools that let an LLM produce values share one provenance control.
+const VERIFY_NUMBERS_PARAM = {
+  verify_numbers: z.boolean().optional().default(true).describe("Numeric provenance guard (default: true): every price or numeric value the LLM returns must appear literally in the page source, else it is returned as null with a reason in `provenance.unverified`. Set false to get the model's raw numbers back, including ones it derived (a count, a sum, a total) rather than read off the page.")
+};
+
 
 // Tool: fetch_url
 registerToolIfEnabled("fetch_url", {
@@ -366,10 +372,22 @@ registerToolIfEnabled("extract_metadata", {
   annotations: { title: "Extract Metadata", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The URL to extract metadata from"),
+    json_ld_types: z.array(z.string()).optional().describe("Filter the returned JSON-LD to nodes of these schema.org types, e.g. [\"Product\",\"Offer\"]. Subtypes match their parent: \"Event\" returns MusicEvent, \"Offer\" returns AggregateOffer, \"ItemList\" returns BreadcrumbList. Nodes are found at any depth, including inside @graph and nested inside a parent node. When set, json_ld carries only the matching nodes instead of the raw dump, and json_ld_type_counts reports how many matched per requested type. Documented types: ItemList, Product, Offer, Event, JobPosting, RealEstateListing — any other schema.org type is matched exactly."),
     ...COMPLIANCE_PARAMS
   }
 }, withAuth("extract_metadata", extractMetadataHandler));
 
+// Tool: extract_embedded_state
+registerToolIfEnabled("extract_embedded_state", {
+  description: "Use this when a page's data lives in its embedded JavaScript state rather than its rendered HTML — Next.js (__NEXT_DATA__ and React Server Component payloads), Nuxt, Apollo, Redux (__INITIAL_STATE__, __PRELOADED_STATE__), and <script type=\"application/json\"> blocks. One fetch, exact values, no LLM in the extraction path, so nothing can be fabricated. Payloads are routinely over a megabyte — pass `path` to return one subtree instead of the whole blob. Example: extract_embedded_state({url: \"https://www.ticketmaster.com/discover/concerts\", path: \"next_data.props.pageProps\"})",
+  annotations: { title: "Extract Embedded State", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
+  inputSchema: {
+    url: z.string().url().describe("The URL to read embedded state from"),
+    path: z.string().optional().describe("Return only this subtree instead of the whole payload. Dotted keys and array indexes, e.g. \"next_data.props.pageProps\" or \"next_f[0].f\" — not JSONPath (no wildcards, filters or recursion). State payloads are routinely over a megabyte; scope them."),
+    ...COMPLIANCE_PARAMS
+  }
+}, withAuth("extract_embedded_state", extractEmbeddedStateHandler));
+
 // Tool: scrape_structured
 registerToolIfEnabled("scrape_structured", {
   description: "Use this when you know the exact CSS selectors for the data you want — e.g. scraping a pricing table or product list with consistent markup. More reliable than LLM extraction for well-structured pages. By default each selector is matched independently across the whole page, so the returned arrays are NOT row-aligned: data.price[0] need not belong to the same row as data.name[0]. Pass row_selector to get aligned records instead — one object per row, null for a field the row lacks. Example: scrape_structured({url: \"https://shop.com/products\", row_selector: \".product-card\", selectors: {price: \".price\", name: \".product-title\"}})",
@@ -579,12 +597,12 @@ registerToolIfEnabled("map_site", {
     ...COMPLIANCE_PARAMS
   },
   outputSchema: OUTPUT_SCHEMAS.map_site
-}, withAuth("map_site", async ({ url, include_sitemap, max_urls, group_by_path, include_metadata, domain_filter, import_filter_config, search }) => {
+}, withAuth("map_site", async (params) => {
   try {
-    if (!url) {
+    if (!params.url) {
       return { content: [{ type: "text", text: "URL parameter is required" }], isError: true };
     }
-    const result = await mapSiteTool.execute({ url, include_sitemap, max_urls, group_by_path, include_metadata, domain_filter, import_filter_config, search });
+    const result = await mapSiteTool.execute(params);
     return dualOutput(result);
   } catch (error) {
     return { content: [{ type: "text", text: `Site mapping failed: ${error.message}` }], isError: true };
@@ -600,12 +618,12 @@ registerToolIfEnabled("extract_content", {
     options: z.object({}).passthrough().optional().describe("Additional extraction options"),
     ...COMPLIANCE_PARAMS
   }
-}, withAuth("extract_content", async ({ url, options }) => {
+}, withAuth("extract_content", async (params) => {
   try {
-    if (!url) {
+    if (!params.url) {
       return { content: [{ type: "text", text: "URL parameter is required" }], isError: true };
     }
-    const result = await extractContentTool.execute({ url, options });
+    const result = await extractContentTool.execute(params);
     return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
   } catch (error) {
     return { content: [{ type: "text", text: `Content extraction failed: ${error.message}` }], isError: true };
@@ -624,12 +642,12 @@ registerToolIfEnabled("process_document", {
     options: z.object({}).passthrough().optional().describe("Additional processing options (maxPages, pageRange:{start,end}, extractText, extractMetadata, outputFormat, ...)"),
     ...COMPLIANCE_PARAMS
   }
-}, withAuth("process_document", async ({ source, sourceType, options }) => {
+}, withAuth("process_document", async (params) => {
   try {
-    if (!source) {
+    if (!params.source) {
       return { content: [{ type: "text", text: "Source parameter is required" }], isError: true };
     }
-    const result = await processDocumentTool.execute({ source, sourceType, options });
+    const result = await processDocumentTool.execute(params);
     return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
   } catch (error) {
     return { content: [{ type: "text", text: `Document processing failed: ${error.message}` }], isError: true };
@@ -694,12 +712,16 @@ registerToolIfEnabled("extract_structured", {
     }).optional().describe("LLM provider configuration for AI-powered extraction"),
     fallbackToSelectors: z.boolean().optional().default(true).describe("Fall back to CSS selector extraction if LLM is unavailable"),
     selectorHints: z.record(z.string()).optional().describe("CSS selector hints to guide extraction"),
-    ...COMPLIANCE_PARAMS
+    ...COMPLIANCE_PARAMS,
+    ...VERIFY_NUMBERS_PARAM
   },
   outputSchema: OUTPUT_SCHEMAS.extract_structured
-}, withAuth("extract_structured", async ({ url, schema, prompt, llmConfig, fallbackToSelectors, selectorHints }) => {
+}, withAuth("extract_structured", async (params) => {
   try {
-    const result = await extractStructuredTool.execute({ url, schema, prompt, llmConfig, fallbackToSelectors, selectorHints });
+    // Forward params whole. This wrapper used to destructure a fixed six, which
+    // silently dropped respect_robots and user_agent — both declared here and
+    // read by the tool, so the G5 override was accepted and ignored.
+    const result = await extractStructuredTool.execute(params);
     return dualOutput(result);
   } catch (error) {
     return { content: [{ type: "text", text: `Structured extraction failed: ${error.message}` }], isError: true };
@@ -718,7 +740,8 @@ registerToolIfEnabled("extract_with_llm", {
     provider: z.enum(["openai", "anthropic", "ollama", "auto"]).optional().default("auto").describe("LLM provider. Defaults to 'ollama' (local, no key, http://localhost:11434). Use 'openai' or 'anthropic' for cloud models (requires the matching API key)."),
     model: z.string().optional().describe("Override the model. For ollama, pass a name returned by list_ollama_models (e.g. 'llama3.2', 'qwen2.5:7b'). Defaults: openai='gpt-4o-mini', anthropic='claude-haiku-4-5-20251001', ollama='llama3.2' or $OLLAMA_DEFAULT_MODEL."),
     maxTokens: z.number().optional().default(4096).describe("Maximum output tokens"),
-    ...COMPLIANCE_PARAMS
+    ...COMPLIANCE_PARAMS,
+    ...VERIFY_NUMBERS_PARAM
   }
 }, withAuth("extract_with_llm", async (params) => {
   try {
@@ -1555,11 +1578,12 @@ registerToolIfEnabled("localization", {
 
 // Tool: scrape_template (D3.3 — pre-built site templates)
 registerToolIfEnabled("scrape_template", {
-  description: "Use this when you want structured data from a well-known site without writing custom selectors. Pass template:\"list\" to see all available templates. Supports: shopify-product (any Shopify storefront, read from the store's own /products/<handle>.json rather than the rendered page), amazon-product, linkedin-profile, github-repo, youtube-video, tweet, reddit-thread, hacker-news-front-page, producthunt-launch, stackoverflow-question, npm-package (read from the npm registry API rather than the npmjs.com page, which blocks plain fetches). Example: scrape_template({template:\"github-repo\", url:\"https://github.com/user/repo\"})",
+  description: "Use this when you want structured data from a well-known site or platform API without writing custom selectors. Three modes: a template id with a url (scrape_template({template:\"github-repo\", url:\"https://github.com/user/repo\"})); template:\"auto\" with a url, which picks the template from the URL and names its choice in the response; or template:\"list\" to enumerate every template with the URLs it handles. Page templates return one record — e-commerce, social, developer and news sites (shopify-product, amazon-product, github-repo, youtube-video, tweet, reddit-thread, hacker-news-front-page, producthunt-launch, stackoverflow-question, npm-package, linkedin-profile). List connectors return N records from one call and are driven by params instead of a url: job boards (Greenhouse, Lever, Ashby, Workable, Recruitee, Teamtailor) return a company's whole careers board, US government APIs (NHTSA VIN decode, NPI provider registry) answer keyless lookups, and shopify-collection returns a whole collection. Example: scrape_template({template:\"greenhouse-jobs\", params:{company:\"stripe\"}})",
   annotations: { title: "Scrape Template", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
-    template: z.string().describe("Template ID (e.g. github-repo) or list to enumerate available templates"),
-    url: z.string().url().optional().describe("URL to scrape — required unless template is list"),
+    template: z.string().describe("Template ID (e.g. github-repo), \"auto\" to detect one from the url, or \"list\" to enumerate available templates"),
+    url: z.string().url().optional().describe("URL to scrape — required unless template is list, or params drive a list connector"),
+    params: z.record(z.any()).optional().describe("Parameters for a list connector, e.g. {company:\"stripe\"} for greenhouse-jobs or {store:\"www.allbirds.com\", collection:\"mens\"} for shopify-collection. Use template:\"list\" to see which templates take params"),
     timeout: z.number().min(5000).max(60000).optional().default(15000).describe("Request timeout in milliseconds"),
     ...COMPLIANCE_PARAMS
   }

package/src/cli/commands/template.js

@@ -8,8 +8,9 @@ import { runTool } from '../lib/runTool.js';
 export function register(program) {
   program
     .command('template [id] [target]')
-    .description('Scrape using a pre-built site template (e.g. amazon-product, github-repo)')
+    .description('Scrape using a pre-built site template (e.g. amazon-product, github-repo, or auto to detect one from the URL)')
     .option('--list', 'List all available templates')
+    .option('--params <json>', 'JSON parameters for a list connector, e.g. \'{"company":"stripe"}\'')
     .action(async (id, target, opts, cmd) => {
       const globals = cmd.parent.opts();
       const cliFlags = { json: globals.json, pretty: globals.pretty, quiet: globals.quiet };
@@ -21,11 +22,21 @@ export function register(program) {
         return;
       }
 
-      if (!id || !target) {
-        process.stderr.write('Error: template requires <id> and <target>, or use --list\n');
+      let params;
+      if (opts.params) {
+        try {
+          params = JSON.parse(opts.params);
+        } catch (e) {
+          process.stderr.write(`Error parsing --params JSON: ${e.message}\n`);
+          process.exit(1);
+        }
+      }
+
+      if (!id || (!target && !params)) {
+        process.stderr.write('Error: template requires <id> and <target>, or <id> with --params, or use --list\n');
         process.exit(1);
       }
 
-      await runTool(tool, { template: id, url: target }, cliFlags);
+      await runTool(tool, { template: id, url: target, params }, cliFlags);
     });
 }

package/src/core/AuthManager.js

@@ -562,6 +562,7 @@ class AuthManager {
       process_document: 2,
       localization: 2,
       scrape: 2,
+      extract_embedded_state: 2,
       reddit_search: 5, // a Reddit-wide search spends a web search to discover posts, same as search_web
 
       // 3 credits

package/src/schemas/toolOutputSchemas.js

@@ -259,6 +259,17 @@ const extractStructuredShape = {
     errors: z.array(z.string()).optional()
   }).passthrough().optional(),
   extractionNotes: z.array(z.string()).optional(),
+  provenance: z.object({
+    enabled: z.boolean().optional().describe('Whether the numeric provenance guard ran'),
+    verified: z.number().optional().describe('Numeric values found literally in the page source'),
+    nulled: z.number().optional().describe('Numeric values replaced with null because the source does not contain them'),
+    unverified: z.array(z.object({
+      path: z.string().optional().describe('Path to the field, e.g. configurations[2].price'),
+      value: z.unknown().optional().describe('The value that was removed'),
+      reason: z.string().optional().describe('"not_found_in_source"')
+    }).passthrough()).optional(),
+    skipped: z.string().optional().describe('"empty_source" when there was nothing to check against')
+  }).passthrough().optional(),
   _cost: costShape
 };
 

package/src/skills/agent-skills/crawlforge-getting-started/SKILL.md

@@ -8,7 +8,7 @@ metadata:
 
 # CrawlForge: Getting Started
 
-CrawlForge is an MCP server with **28 tools** for web scraping, crawling,
+CrawlForge is an MCP server with **29 tools** for web scraping, crawling,
 extraction, research, change tracking, and AI-compliance. This skill orients you
 and routes each request to the right specialized skill.
 
@@ -52,13 +52,13 @@ stored at `~/.crawlforge/config.json`.
 | Watch a page for changes / monitor pricing | **crawlforge-change-tracking** |
 | Scrape many URLs, run browser actions, generate llms.txt | **crawlforge-batch-automation** |
 
-## The 28 tools at a glance
+## The 29 tools at a glance
 
 - **Basic (5):** fetch_url, extract_text, extract_links, extract_metadata, scrape_structured
 - **Unified (1):** scrape (multi-format single fetch)
 - **Search & research (5):** search_web, serp_rank, reddit_search, deep_research, agent
 - **Crawl (2):** crawl_deep, map_site
-- **Extract & analyze (7):** extract_content, process_document, summarize_content, analyze_content, extract_structured, extract_with_llm, list_ollama_models
+- **Extract & analyze (8):** extract_content, process_document, summarize_content, analyze_content, extract_structured, extract_with_llm, extract_embedded_state, list_ollama_models
 - **Batch & automation (4):** batch_scrape, get_batch_results, scrape_with_actions, generate_llms_txt
 - **Stealth & locale (2):** stealth_mode, localization
 - **Templates & tracking (2):** scrape_template, track_changes

package/src/skills/agent-skills/crawlforge-getting-started/references/credits.md

@@ -21,6 +21,7 @@ metered; there is no free tier. Tools marked "scales" cost more as work grows.
 |------|-------|
 | `scrape` | Unified multi-format single fetch. |
 | `scrape_structured` | CSS-selector extraction. |
+| `extract_embedded_state` | Embedded JS state (`__NEXT_DATA__`, RSC, Nuxt, Apollo, Redux). |
 | `extract_content` | Readability-cleaned article. |
 | `map_site` | URL discovery / sitemap. |
 | `process_document` | PDF / DOCX / TXT parsing. |

package/src/skills/agent-skills/crawlforge-structured-extraction/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: crawlforge-structured-extraction
-description: "Extracts structured JSON and analyzes content with CrawlForge's extract_structured, extract_with_llm, scrape_structured, scrape_template, process_document, analyze_content, summarize_content, and list_ollama_models tools. Use when the user wants to extract specific fields, pull data into a JSON schema, extract by natural-language prompt, scrape with CSS selectors, get product, profile, or repo data from known sites (Amazon, LinkedIn, GitHub, YouTube, Reddit, and more), parse a PDF or DOCX, summarize a page, or analyze sentiment, entities, or keywords. Defaults to local Ollama for LLM extraction; OpenAI and Anthropic optional."
+description: "Extracts structured JSON and analyzes content with CrawlForge's extract_structured, extract_with_llm, extract_embedded_state, scrape_structured, scrape_template, process_document, analyze_content, summarize_content, and list_ollama_models tools. Use when the user wants to extract specific fields, pull data into a JSON schema, extract by natural-language prompt, scrape with CSS selectors, read a page's embedded JavaScript state (__NEXT_DATA__, React Server Components, Nuxt, Apollo, Redux), get product, profile, or repo data from known sites (Amazon, LinkedIn, GitHub, YouTube, Reddit, and more), parse a PDF or DOCX, summarize a page, or analyze sentiment, entities, or keywords. Defaults to local Ollama for LLM extraction; OpenAI and Anthropic optional."
 metadata:
   version: 4.8.0
   source: crawlforge-mcp-server
@@ -17,6 +17,7 @@ extraction method by how predictable the page is and whether an LLM is needed.
 |-----------------|------|------|
 | A well-known site (Amazon, GitHub, LinkedIn...) | `scrape_template` | 1 |
 | Exact CSS selectors for the fields | `scrape_structured` | 2 |
+| The data is in the page's JS state, not its HTML | `extract_embedded_state` | 2 |
 | A JSON schema to fill (LLM, CSS fallback) | `extract_structured` | 3 |
 | A natural-language extraction instruction | `extract_with_llm` | 3 |
 | A PDF / DOCX / TXT to parse | `process_document` | 2 |
@@ -24,8 +25,26 @@ extraction method by how predictable the page is and whether an LLM is needed.
 | Sentiment / entities / keywords / readability | `analyze_content` | 3 |
 | To list local LLMs available for extraction | `list_ollama_models` | 1 |
 
-Cheapest-first rule: try `scrape_template` → `scrape_structured` (deterministic)
-before reaching for the LLM tools.
+Cheapest-first rule: try `scrape_template` → `scrape_structured` /
+`extract_embedded_state` (all deterministic) before reaching for the LLM tools.
+On a React/Next/Nuxt page the values are usually sitting in the embedded state
+already, exact and typed — that beats asking a model to read them off the render.
+
+## extract_embedded_state — the page's own JS state (cost: 2)
+
+```json
+{ "tool": "extract_embedded_state", "params": { "url": "https://www.ticketmaster.com/discover/concerts", "path": "next_data.props.pageProps" } }
+```
+
+Finds `__NEXT_DATA__`, `self.__next_f` (React Server Component payloads),
+`window.__NUXT__`, `__APOLLO_STATE__`, `__INITIAL_STATE__`, `__PRELOADED_STATE__`
+and `<script type="application/json">` blocks, keyed by source name. No LLM in
+the path, so values are exact rather than inferred.
+
+These payloads are routinely over a megabyte — pass `path` (dotted keys and
+array indexes, e.g. `next_data.props.pageProps` or `next_f[0].f`) to return one
+subtree. Without it, a large result comes back with a warning naming the biggest
+source and a ready-to-paste path.
 
 ## scrape_template — known sites, zero selectors (cost: 1)
 

package/src/tools/basic/extractMetadata.js

@@ -2,28 +2,12 @@
  * extract_metadata — Extract page metadata (title, description, OG tags, etc.).
  * Extracted from server.js inline handler.
  * B1: Parse JSON-LD and microdata; stronger title fallback chain (og:title → <title> → h1).
+ * 3.3: json_ld_types promotes JSON-LD from a raw dump to a filtered extraction path.
  */
 
 import { load } from 'cheerio';
 import { fetchWithTimeout } from './_fetch.js';
-
-/**
- * Parse all JSON-LD blocks from the document.
- * @param {import('cheerio').CheerioAPI} $
- * @returns {Array}
- */
-function parseJsonLd($) {
-  const results = [];
-  $('script[type="application/ld+json"]').each((_, el) => {
-    try {
-      const raw = $(el).html();
-      if (raw) results.push(JSON.parse(raw));
-    } catch {
-      // Skip invalid blocks
-    }
-  });
-  return results;
-}
+import { parseJsonLd, filterJsonLdByType } from '../../utils/jsonLd.js';
 
 /**
  * Parse microdata items (elements with itemscope).
@@ -60,9 +44,10 @@ function parseMicrodata($) {
 }
 
 /**
- * @param {{ url: string, user_agent?: string, respect_robots?: boolean }} params
+ * @param {{ url: string, user_agent?: string, respect_robots?: boolean,
+ *   json_ld_types?: string[] }} params
  */
-export async function extractMetadataHandler({ url, user_agent, respect_robots }) {
+export async function extractMetadataHandler({ url, user_agent, respect_robots, json_ld_types }) {
   try {
     const response = await fetchWithTimeout(url, {
       userAgent: user_agent,
@@ -115,25 +100,33 @@ export async function extractMetadataHandler({ url, user_agent, respect_robots }
     const jsonLd = parseJsonLd($);
     const microdata = parseMicrodata($);
 
+    const result = {
+      title,
+      description,
+      keywords: keywords.split(',').map(k => k.trim()).filter(Boolean),
+      canonical_url: canonical,
+      author,
+      robots,
+      viewport,
+      charset,
+      og_tags: ogTags,
+      twitter_tags: twitterTags,
+      json_ld: jsonLd,
+      microdata,
+      url: response.url
+    };
+
+    // With a type filter, json_ld carries only the matching nodes — returning
+    // the raw dump as well would double the payload on the large pages that
+    // make filtering worth asking for.
+    if (json_ld_types?.length) {
+      const { items, counts } = filterJsonLdByType(jsonLd, json_ld_types);
+      result.json_ld = items;
+      result.json_ld_type_counts = counts;
+    }
+
     return {
-      content: [{
-        type: 'text',
-        text: JSON.stringify({
-          title,
-          description,
-          keywords: keywords.split(',').map(k => k.trim()).filter(Boolean),
-          canonical_url: canonical,
-          author,
-          robots,
-          viewport,
-          charset,
-          og_tags: ogTags,
-          twitter_tags: twitterTags,
-          json_ld: jsonLd,
-          microdata,
-          url: response.url
-        }, null, 2)
-      }]
+      content: [{ type: 'text', text: JSON.stringify(result, null, 2) }]
     };
   } catch (error) {
     return {

package/src/tools/extract/extractEmbeddedState.js

@@ -0,0 +1,73 @@
+/**
+ * extract_embedded_state — return the JSON state a page already ships in its
+ * own HTML: __NEXT_DATA__, RSC flight chunks (self.__next_f), __NUXT__,
+ * __APOLLO_STATE__, __INITIAL_STATE__, __PRELOADED_STATE__ and
+ * <script type="application/json"> blocks.
+ *
+ * One fetch, exact values, no LLM in the extraction path — the numbers come
+ * from the site's own serialized state, so they cannot be fabricated.
+ */
+
+import { fetchAndParse } from './_fetchAndParse.js';
+// Both live in crawlforge-extractors so the REST API's extract_embedded_state
+// runs this exact reader — one RSC flight-stream parser, not two.
+import { extractEmbeddedState, selectJsonPath } from 'crawlforge-extractors';
+
+// Above this, an unscoped result is big enough to be a problem for the caller
+// (context window, transport) rather than just large. Warn — never truncate:
+// a half-serialized object is worse than a big one, and `path` already gives
+// the caller an exact way to ask for less.
+const LARGE_RESULT_BYTES = 256_000;
+
+/**
+ * @param {{ url: string, path?: string, user_agent?: string, respect_robots?: boolean }} params
+ */
+export async function extractEmbeddedStateHandler({ url, path, user_agent, respect_robots }) {
+  try {
+    // The raw `html` is used, not `$`: fetchAndParse strips <script> from the
+    // parsed tree by default, and every source here lives in a script tag.
+    const { html, finalUrl, warnings: fetchWarnings } = await fetchAndParse(url, {
+      userAgent: user_agent,
+      respectRobots: respect_robots,
+      tool: 'extract_embedded_state'
+    });
+
+    const state = extractEmbeddedState(html);
+    const warnings = [...fetchWarnings, ...state.warnings];
+
+    if (state.found.length === 0) {
+      warnings.push(
+        'No embedded state found. The page may render entirely on the client, or ship its data in a format this tool does not read.'
+      );
+    }
+
+    const data = path ? selectJsonPath(state.data, path) : state.data;
+    const bytes = Buffer.byteLength(JSON.stringify(data) ?? '');
+
+    if (!path && bytes > LARGE_RESULT_BYTES) {
+      const largest = state.found.reduce((a, b) => (b.bytes > a.bytes ? b : a));
+      warnings.push(
+        `Result is ${bytes} bytes; "${largest.name}" alone is ${largest.bytes}. Re-run with path to scope it, e.g. path:"${largest.name}.${Object.keys(state.data[largest.name])[0]}".`
+      );
+    }
+
+    return {
+      content: [{
+        type: 'text',
+        text: JSON.stringify({
+          url: finalUrl,
+          found: state.found,
+          path: path || null,
+          bytes,
+          data,
+          warnings
+        }, null, 2)
+      }]
+    };
+  } catch (error) {
+    return {
+      content: [{ type: 'text', text: `Failed to extract embedded state: ${error.message}` }],
+      isError: true
+    };
+  }
+}

package/src/tools/extract/extractStructured.js

@@ -11,6 +11,7 @@ import { LLMManager } from '../../core/llm/LLMManager.js';
 import { CRAWLFORGE_USER_AGENT } from '../../utils/fetchIdentity.js';
 import { fetchAndParse, flattenBodyText } from './_fetchAndParse.js';
 import { extractMainContent } from '../scrape/_mainContent.js';
+import { verifyNumericProvenance } from '../../utils/provenance.js';
 
 // Semantic element selectors for well-known field names, tried as a last
 // resort in the CSS fallback so common fields (e.g. "title") still resolve when
@@ -79,7 +80,8 @@ const ExtractStructuredSchema = z.object({
   fallbackToSelectors: z.boolean().optional().default(true),
   selectorHints: z.record(z.string()).optional(),
   respect_robots: z.boolean().optional(),
-  user_agent: z.string().optional()
+  user_agent: z.string().optional(),
+  verify_numbers: z.boolean().optional().default(true)
 });
 
 export class ExtractStructuredTool {
@@ -136,7 +138,7 @@ export class ExtractStructuredTool {
 
     try {
       const validated = ExtractStructuredSchema.parse(params);
-      const { url, schema, prompt, llmConfig, fallbackToSelectors, selectorHints, respect_robots, user_agent } = validated;
+      const { url, schema, prompt, llmConfig, fallbackToSelectors, selectorHints, respect_robots, user_agent, verify_numbers } = validated;
 
       // Step 1: Fetch and parse — shared helper strips scripts/styles/iframes/svgs
       const { html, $, textContent, warnings } = await fetchAndParse(url, {
@@ -180,6 +182,43 @@ export class ExtractStructuredTool {
         llmErrorMessage = llmError.message;
       }
 
+      // Step 3b (3.4): numeric provenance. Only the LLM path invents numbers —
+      // the CSS and keyword fallbacks can only return text they read off the
+      // page — so the guard is scoped to it.
+      //
+      // It is checked against the FULL source, never `mainContentText()`: on
+      // the Apple MacBook Air page Readability keeps the FAQ block and every
+      // price is left behind in an embedded JSON blob, so checking against what
+      // the model was shown would null every correct price.
+      let provenance = { enabled: false };
+      if (extractionResult && extractionMethod === 'llm' && verify_numbers) {
+        const checked = verifyNumericProvenance(extractionResult.data || {}, `${html}\n${textContent}`);
+        // The model's own `valid` flag described the data before the guard ran.
+        // A required field the guard nulled is not filled in any more, so that
+        // flag cannot stand or the response reports a fabrication as valid.
+        const nulledRequired = checked.unverified
+          .map((entry) => entry.path)
+          .filter((path) => (schema.required || []).includes(path));
+        extractionResult = {
+          ...extractionResult,
+          data: checked.data,
+          ...(nulledRequired.length > 0 ? {
+            valid: false,
+            validationErrors: [
+              ...(extractionResult.validationErrors || []),
+              ...nulledRequired.map((field) => `Field "${field}" was not found in the page source`)
+            ]
+          } : {})
+        };
+        provenance = {
+          enabled: true,
+          verified: checked.verified,
+          nulled: checked.nulled,
+          unverified: checked.unverified
+        };
+        if (checked.skipped) provenance.skipped = checked.skipped;
+      }
+
       // Step 4: CSS selector fallback if LLM unavailable or failed
       if (!extractionResult && fallbackToSelectors !== false) {
         // D1.4: no LLM configured and the schema demands more than 3 required
@@ -223,6 +262,12 @@ export class ExtractStructuredTool {
       if (llmErrorMessage) {
         extractionNotes.push(`LLM extraction failed: ${llmErrorMessage}`);
       }
+      if (provenance.nulled > 0) {
+        extractionNotes.push(
+          `Numeric provenance: ${provenance.nulled} value(s) the model returned are not in the page source and were replaced with null: ` +
+          provenance.unverified.map((u) => `${u.path}=${JSON.stringify(u.value)}`).join(', ')
+        );
+      }
 
       // A required field that came back missing or empty is a failed
       // extraction, not a successful one carrying a note: surface it at the
@@ -249,6 +294,7 @@ export class ExtractStructuredTool {
           errors: extractionResult.validationErrors || []
         },
         extractionNotes,
+        provenance,
         ...(warnings?.length ? { warnings } : {})
       };
 

package/src/tools/extract/extractWithLlm.js

@@ -10,6 +10,7 @@
 import { z } from 'zod';
 import { fetchAndParse } from './_fetchAndParse.js';
 import { ollamaBaseUrl, ollamaHeaders, selectOllamaModel } from '../../utils/ollamaConfig.js';
+import { verifyNumericProvenance } from '../../utils/provenance.js';
 // D1.3: SamplingClient for MCP sampling fallback (lazy — only imported if needed)
 let _SamplingClient = null;
 async function getSamplingClient() {
@@ -485,6 +486,7 @@ export class ExtractWithLlm {
    * @param {number}  [params.maxTokens] - Max output tokens (default 4096)
    * @param {boolean} [params.respect_robots] - Per-request robots.txt override
    * @param {string}  [params.user_agent]     - Per-request identity override
+   * @param {boolean} [params.verify_numbers] - Numeric provenance guard (default true)
    * @returns {Promise<Object>}
    */
   async execute(params) {
@@ -497,7 +499,8 @@ export class ExtractWithLlm {
       model: modelParam,
       maxTokens = 4096,
       respect_robots,
-      user_agent
+      user_agent,
+      verify_numbers = true
     } = params;
 
     // Validate: exactly one of url or content must be provided
@@ -530,18 +533,25 @@ export class ExtractWithLlm {
 
     // Step 1: Get text to extract from
     let text;
+    // What the provenance guard checks against. Deliberately wider than what
+    // the model is shown: the raw html carries numbers the flattened text does
+    // not (Apple's prices exist only inside an embedded JSON blob), and a value
+    // missing from `text` but present on the page must not be nulled.
+    let sourceForProvenance;
     let fetchWarnings = [];
     try {
       if (url) {
-        const { textContent, warnings } = await fetchAndParse(url, {
+        const { html, textContent, warnings } = await fetchAndParse(url, {
           respectRobots: respect_robots,
           userAgent: user_agent,
           tool: 'extract_with_llm'
         });
         text = textContent;
+        sourceForProvenance = `${html}\n${textContent}`;
         fetchWarnings = warnings || [];
       } else {
         text = content;
+        sourceForProvenance = content;
       }
     } catch (fetchErr) {
       return { success: false, error: `Failed to fetch content: ${fetchErr.message}` };
@@ -654,10 +664,27 @@ export class ExtractWithLlm {
       }
     }
 
+    // 3.4: numeric provenance. A number the model wrote that is nowhere in the
+    // page it was given was invented, so it comes back null with a reason
+    // rather than as a confident answer.
+    let provenance = { enabled: false };
+    if (verify_numbers) {
+      const checked = verifyNumericProvenance(parsed, sourceForProvenance);
+      parsed = checked.data;
+      provenance = {
+        enabled: true,
+        verified: checked.verified,
+        nulled: checked.nulled,
+        unverified: checked.unverified
+      };
+      if (checked.skipped) provenance.skipped = checked.skipped;
+    }
+
     // C3: surface truncation metadata so callers know the input was clipped
     const result = {
       success: true,
       data: parsed,
+      provenance,
       provider: resolvedModel === 'sampling' ? 'sampling' : provider,
       model: resolvedModel || model,
       usage

package/src/tools/templates/ScrapeTemplateTool.js

@@ -4,44 +4,113 @@
  * Usage pattern (D3.3):
  *   const tool = new ScrapeTemplateTool();
  *   const result = await tool.execute({ template: "github-repo", url: "https://github.com/user/repo" });
+ *
+ * Three ways in: a template id, `"auto"` (the registry picks the template from
+ * the URL and the response names the one it chose), and `"list"`. A list
+ * connector is driven by `params` rather than a URL and returns N entities from
+ * one call — the registry builds the URL, this tool fetches it.
  */
 
 import { TemplateRegistry } from 'crawlforge-extractors';
 import { safeFetch } from '../../utils/ssrfGuard.js';
 import { preflightFetch } from '../../utils/robotsGate.js';
 import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
+import { markPreflightRefusal } from '../../server/requestContext.js';
+
+/**
+ * A caller mistake caught before anything is fetched: no template matches the
+ * URL, a required list parameter is missing, a key-based connector has no key.
+ * We fetched nothing, so — like a robots refusal — it costs the caller nothing.
+ */
+function badRequest(message) {
+  markPreflightRefusal('BAD_REQUEST');
+  return new Error(message);
+}
 
 export class ScrapeTemplateTool {
-  constructor() {
-    this.registry = new TemplateRegistry();
+  /**
+   * @param {{ templates?: object[] }} [config] `templates` replaces the shipped
+   *   set; tests use it to exercise a connector shape nothing ships yet.
+   */
+  constructor(config = {}) {
+    this.registry = new TemplateRegistry(config?.templates);
+  }
+
+  /** The catalogue — no network. */
+  listTemplates() {
+    const templates = this.registry.list();
+    return { templates, count: templates.length };
   }
 
   /**
    * Execute the scrape_template tool.
-   * @param {{ template: string, url: string, timeout?: number,
-   *          user_agent?: string, respect_robots?: boolean }} params
+   * @param {{ template: string, url?: string, params?: object, timeout?: number,
+   *          user_agent?: string, respect_robots?: boolean }} request
    * @returns {Promise<object>}
    */
-  async execute({ template, url, timeout = 15000, user_agent, respect_robots }) {
-    // list mode — return available templates without scraping
-    if (template === 'list' || !url) {
-      return {
-        templates: this.registry.list(),
-        count: this.registry.list().length
-      };
+  async execute({ template, url, params, timeout = 15000, user_agent, respect_robots }) {
+    if (template === 'list') return this.listTemplates();
+
+    let templateId = template;
+    if (template === 'auto') {
+      if (!url) {
+        throw badRequest(
+          'template "auto" needs a url to detect from. Pass a url, or template:"list" to see every template.'
+        );
+      }
+      const detected = this.registry.detect(url);
+      if (!detected) {
+        throw badRequest(
+          `No template matches ${url}. Pass template:"list" to see every template and the URLs ` +
+          'each one handles, or name a template explicitly.'
+        );
+      }
+      templateId = detected.id;
+    } else if (!url && !params) {
+      // A template named with nothing to run it against still lists, as before.
+      return this.listTemplates();
     }
 
     // Validate template exists before making network call
-    const tpl = this.registry.get(template);
+    const tpl = this.registry.get(templateId);
     if (!tpl) {
       const available = this.registry.list().map(t => t.id).join(', ');
-      throw new Error(`Unknown template "${template}". Available templates: ${available}`);
+      throw new Error(`Unknown template "${templateId}". Available templates: ${available}`);
+    }
+
+    // A key-based connector is answered here or not at all: the registry never
+    // reads process.env, and a missing key must not reach the target as a 401.
+    let apiKey;
+    if (tpl.requiresApiKey) {
+      apiKey = process.env[tpl.credentialRef];
+      if (!apiKey) {
+        throw badRequest(
+          `Template "${templateId}" reads an API-keyed endpoint. Set ${tpl.credentialRef} in the ` +
+          'server environment and try again.'
+        );
+      }
     }
 
-    // A template may redirect its own fetch to a machine-readable endpoint
-    // (shopify-product reads /products/<handle>.json). Same host either way,
-    // so the SSRF guard below still applies.
-    const fetchUrl = template === 'list' ? url : (tpl.resolveUrl ? tpl.resolveUrl(url) : url);
+    // The URL actually fetched comes from one of three places. The robots gate
+    // below runs against that URL, never the caller's input — listUrl in
+    // particular reaches a host the caller never named.
+    let fetchUrl;
+    if (params && tpl.listUrl) {
+      try {
+        fetchUrl = tpl.listUrl(apiKey ? { ...params, apiKey } : params);
+      } catch (error) {
+        // listUrl throws naming the parameter it wanted: a caller mistake, not
+        // a fetch that failed.
+        throw badRequest(error.message);
+      }
+    } else if (!url) {
+      throw badRequest(`Template "${templateId}" is reached by url, not params. Pass a url.`);
+    } else {
+      // A template may redirect its own fetch to a machine-readable endpoint
+      // (shopify-product reads /products/<handle>.json). Same host either way,
+      // so the SSRF guard below still applies.
+      fetchUrl = tpl.resolveUrl ? tpl.resolveUrl(url) : url;
+    }
 
     // Robots gate + per-host politeness before any request to the target.
     const gate = await preflightFetch(fetchUrl, {
@@ -76,8 +145,28 @@ export class ScrapeTemplateTool {
       throw error;
     }
 
+    // What the response reports. A params-driven call named no URL, so the one
+    // we built is the one to report — and the one the extractor quotes in its
+    // own error messages. Except on a key-based connector, where that URL
+    // carries the key: it goes into listUrl and nowhere else.
+    const keyed = Boolean(tpl.requiresApiKey);
+    const reportedUrl = url ?? (keyed ? undefined : fetchUrl);
+    const reportedFetchUrl = keyed ? reportedUrl : fetchUrl;
+
+    let echoParams;
+    if (params) {
+      echoParams = { ...params };
+      delete echoParams.apiKey;
+    }
+
     // Run the template extractor
-    const result = await this.registry.run(template, html, url, fetchUrl);
+    const result = tpl.extractList
+      ? await this.registry.runList(templateId, html, { url: reportedUrl, params: echoParams })
+      : await this.registry.run(templateId, html, reportedUrl, reportedFetchUrl);
+
+    // run() stamps fetchedUrl itself; runList() does not.
+    if (tpl.extractList && reportedFetchUrl !== reportedUrl) result.fetchedUrl = reportedFetchUrl;
+
     return gate.warnings.length > 0 ? { ...result, warnings: gate.warnings } : result;
   }
 }

package/src/utils/jsonLd.js

@@ -0,0 +1,122 @@
+/**
+ * JSON-LD parsing and schema.org type filtering.
+ *
+ * Pure: no fetching. Callers pass a loaded cheerio document (parse) or already
+ * parsed blocks (filter).
+ */
+
+/**
+ * schema.org descendants that a filter on the parent type must also match.
+ *
+ * Real pages publish the specific subtype and almost never the parent, so an
+ * exact-string @type match returns nothing on exactly the pages callers ask
+ * about: ticketmaster.com/discover/concerts emits MusicEvent (never Event),
+ * apple.com/shop/buy-mac/macbook-air emits AggregateOffer and BreadcrumbList
+ * (never Offer or ItemList). Each list is the transitive descendant set of the
+ * key in schema.org v30. Types outside this table are matched exactly.
+ */
+export const JSON_LD_SUBTYPES = {
+  ItemList: ['BreadcrumbList', 'HowToSection', 'HowToStep', 'OfferCatalog'],
+  Product: [
+    'DietarySupplement', 'Drug', 'IndividualProduct', 'ProductCollection',
+    'ProductGroup', 'ProductModel', 'SomeProducts', 'Vehicle', 'BusOrCoach',
+    'Car', 'Motorcycle', 'MotorizedBicycle'
+  ],
+  Offer: ['AggregateOffer', 'OfferForLease', 'OfferForPurchase'],
+  Event: [
+    'BusinessEvent', 'ChildrensEvent', 'ComedyEvent', 'CourseInstance',
+    'DanceEvent', 'DeliveryEvent', 'EducationEvent', 'EventSeries',
+    'ExhibitionEvent', 'Festival', 'FoodEvent', 'Hackathon', 'LiteraryEvent',
+    'MusicEvent', 'PublicationEvent', 'BroadcastEvent', 'OnDemandEvent',
+    'SaleEvent', 'ScreeningEvent', 'SocialEvent', 'SportsEvent', 'TheaterEvent',
+    'VisualArtsEvent'
+  ],
+  JobPosting: [],
+  RealEstateListing: []
+};
+
+// Lowercased filter name → set of lowercased @type values it accepts.
+const MATCH_SETS = new Map(
+  Object.entries(JSON_LD_SUBTYPES).map(([parent, subtypes]) => [
+    parent.toLowerCase(),
+    new Set([parent, ...subtypes].map((t) => t.toLowerCase()))
+  ])
+);
+
+/**
+ * Some publishers write @type as a full IRI ("https://schema.org/Product").
+ * @param {unknown} value
+ * @returns {string|null}
+ */
+function normalizeType(value) {
+  if (typeof value !== 'string') return null;
+  return value.replace(/^https?:\/\/schema\.org\//, '').trim().toLowerCase() || null;
+}
+
+/**
+ * Parse all JSON-LD blocks from the document. A malformed block is skipped so
+ * one bad block does not lose the good ones.
+ * @param {import('cheerio').CheerioAPI} $
+ * @returns {Array}
+ */
+export function parseJsonLd($) {
+  const results = [];
+  $('script[type="application/ld+json"]').each((_, el) => {
+    try {
+      const raw = $(el).html();
+      if (raw) results.push(JSON.parse(raw));
+    } catch {
+      // Skip invalid blocks
+    }
+  });
+  return results;
+}
+
+/**
+ * Collect the JSON-LD nodes matching the requested schema.org types.
+ *
+ * Nodes are found at any depth, so @graph wrappers, top-level arrays and types
+ * nested inside another node (an Offer inside an Event) are all reachable.
+ * @type itself may be a string or an array of strings.
+ *
+ * @param {Array} blocks - parsed JSON-LD blocks, as returned by parseJsonLd
+ * @param {string[]} types - schema.org type names to keep
+ * @returns {{ items: Array, counts: Record<string, number> }} matching nodes in
+ *   document order, and how many matched per requested type
+ */
+export function filterJsonLdByType(blocks, types) {
+  const wanted = types.map((requested) => ({
+    requested,
+    match: MATCH_SETS.get(requested.toLowerCase()) || new Set([requested.toLowerCase()])
+  }));
+  const counts = Object.fromEntries(types.map((t) => [t, 0]));
+  const items = [];
+
+  const visit = (node) => {
+    if (Array.isArray(node)) {
+      node.forEach(visit);
+      return;
+    }
+    if (!node || typeof node !== 'object') return;
+
+    const raw = node['@type'];
+    const nodeTypes = (Array.isArray(raw) ? raw : [raw]).map(normalizeType).filter(Boolean);
+    if (nodeTypes.length) {
+      let matched = false;
+      for (const { requested, match } of wanted) {
+        if (nodeTypes.some((t) => match.has(t))) {
+          counts[requested] += 1;
+          matched = true;
+        }
+      }
+      if (matched) items.push(node);
+    }
+
+    // Descend even into a matched node: its children may match another
+    // requested type (Ticketmaster nests each Offer inside its MusicEvent).
+    for (const value of Object.values(node)) visit(value);
+  };
+
+  blocks.forEach(visit);
+  return { items, counts };
+}

package/src/utils/provenance.js

@@ -0,0 +1,197 @@
+/**
+ * provenance -- numeric provenance guard for LLM-extracted data.
+ *
+ * A model handed page text that does not contain the number it was asked for
+ * does not say so: it writes a plausible one. On
+ * https://www.apple.com/shop/buy-mac/macbook-air every MacBook Air price lives
+ * only inside the page's embedded PRODUCT_SELECTION_BOOTSTRAP JSON — the
+ * rendered text carries no price at all — and extraction came back with 20
+ * confident, fabricated prices.
+ *
+ * The guard is the cheapest possible test of the only thing that matters: a
+ * number the model returned has to be *on the page*. Anything that is not is
+ * replaced with null and reported with a reason, so a caller can tell "the page
+ * does not say" apart from "the model said".
+ *
+ * Two rules make it safe to run by default:
+ *
+ * 1. It is checked against the FULL fetched source (raw html + flattened text),
+ *    never the trimmed main content the model was fed. Readability keeps the
+ *    FAQ block on that Apple page and drops every price; checking against what
+ *    the model saw would null 100% of correct prices — a false null destroys a
+ *    good extraction and is far more expensive than a false pass.
+ *
+ * 2. Matching is normalised on both sides, so 1299 is found in "$1,299.00",
+ *    "1.299,00", "1 299", "1299.00" and in a value split across markup. Every
+ *    ambiguous reading of a source number is admitted, because an extra reading
+ *    can only make the guard more permissive, never null a real value.
+ */
+
+/** Spaces (incl. NBSP / narrow NBSP) and the Swiss apostrophe group digits. */
+const GROUPING_CHARS = /[\s\u00a0\u202f\u2009']/g;
+
+/** A number as it appears in text: digits plus grouping/decimal punctuation. */
+const GROUPED_TOKEN = /\d[\d.,\u00a0\u202f\u2009' ]*\d|\d/g;
+
+/** Bare digit runs — recovers "1", "2", "3" from a "1, 2, 3" grouped token. */
+const DIGIT_RUN = /\d+/g;
+
+/** Currency symbols ($ £ € ¥ …) are stripped before a value is read. */
+const CURRENCY_SYMBOLS = /\p{Sc}/gu;
+
+/**
+ * A string that is a single formatted number and nothing else. Anything with a
+ * word in it ("From $999", "13-inch") is text, not a numeric field, and is left
+ * alone — the guard deliberately under-reaches rather than risk a false null.
+ */
+const NUMERIC_STRING = /^[+-]?\d+(?:[.,]\d{3})*(?:[.,]\d+)?$/;
+
+/** Chained markup between digits (`<span>1</span><span>299</span>`) is welded. */
+const MARKUP_BETWEEN_DIGITS = /(\d)(?:\s*<[^>]{0,120}>\s*)+([\d.,])/g;
+const MAX_WELD_PASSES = 3;
+
+/**
+ * Every numeric reading of one token, canonicalised.
+ *
+ * "1,299.00" -> 1299 | "1.299,00" -> 1299 | "1 299" -> 1299 | "1.299" -> both
+ * 1299 (de-DE grouping) and 1.299 (en-US decimal), since the token alone cannot
+ * settle which the page meant.
+ *
+ * @param {string} token
+ * @returns {string[]} canonical numeric strings
+ */
+function readings(token) {
+  const t = token.replace(GROUPING_CHARS, '');
+  const hasDot = t.includes('.');
+  const hasComma = t.includes(',');
+  const raw = [];
+
+  if (hasDot && hasComma) {
+    // The rightmost of the two is the decimal separator; the other groups.
+    const decimal = t.lastIndexOf('.') > t.lastIndexOf(',') ? '.' : ',';
+    const grouping = decimal === '.' ? ',' : '.';
+    raw.push(t.split(grouping).join('').replace(decimal, '.'));
+  } else if (hasDot || hasComma) {
+    const sep = hasDot ? '.' : ',';
+    raw.push(t.split(sep).join(''));                                  // grouping reading
+    if (t.split(sep).length === 2) raw.push(t.replace(sep, '.'));     // decimal reading
+  } else {
+    raw.push(t);
+  }
+
+  const out = [];
+  for (const candidate of raw) {
+    const num = Number(candidate);
+    if (Number.isFinite(num)) out.push(String(num));
+  }
+  return out;
+}
+
+/**
+ * Every number present in the source, canonicalised into a lookup set.
+ *
+ * The source is scanned twice: as given, and with markup between digits welded
+ * shut, so a price the page splits across two spans is still a number.
+ *
+ * @param {string} source - raw html and/or page text
+ * @returns {Set<string>}
+ */
+function numbersInSource(source) {
+  const variants = [source];
+  if (source.includes('<')) {
+    let welded = source;
+    for (let pass = 0; pass < MAX_WELD_PASSES; pass++) {
+      const next = welded.replace(MARKUP_BETWEEN_DIGITS, '$1$2');
+      if (next === welded) break;
+      welded = next;
+    }
+    if (welded !== source) variants.push(welded);
+  }
+
+  const found = new Set();
+  for (const variant of variants) {
+    for (const [token] of variant.matchAll(GROUPED_TOKEN)) {
+      for (const reading of readings(token)) found.add(reading);
+    }
+    for (const [run] of variant.matchAll(DIGIT_RUN)) {
+      for (const reading of readings(run)) found.add(reading);
+    }
+  }
+  return found;
+}
+
+/**
+ * The numeric readings of an extracted value, or null when the value is not a
+ * numeric field at all.
+ *
+ * A numeric field is identified by the SHAPE of the value, not the name of the
+ * field: a JS number anywhere in the result, or a string that is entirely a
+ * formatted number once currency symbols and spaces are removed. Field names
+ * are no help in either direction — a price can arrive under `mainOffer`, and a
+ * field called `price` can legitimately hold "Contact us".
+ *
+ * @param {*} value
+ * @returns {string[]|null}
+ */
+function valueReadings(value) {
+  if (typeof value === 'number') {
+    return Number.isFinite(value) ? [String(value)] : null;
+  }
+  if (typeof value !== 'string') return null;
+  const stripped = value.replace(CURRENCY_SYMBOLS, '').replace(GROUPING_CHARS, '');
+  if (!NUMERIC_STRING.test(stripped)) return null;
+  return readings(stripped.replace(/^\+/, ''));
+}
+
+/**
+ * Replace every numeric value that is not present in the source with null.
+ *
+ * Derived numbers — a count, a sum, a total the caller asked the model to
+ * compute — are not on the page and will not verify. They are nulled like any
+ * other absent number, but the value that was removed is returned in
+ * `unverified`, so nothing disappears silently: a caller that genuinely wanted
+ * a computed number can read it there, or re-run with the guard off.
+ *
+ * @param {*} data - parsed LLM output (object, array or scalar)
+ * @param {string} source - FULL fetched source, not trimmed main content
+ * @returns {{ data: *, verified: number, nulled: number,
+ *             unverified: Array<{path: string, value: *, reason: string}>,
+ *             skipped?: string }}
+ */
+export function verifyNumericProvenance(data, source) {
+  if (typeof source !== 'string' || source.trim() === '') {
+    // No source to check against. Nulling everything here would be a guess, not
+    // a finding.
+    return { data, verified: 0, nulled: 0, unverified: [], skipped: 'empty_source' };
+  }
+
+  const found = numbersInSource(source);
+  const unverified = [];
+  let verified = 0;
+
+  const walk = (node, path) => {
+    if (Array.isArray(node)) {
+      return node.map((item, i) => walk(item, `${path}[${i}]`));
+    }
+    if (node && typeof node === 'object') {
+      const out = {};
+      for (const [key, value] of Object.entries(node)) {
+        out[key] = walk(value, path ? `${path}.${key}` : key);
+      }
+      return out;
+    }
+
+    const candidates = valueReadings(node);
+    if (candidates === null) return node;
+    if (candidates.some((c) => found.has(c))) {
+      verified++;
+      return node;
+    }
+    unverified.push({ path: path || '(root)', value: node, reason: 'not_found_in_source' });
+    return null;
+  };
+
+  return { data: walk(data, ''), verified, nulled: unverified.length, unverified };
+}
+
+export default verifyNumericProvenance;