crawlforge-mcp-server 5.3.1 → 5.4.0

package/README.md

@@ -3,7 +3,7 @@
 </p>
 
 <p align="center">
-  <b>28 web scraping, crawling, deep-research &amp; autonomous-extraction tools for Claude, Cursor &amp; any MCP client.</b><br>
+  <b>29 web scraping, crawling, deep-research &amp; autonomous-extraction tools for Claude, Cursor &amp; any MCP client.</b><br>
   Clean Markdown &amp; structured JSON from any site. Get started with <b>1,000 free credits</b> — no credit card required.
 </p>
 
@@ -35,7 +35,7 @@
 
 ## 🎯 Why CrawlForge?
 
-- **28 MCP-native tools** — scraping, crawling, search, real Google SERP rank tracking, deep research, an autonomous `agent`, a unified multi-format `scrape`, document processing, stealth browsing, and more, callable directly from your AI assistant.
+- **29 MCP-native tools** — scraping, crawling, search, real Google SERP rank tracking, deep research, an autonomous `agent`, a unified multi-format `scrape`, document processing, stealth browsing, and more, callable directly from your AI assistant.
 - **Generous free tier** — 1,000 credits to start instantly, no credit card. The grant is one-time rather than monthly, and the credits never expire.
 - **Local-LLM by default** — `extract_with_llm` runs against a local **Ollama** model out of the box: no LLM API key, no per-token cost, and your data never leaves your machine. Cloud (OpenAI/Anthropic) is opt-in.
 - **LLM-ready output** — clean Markdown, structured JSON (schema-driven), screenshots, links, and metadata from a single fetch.
@@ -47,7 +47,7 @@
 
 | | **CrawlForge MCP** | Firecrawl | Raw scraping API |
 |---|:---:|:---:|:---:|
-| Native MCP server | ✅ 28 tools | ✅ | ❌ |
+| Native MCP server | ✅ 29 tools | ✅ | ❌ |
 | Free tier | ✅ 1,000 credits, rollover | Limited | Varies |
 | Self-hosted / local LLM extraction (Ollama) | ✅ default, $0/token | ❌ | ❌ |
 | Autonomous agent (no URLs needed) | ✅ `agent` | ✅ | ❌ |
@@ -180,6 +180,7 @@ CrawlForge requires a CrawlForge API key — **every tool is metered and consume
 | `get_batch_results` | 1 | Retrieve paginated results for a `batch_scrape` job by `batchId` |
 | `scrape` | 2 | **Unified single-fetch, multi-format extraction.** Pass a `formats` array (markdown/html/rawHtml/text/links/metadata/screenshot/json-schema) plus `onlyMainContent`; one fetch serves every requested format with per-format partial-success warnings |
 | `scrape_structured` | 2 | Extract structured data with CSS selectors |
+| `extract_embedded_state` | 2 | Read a page's embedded JavaScript state — `__NEXT_DATA__`, React Server Component payloads, Nuxt, Apollo, Redux, `<script type="application/json">` — with a `path` to scope the result. No LLM in the extraction path |
 | `extract_content` | 2 | Enhanced content extraction |
 | `map_site` | 2 | Discover and map website structure (optional `search=` ranks the discovered URLs) |
 | `process_document` | 2 | Multi-format document processing |
@@ -216,7 +217,7 @@ For the full canonical capabilities reference (all tools, CLI commands, stealth
 | **Business** ($399) | 250,000 / month | Large scale operations |
 
 **All plans include:**
-- Access to all 28 tools
+- Access to all 29 tools
 - Credits never expire; paid-plan credits roll over month to month
 - API access and webhook notifications
 

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.3.1",
+  "version": "5.4.0",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
-  "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 28 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
+  "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 29 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",
   "bin": {
     "crawlforge": "src/cli/index.js",
@@ -114,7 +114,7 @@
     "cheerio": "^1.1.2",
     "commander": "^14.0.3",
     "compromise": "^14.14.4",
-    "crawlforge-extractors": "^1.2.3",
+    "crawlforge-extractors": "^1.3.0",
     "diff": "^9.0.0",
     "dotenv": "^17.2.1",
     "franc": "^6.2.0",

package/server.js

@@ -52,6 +52,7 @@ import { extractTextHandler } from "./src/tools/basic/extractText.js";
 import { extractLinksHandler } from "./src/tools/basic/extractLinks.js";
 import { extractMetadataHandler } from "./src/tools/basic/extractMetadata.js";
 import { scrapeStructuredHandler } from "./src/tools/basic/scrapeStructured.js";
+import { extractEmbeddedStateHandler } from "./src/tools/extract/extractEmbeddedState.js";
 // D1.1 Resources + D1.2 Prompts + D1.4 Elicitation
 import { ResourceRegistry } from "./src/resources/ResourceRegistry.js";
 import { PROMPTS, getPromptMessages } from "./src/prompts/PromptRegistry.js";
@@ -105,7 +106,7 @@ const taskStore = createTaskStore({ logger });
 const server = new McpServer({
   name: "crawlforge",
   version: "5.3.1",
-  description: "Production-ready MCP server with 28 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
+  description: "Production-ready MCP server with 29 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
   icons: [{ src: "https://www.crawlforge.dev/icon.png", mimeType: "image/png", sizes: ["any"] }],
@@ -141,7 +142,7 @@ server.registerPrompt("getting-started", {
       role: "user",
       content: {
         type: "text",
-        text: "You have access to CrawlForge MCP with 28 web scraping tools. Key tools:\n\n" +
+        text: "You have access to CrawlForge MCP with 29 web scraping tools. Key tools:\n\n" +
           "- fetch_url: Fetch raw HTML/content from any URL\n" +
           "- extract_text: Extract clean text from a webpage\n" +
           "- extract_content: Smart content extraction with readability\n" +
@@ -322,6 +323,11 @@ const COMPLIANCE_PARAMS = {
   user_agent: z.string().optional().describe("Override the outbound User-Agent. CrawlForge identifies itself honestly by default; use this only for targets you have your own agreement with.")
 };
 
+// 3.4: the two tools that let an LLM produce values share one provenance control.
+const VERIFY_NUMBERS_PARAM = {
+  verify_numbers: z.boolean().optional().default(true).describe("Numeric provenance guard (default: true): every price or numeric value the LLM returns must appear literally in the page source, else it is returned as null with a reason in `provenance.unverified`. Set false to get the model's raw numbers back, including ones it derived (a count, a sum, a total) rather than read off the page.")
+};
+
 
 // Tool: fetch_url
 registerToolIfEnabled("fetch_url", {
@@ -366,10 +372,22 @@ registerToolIfEnabled("extract_metadata", {
   annotations: { title: "Extract Metadata", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
     url: z.string().url().describe("The URL to extract metadata from"),
+    json_ld_types: z.array(z.string()).optional().describe("Filter the returned JSON-LD to nodes of these schema.org types, e.g. [\"Product\",\"Offer\"]. Subtypes match their parent: \"Event\" returns MusicEvent, \"Offer\" returns AggregateOffer, \"ItemList\" returns BreadcrumbList. Nodes are found at any depth, including inside @graph and nested inside a parent node. When set, json_ld carries only the matching nodes instead of the raw dump, and json_ld_type_counts reports how many matched per requested type. Documented types: ItemList, Product, Offer, Event, JobPosting, RealEstateListing — any other schema.org type is matched exactly."),
     ...COMPLIANCE_PARAMS
   }
 }, withAuth("extract_metadata", extractMetadataHandler));
 
+// Tool: extract_embedded_state
+registerToolIfEnabled("extract_embedded_state", {
+  description: "Use this when a page's data lives in its embedded JavaScript state rather than its rendered HTML — Next.js (__NEXT_DATA__ and React Server Component payloads), Nuxt, Apollo, Redux (__INITIAL_STATE__, __PRELOADED_STATE__), and <script type=\"application/json\"> blocks. One fetch, exact values, no LLM in the extraction path, so nothing can be fabricated. Payloads are routinely over a megabyte — pass `path` to return one subtree instead of the whole blob. Example: extract_embedded_state({url: \"https://www.ticketmaster.com/discover/concerts\", path: \"next_data.props.pageProps\"})",
+  annotations: { title: "Extract Embedded State", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
+  inputSchema: {
+    url: z.string().url().describe("The URL to read embedded state from"),
+    path: z.string().optional().describe("Return only this subtree instead of the whole payload. Dotted keys and array indexes, e.g. \"next_data.props.pageProps\" or \"next_f[0].f\" — not JSONPath (no wildcards, filters or recursion). State payloads are routinely over a megabyte; scope them."),
+    ...COMPLIANCE_PARAMS
+  }
+}, withAuth("extract_embedded_state", extractEmbeddedStateHandler));
+
 // Tool: scrape_structured
 registerToolIfEnabled("scrape_structured", {
   description: "Use this when you know the exact CSS selectors for the data you want — e.g. scraping a pricing table or product list with consistent markup. More reliable than LLM extraction for well-structured pages. By default each selector is matched independently across the whole page, so the returned arrays are NOT row-aligned: data.price[0] need not belong to the same row as data.name[0]. Pass row_selector to get aligned records instead — one object per row, null for a field the row lacks. Example: scrape_structured({url: \"https://shop.com/products\", row_selector: \".product-card\", selectors: {price: \".price\", name: \".product-title\"}})",
@@ -579,12 +597,12 @@ registerToolIfEnabled("map_site", {
     ...COMPLIANCE_PARAMS
   },
   outputSchema: OUTPUT_SCHEMAS.map_site
-}, withAuth("map_site", async ({ url, include_sitemap, max_urls, group_by_path, include_metadata, domain_filter, import_filter_config, search }) => {
+}, withAuth("map_site", async (params) => {
   try {
-    if (!url) {
+    if (!params.url) {
       return { content: [{ type: "text", text: "URL parameter is required" }], isError: true };
     }
-    const result = await mapSiteTool.execute({ url, include_sitemap, max_urls, group_by_path, include_metadata, domain_filter, import_filter_config, search });
+    const result = await mapSiteTool.execute(params);
     return dualOutput(result);
   } catch (error) {
     return { content: [{ type: "text", text: `Site mapping failed: ${error.message}` }], isError: true };
@@ -600,12 +618,12 @@ registerToolIfEnabled("extract_content", {
     options: z.object({}).passthrough().optional().describe("Additional extraction options"),
     ...COMPLIANCE_PARAMS
   }
-}, withAuth("extract_content", async ({ url, options }) => {
+}, withAuth("extract_content", async (params) => {
   try {
-    if (!url) {
+    if (!params.url) {
       return { content: [{ type: "text", text: "URL parameter is required" }], isError: true };
     }
-    const result = await extractContentTool.execute({ url, options });
+    const result = await extractContentTool.execute(params);
     return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
   } catch (error) {
     return { content: [{ type: "text", text: `Content extraction failed: ${error.message}` }], isError: true };
@@ -624,12 +642,12 @@ registerToolIfEnabled("process_document", {
     options: z.object({}).passthrough().optional().describe("Additional processing options (maxPages, pageRange:{start,end}, extractText, extractMetadata, outputFormat, ...)"),
     ...COMPLIANCE_PARAMS
   }
-}, withAuth("process_document", async ({ source, sourceType, options }) => {
+}, withAuth("process_document", async (params) => {
   try {
-    if (!source) {
+    if (!params.source) {
       return { content: [{ type: "text", text: "Source parameter is required" }], isError: true };
     }
-    const result = await processDocumentTool.execute({ source, sourceType, options });
+    const result = await processDocumentTool.execute(params);
     return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
   } catch (error) {
     return { content: [{ type: "text", text: `Document processing failed: ${error.message}` }], isError: true };
@@ -694,12 +712,16 @@ registerToolIfEnabled("extract_structured", {
     }).optional().describe("LLM provider configuration for AI-powered extraction"),
     fallbackToSelectors: z.boolean().optional().default(true).describe("Fall back to CSS selector extraction if LLM is unavailable"),
     selectorHints: z.record(z.string()).optional().describe("CSS selector hints to guide extraction"),
-    ...COMPLIANCE_PARAMS
+    ...COMPLIANCE_PARAMS,
+    ...VERIFY_NUMBERS_PARAM
   },
   outputSchema: OUTPUT_SCHEMAS.extract_structured
-}, withAuth("extract_structured", async ({ url, schema, prompt, llmConfig, fallbackToSelectors, selectorHints }) => {
+}, withAuth("extract_structured", async (params) => {
   try {
-    const result = await extractStructuredTool.execute({ url, schema, prompt, llmConfig, fallbackToSelectors, selectorHints });
+    // Forward params whole. This wrapper used to destructure a fixed six, which
+    // silently dropped respect_robots and user_agent — both declared here and
+    // read by the tool, so the G5 override was accepted and ignored.
+    const result = await extractStructuredTool.execute(params);
     return dualOutput(result);
   } catch (error) {
     return { content: [{ type: "text", text: `Structured extraction failed: ${error.message}` }], isError: true };
@@ -718,7 +740,8 @@ registerToolIfEnabled("extract_with_llm", {
     provider: z.enum(["openai", "anthropic", "ollama", "auto"]).optional().default("auto").describe("LLM provider. Defaults to 'ollama' (local, no key, http://localhost:11434). Use 'openai' or 'anthropic' for cloud models (requires the matching API key)."),
     model: z.string().optional().describe("Override the model. For ollama, pass a name returned by list_ollama_models (e.g. 'llama3.2', 'qwen2.5:7b'). Defaults: openai='gpt-4o-mini', anthropic='claude-haiku-4-5-20251001', ollama='llama3.2' or $OLLAMA_DEFAULT_MODEL."),
     maxTokens: z.number().optional().default(4096).describe("Maximum output tokens"),
-    ...COMPLIANCE_PARAMS
+    ...COMPLIANCE_PARAMS,
+    ...VERIFY_NUMBERS_PARAM
   }
 }, withAuth("extract_with_llm", async (params) => {
   try {
@@ -1555,11 +1578,12 @@ registerToolIfEnabled("localization", {
 
 // Tool: scrape_template (D3.3 — pre-built site templates)
 registerToolIfEnabled("scrape_template", {
-  description: "Use this when you want structured data from a well-known site without writing custom selectors. Pass template:\"list\" to see all available templates. Supports: shopify-product (any Shopify storefront, read from the store's own /products/<handle>.json rather than the rendered page), amazon-product, linkedin-profile, github-repo, youtube-video, tweet, reddit-thread, hacker-news-front-page, producthunt-launch, stackoverflow-question, npm-package (read from the npm registry API rather than the npmjs.com page, which blocks plain fetches). Example: scrape_template({template:\"github-repo\", url:\"https://github.com/user/repo\"})",
+  description: "Use this when you want structured data from a well-known site or platform API without writing custom selectors. Three modes: a template id with a url (scrape_template({template:\"github-repo\", url:\"https://github.com/user/repo\"})); template:\"auto\" with a url, which picks the template from the URL and names its choice in the response; or template:\"list\" to enumerate every template with the URLs it handles. Page templates return one record — e-commerce, social, developer and news sites (shopify-product, amazon-product, github-repo, youtube-video, tweet, reddit-thread, hacker-news-front-page, producthunt-launch, stackoverflow-question, npm-package, linkedin-profile). List connectors return N records from one call and are driven by params instead of a url: job boards (Greenhouse, Lever, Ashby, Workable, Recruitee, Teamtailor) return a company's whole careers board, US government APIs (NHTSA VIN decode, NPI provider registry) answer keyless lookups, and shopify-collection returns a whole collection. Example: scrape_template({template:\"greenhouse-jobs\", params:{company:\"stripe\"}})",
   annotations: { title: "Scrape Template", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
   inputSchema: {
-    template: z.string().describe("Template ID (e.g. github-repo) or list to enumerate available templates"),
-    url: z.string().url().optional().describe("URL to scrape — required unless template is list"),
+    template: z.string().describe("Template ID (e.g. github-repo), \"auto\" to detect one from the url, or \"list\" to enumerate available templates"),
+    url: z.string().url().optional().describe("URL to scrape — required unless template is list, or params drive a list connector"),
+    params: z.record(z.any()).optional().describe("Parameters for a list connector, e.g. {company:\"stripe\"} for greenhouse-jobs or {store:\"www.allbirds.com\", collection:\"mens\"} for shopify-collection. Use template:\"list\" to see which templates take params"),
     timeout: z.number().min(5000).max(60000).optional().default(15000).describe("Request timeout in milliseconds"),
     ...COMPLIANCE_PARAMS
   }

package/src/cli/commands/template.js

@@ -8,8 +8,9 @@ import { runTool } from '../lib/runTool.js';
 export function register(program) {
   program
     .command('template [id] [target]')
-    .description('Scrape using a pre-built site template (e.g. amazon-product, github-repo)')
+    .description('Scrape using a pre-built site template (e.g. amazon-product, github-repo, or auto to detect one from the URL)')
     .option('--list', 'List all available templates')
+    .option('--params <json>', 'JSON parameters for a list connector, e.g. \'{"company":"stripe"}\'')
     .action(async (id, target, opts, cmd) => {
       const globals = cmd.parent.opts();
       const cliFlags = { json: globals.json, pretty: globals.pretty, quiet: globals.quiet };
@@ -21,11 +22,21 @@ export function register(program) {
         return;
       }
 
-      if (!id || !target) {
-        process.stderr.write('Error: template requires <id> and <target>, or use --list\n');
+      let params;
+      if (opts.params) {
+        try {
+          params = JSON.parse(opts.params);
+        } catch (e) {
+          process.stderr.write(`Error parsing --params JSON: ${e.message}\n`);
+          process.exit(1);
+        }
+      }
+
+      if (!id || (!target && !params)) {
+        process.stderr.write('Error: template requires <id> and <target>, or <id> with --params, or use --list\n');
         process.exit(1);
       }
 
-      await runTool(tool, { template: id, url: target }, cliFlags);
+      await runTool(tool, { template: id, url: target, params }, cliFlags);
     });
 }

package/src/core/AuthManager.js

@@ -562,6 +562,7 @@ class AuthManager {
       process_document: 2,
       localization: 2,
       scrape: 2,
+      extract_embedded_state: 2,
       reddit_search: 5, // a Reddit-wide search spends a web search to discover posts, same as search_web
 
       // 3 credits

package/src/schemas/toolOutputSchemas.js

@@ -259,6 +259,17 @@ const extractStructuredShape = {
     errors: z.array(z.string()).optional()
   }).passthrough().optional(),
   extractionNotes: z.array(z.string()).optional(),
+  provenance: z.object({
+    enabled: z.boolean().optional().describe('Whether the numeric provenance guard ran'),
+    verified: z.number().optional().describe('Numeric values found literally in the page source'),
+    nulled: z.number().optional().describe('Numeric values replaced with null because the source does not contain them'),
+    unverified: z.array(z.object({
+      path: z.string().optional().describe('Path to the field, e.g. configurations[2].price'),
+      value: z.unknown().optional().describe('The value that was removed'),
+      reason: z.string().optional().describe('"not_found_in_source"')
+    }).passthrough()).optional(),
+    skipped: z.string().optional().describe('"empty_source" when there was nothing to check against')
+  }).passthrough().optional(),
   _cost: costShape
 };
 

package/src/skills/agent-skills/crawlforge-getting-started/SKILL.md

@@ -8,7 +8,7 @@ metadata:
 
 # CrawlForge: Getting Started
 
-CrawlForge is an MCP server with **28 tools** for web scraping, crawling,
+CrawlForge is an MCP server with **29 tools** for web scraping, crawling,
 extraction, research, change tracking, and AI-compliance. This skill orients you
 and routes each request to the right specialized skill.
 
@@ -52,13 +52,13 @@ stored at `~/.crawlforge/config.json`.
 | Watch a page for changes / monitor pricing | **crawlforge-change-tracking** |
 | Scrape many URLs, run browser actions, generate llms.txt | **crawlforge-batch-automation** |
 
-## The 28 tools at a glance
+## The 29 tools at a glance
 
 - **Basic (5):** fetch_url, extract_text, extract_links, extract_metadata, scrape_structured
 - **Unified (1):** scrape (multi-format single fetch)
 - **Search & research (5):** search_web, serp_rank, reddit_search, deep_research, agent
 - **Crawl (2):** crawl_deep, map_site
-- **Extract & analyze (7):** extract_content, process_document, summarize_content, analyze_content, extract_structured, extract_with_llm, list_ollama_models
+- **Extract & analyze (8):** extract_content, process_document, summarize_content, analyze_content, extract_structured, extract_with_llm, extract_embedded_state, list_ollama_models
 - **Batch & automation (4):** batch_scrape, get_batch_results, scrape_with_actions, generate_llms_txt
 - **Stealth & locale (2):** stealth_mode, localization
 - **Templates & tracking (2):** scrape_template, track_changes

package/src/skills/agent-skills/crawlforge-getting-started/references/credits.md

@@ -21,6 +21,7 @@ metered; there is no free tier. Tools marked "scales" cost more as work grows.
 |------|-------|
 | `scrape` | Unified multi-format single fetch. |
 | `scrape_structured` | CSS-selector extraction. |
+| `extract_embedded_state` | Embedded JS state (`__NEXT_DATA__`, RSC, Nuxt, Apollo, Redux). |
 | `extract_content` | Readability-cleaned article. |
 | `map_site` | URL discovery / sitemap. |
 | `process_document` | PDF / DOCX / TXT parsing. |

package/src/skills/agent-skills/crawlforge-structured-extraction/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: crawlforge-structured-extraction
-description: "Extracts structured JSON and analyzes content with CrawlForge's extract_structured, extract_with_llm, scrape_structured, scrape_template, process_document, analyze_content, summarize_content, and list_ollama_models tools. Use when the user wants to extract specific fields, pull data into a JSON schema, extract by natural-language prompt, scrape with CSS selectors, get product, profile, or repo data from known sites (Amazon, LinkedIn, GitHub, YouTube, Reddit, and more), parse a PDF or DOCX, summarize a page, or analyze sentiment, entities, or keywords. Defaults to local Ollama for LLM extraction; OpenAI and Anthropic optional."
+description: "Extracts structured JSON and analyzes content with CrawlForge's extract_structured, extract_with_llm, extract_embedded_state, scrape_structured, scrape_template, process_document, analyze_content, summarize_content, and list_ollama_models tools. Use when the user wants to extract specific fields, pull data into a JSON schema, extract by natural-language prompt, scrape with CSS selectors, read a page's embedded JavaScript state (__NEXT_DATA__, React Server Components, Nuxt, Apollo, Redux), get product, profile, or repo data from known sites (Amazon, LinkedIn, GitHub, YouTube, Reddit, and more), parse a PDF or DOCX, summarize a page, or analyze sentiment, entities, or keywords. Defaults to local Ollama for LLM extraction; OpenAI and Anthropic optional."
 metadata:
   version: 4.8.0
   source: crawlforge-mcp-server
@@ -17,6 +17,7 @@ extraction method by how predictable the page is and whether an LLM is needed.
 |-----------------|------|------|
 | A well-known site (Amazon, GitHub, LinkedIn...) | `scrape_template` | 1 |
 | Exact CSS selectors for the fields | `scrape_structured` | 2 |
+| The data is in the page's JS state, not its HTML | `extract_embedded_state` | 2 |
 | A JSON schema to fill (LLM, CSS fallback) | `extract_structured` | 3 |
 | A natural-language extraction instruction | `extract_with_llm` | 3 |
 | A PDF / DOCX / TXT to parse | `process_document` | 2 |
@@ -24,8 +25,26 @@ extraction method by how predictable the page is and whether an LLM is needed.
 | Sentiment / entities / keywords / readability | `analyze_content` | 3 |
 | To list local LLMs available for extraction | `list_ollama_models` | 1 |
 
-Cheapest-first rule: try `scrape_template` → `scrape_structured` (deterministic)
-before reaching for the LLM tools.
+Cheapest-first rule: try `scrape_template` → `scrape_structured` /
+`extract_embedded_state` (all deterministic) before reaching for the LLM tools.
+On a React/Next/Nuxt page the values are usually sitting in the embedded state
+already, exact and typed — that beats asking a model to read them off the render.
+
+## extract_embedded_state — the page's own JS state (cost: 2)
+
+```json
+{ "tool": "extract_embedded_state", "params": { "url": "https://www.ticketmaster.com/discover/concerts", "path": "next_data.props.pageProps" } }
+```
+
+Finds `__NEXT_DATA__`, `self.__next_f` (React Server Component payloads),
+`window.__NUXT__`, `__APOLLO_STATE__`, `__INITIAL_STATE__`, `__PRELOADED_STATE__`
+and `<script type="application/json">` blocks, keyed by source name. No LLM in
+the path, so values are exact rather than inferred.
+
+These payloads are routinely over a megabyte — pass `path` (dotted keys and
+array indexes, e.g. `next_data.props.pageProps` or `next_f[0].f`) to return one
+subtree. Without it, a large result comes back with a warning naming the biggest
+source and a ready-to-paste path.
 
 ## scrape_template — known sites, zero selectors (cost: 1)
 

package/src/tools/basic/extractMetadata.js

@@ -2,28 +2,12 @@
  * extract_metadata — Extract page metadata (title, description, OG tags, etc.).
  * Extracted from server.js inline handler.
  * B1: Parse JSON-LD and microdata; stronger title fallback chain (og:title → <title> → h1).
+ * 3.3: json_ld_types promotes JSON-LD from a raw dump to a filtered extraction path.
  */
 
 import { load } from 'cheerio';
 import { fetchWithTimeout } from './_fetch.js';
-
-/**
- * Parse all JSON-LD blocks from the document.
- * @param {import('cheerio').CheerioAPI} $
- * @returns {Array}
- */
-function parseJsonLd($) {
-  const results = [];
-  $('script[type="application/ld+json"]').each((_, el) => {
-    try {
-      const raw = $(el).html();
-      if (raw) results.push(JSON.parse(raw));
-    } catch {
-      // Skip invalid blocks
-    }
-  });
-  return results;
-}
+import { parseJsonLd, filterJsonLdByType } from '../../utils/jsonLd.js';
 
 /**
  * Parse microdata items (elements with itemscope).
@@ -60,9 +44,10 @@ function parseMicrodata($) {
 }
 
 /**
- * @param {{ url: string, user_agent?: string, respect_robots?: boolean }} params
+ * @param {{ url: string, user_agent?: string, respect_robots?: boolean,
+ *   json_ld_types?: string[] }} params
  */
-export async function extractMetadataHandler({ url, user_agent, respect_robots }) {
+export async function extractMetadataHandler({ url, user_agent, respect_robots, json_ld_types }) {
   try {
     const response = await fetchWithTimeout(url, {
       userAgent: user_agent,
@@ -115,25 +100,33 @@ export async function extractMetadataHandler({ url, user_agent, respect_robots }
     const jsonLd = parseJsonLd($);
     const microdata = parseMicrodata($);
 
+    const result = {
+      title,
+      description,
+      keywords: keywords.split(',').map(k => k.trim()).filter(Boolean),
+      canonical_url: canonical,
+      author,
+      robots,
+      viewport,
+      charset,
+      og_tags: ogTags,
+      twitter_tags: twitterTags,
+      json_ld: jsonLd,
+      microdata,
+      url: response.url
+    };
+
+    // With a type filter, json_ld carries only the matching nodes — returning
+    // the raw dump as well would double the payload on the large pages that
+    // make filtering worth asking for.
+    if (json_ld_types?.length) {
+      const { items, counts } = filterJsonLdByType(jsonLd, json_ld_types);
+      result.json_ld = items;
+      result.json_ld_type_counts = counts;
+    }
+
     return {
-      content: [{
-        type: 'text',
-        text: JSON.stringify({
-          title,
-          description,
-          keywords: keywords.split(',').map(k => k.trim()).filter(Boolean),
-          canonical_url: canonical,
-          author,
-          robots,
-          viewport,
-          charset,
-          og_tags: ogTags,
-          twitter_tags: twitterTags,
-          json_ld: jsonLd,
-          microdata,
-          url: response.url
-        }, null, 2)
-      }]
+      content: [{ type: 'text', text: JSON.stringify(result, null, 2) }]
     };
   } catch (error) {
     return {

package/src/tools/extract/extractEmbeddedState.js

@@ -0,0 +1,72 @@
+/**
+ * extract_embedded_state — return the JSON state a page already ships in its
+ * own HTML: __NEXT_DATA__, RSC flight chunks (self.__next_f), __NUXT__,
+ * __APOLLO_STATE__, __INITIAL_STATE__, __PRELOADED_STATE__ and
+ * <script type="application/json"> blocks.
+ *
+ * One fetch, exact values, no LLM in the extraction path — the numbers come
+ * from the site's own serialized state, so they cannot be fabricated.
+ */
+
+import { fetchAndParse } from './_fetchAndParse.js';
+import { extractEmbeddedState } from '../../utils/embeddedState.js';
+import { selectJsonPath } from '../../utils/jsonPath.js';
+
+// Above this, an unscoped result is big enough to be a problem for the caller
+// (context window, transport) rather than just large. Warn — never truncate:
+// a half-serialized object is worse than a big one, and `path` already gives
+// the caller an exact way to ask for less.
+const LARGE_RESULT_BYTES = 256_000;
+
+/**
+ * @param {{ url: string, path?: string, user_agent?: string, respect_robots?: boolean }} params
+ */
+export async function extractEmbeddedStateHandler({ url, path, user_agent, respect_robots }) {
+  try {
+    // The raw `html` is used, not `$`: fetchAndParse strips <script> from the
+    // parsed tree by default, and every source here lives in a script tag.
+    const { html, finalUrl, warnings: fetchWarnings } = await fetchAndParse(url, {
+      userAgent: user_agent,
+      respectRobots: respect_robots,
+      tool: 'extract_embedded_state'
+    });
+
+    const state = extractEmbeddedState(html);
+    const warnings = [...fetchWarnings, ...state.warnings];
+
+    if (state.found.length === 0) {
+      warnings.push(
+        'No embedded state found. The page may render entirely on the client, or ship its data in a format this tool does not read.'
+      );
+    }
+
+    const data = path ? selectJsonPath(state.data, path) : state.data;
+    const bytes = Buffer.byteLength(JSON.stringify(data) ?? '');
+
+    if (!path && bytes > LARGE_RESULT_BYTES) {
+      const largest = state.found.reduce((a, b) => (b.bytes > a.bytes ? b : a));
+      warnings.push(
+        `Result is ${bytes} bytes; "${largest.name}" alone is ${largest.bytes}. Re-run with path to scope it, e.g. path:"${largest.name}.${Object.keys(state.data[largest.name])[0]}".`
+      );
+    }
+
+    return {
+      content: [{
+        type: 'text',
+        text: JSON.stringify({
+          url: finalUrl,
+          found: state.found,
+          path: path || null,
+          bytes,
+          data,
+          warnings
+        }, null, 2)
+      }]
+    };
+  } catch (error) {
+    return {
+      content: [{ type: 'text', text: `Failed to extract embedded state: ${error.message}` }],
+      isError: true
+    };
+  }
+}

package/src/tools/extract/extractStructured.js

@@ -11,6 +11,7 @@ import { LLMManager } from '../../core/llm/LLMManager.js';
 import { CRAWLFORGE_USER_AGENT } from '../../utils/fetchIdentity.js';
 import { fetchAndParse, flattenBodyText } from './_fetchAndParse.js';
 import { extractMainContent } from '../scrape/_mainContent.js';
+import { verifyNumericProvenance } from '../../utils/provenance.js';
 
 // Semantic element selectors for well-known field names, tried as a last
 // resort in the CSS fallback so common fields (e.g. "title") still resolve when
@@ -79,7 +80,8 @@ const ExtractStructuredSchema = z.object({
   fallbackToSelectors: z.boolean().optional().default(true),
   selectorHints: z.record(z.string()).optional(),
   respect_robots: z.boolean().optional(),
-  user_agent: z.string().optional()
+  user_agent: z.string().optional(),
+  verify_numbers: z.boolean().optional().default(true)
 });
 
 export class ExtractStructuredTool {
@@ -136,7 +138,7 @@ export class ExtractStructuredTool {
 
     try {
       const validated = ExtractStructuredSchema.parse(params);
-      const { url, schema, prompt, llmConfig, fallbackToSelectors, selectorHints, respect_robots, user_agent } = validated;
+      const { url, schema, prompt, llmConfig, fallbackToSelectors, selectorHints, respect_robots, user_agent, verify_numbers } = validated;
 
       // Step 1: Fetch and parse — shared helper strips scripts/styles/iframes/svgs
       const { html, $, textContent, warnings } = await fetchAndParse(url, {
@@ -180,6 +182,43 @@ export class ExtractStructuredTool {
         llmErrorMessage = llmError.message;
       }
 
+      // Step 3b (3.4): numeric provenance. Only the LLM path invents numbers —
+      // the CSS and keyword fallbacks can only return text they read off the
+      // page — so the guard is scoped to it.
+      //
+      // It is checked against the FULL source, never `mainContentText()`: on
+      // the Apple MacBook Air page Readability keeps the FAQ block and every
+      // price is left behind in an embedded JSON blob, so checking against what
+      // the model was shown would null every correct price.
+      let provenance = { enabled: false };
+      if (extractionResult && extractionMethod === 'llm' && verify_numbers) {
+        const checked = verifyNumericProvenance(extractionResult.data || {}, `${html}\n${textContent}`);
+        // The model's own `valid` flag described the data before the guard ran.
+        // A required field the guard nulled is not filled in any more, so that
+        // flag cannot stand or the response reports a fabrication as valid.
+        const nulledRequired = checked.unverified
+          .map((entry) => entry.path)
+          .filter((path) => (schema.required || []).includes(path));
+        extractionResult = {
+          ...extractionResult,
+          data: checked.data,
+          ...(nulledRequired.length > 0 ? {
+            valid: false,
+            validationErrors: [
+              ...(extractionResult.validationErrors || []),
+              ...nulledRequired.map((field) => `Field "${field}" was not found in the page source`)
+            ]
+          } : {})
+        };
+        provenance = {
+          enabled: true,
+          verified: checked.verified,
+          nulled: checked.nulled,
+          unverified: checked.unverified
+        };
+        if (checked.skipped) provenance.skipped = checked.skipped;
+      }
+
       // Step 4: CSS selector fallback if LLM unavailable or failed
       if (!extractionResult && fallbackToSelectors !== false) {
         // D1.4: no LLM configured and the schema demands more than 3 required
@@ -223,6 +262,12 @@ export class ExtractStructuredTool {
       if (llmErrorMessage) {
         extractionNotes.push(`LLM extraction failed: ${llmErrorMessage}`);
       }
+      if (provenance.nulled > 0) {
+        extractionNotes.push(
+          `Numeric provenance: ${provenance.nulled} value(s) the model returned are not in the page source and were replaced with null: ` +
+          provenance.unverified.map((u) => `${u.path}=${JSON.stringify(u.value)}`).join(', ')
+        );
+      }
 
       // A required field that came back missing or empty is a failed
       // extraction, not a successful one carrying a note: surface it at the
@@ -249,6 +294,7 @@ export class ExtractStructuredTool {
           errors: extractionResult.validationErrors || []
         },
         extractionNotes,
+        provenance,
         ...(warnings?.length ? { warnings } : {})
       };