crawlforge-mcp-server 5.3.0 → 5.4.0

package/src/tools/basic/extractMetadata.js

@@ -2,28 +2,12 @@
  * extract_metadata — Extract page metadata (title, description, OG tags, etc.).
  * Extracted from server.js inline handler.
  * B1: Parse JSON-LD and microdata; stronger title fallback chain (og:title → <title> → h1).
+ * 3.3: json_ld_types promotes JSON-LD from a raw dump to a filtered extraction path.
  */
 
 import { load } from 'cheerio';
 import { fetchWithTimeout } from './_fetch.js';
-
-/**
- * Parse all JSON-LD blocks from the document.
- * @param {import('cheerio').CheerioAPI} $
- * @returns {Array}
- */
-function parseJsonLd($) {
-  const results = [];
-  $('script[type="application/ld+json"]').each((_, el) => {
-    try {
-      const raw = $(el).html();
-      if (raw) results.push(JSON.parse(raw));
-    } catch {
-      // Skip invalid blocks
-    }
-  });
-  return results;
-}
+import { parseJsonLd, filterJsonLdByType } from '../../utils/jsonLd.js';
 
 /**
  * Parse microdata items (elements with itemscope).
@@ -60,9 +44,10 @@ function parseMicrodata($) {
 }
 
 /**
- * @param {{ url: string, user_agent?: string, respect_robots?: boolean }} params
+ * @param {{ url: string, user_agent?: string, respect_robots?: boolean,
+ *   json_ld_types?: string[] }} params
  */
-export async function extractMetadataHandler({ url, user_agent, respect_robots }) {
+export async function extractMetadataHandler({ url, user_agent, respect_robots, json_ld_types }) {
   try {
     const response = await fetchWithTimeout(url, {
       userAgent: user_agent,
@@ -115,25 +100,33 @@ export async function extractMetadataHandler({ url, user_agent, respect_robots }
     const jsonLd = parseJsonLd($);
     const microdata = parseMicrodata($);
 
+    const result = {
+      title,
+      description,
+      keywords: keywords.split(',').map(k => k.trim()).filter(Boolean),
+      canonical_url: canonical,
+      author,
+      robots,
+      viewport,
+      charset,
+      og_tags: ogTags,
+      twitter_tags: twitterTags,
+      json_ld: jsonLd,
+      microdata,
+      url: response.url
+    };
+
+    // With a type filter, json_ld carries only the matching nodes — returning
+    // the raw dump as well would double the payload on the large pages that
+    // make filtering worth asking for.
+    if (json_ld_types?.length) {
+      const { items, counts } = filterJsonLdByType(jsonLd, json_ld_types);
+      result.json_ld = items;
+      result.json_ld_type_counts = counts;
+    }
+
     return {
-      content: [{
-        type: 'text',
-        text: JSON.stringify({
-          title,
-          description,
-          keywords: keywords.split(',').map(k => k.trim()).filter(Boolean),
-          canonical_url: canonical,
-          author,
-          robots,
-          viewport,
-          charset,
-          og_tags: ogTags,
-          twitter_tags: twitterTags,
-          json_ld: jsonLd,
-          microdata,
-          url: response.url
-        }, null, 2)
-      }]
+      content: [{ type: 'text', text: JSON.stringify(result, null, 2) }]
     };
   } catch (error) {
     return {

package/src/tools/extract/extractEmbeddedState.js

@@ -0,0 +1,72 @@
+/**
+ * extract_embedded_state — return the JSON state a page already ships in its
+ * own HTML: __NEXT_DATA__, RSC flight chunks (self.__next_f), __NUXT__,
+ * __APOLLO_STATE__, __INITIAL_STATE__, __PRELOADED_STATE__ and
+ * <script type="application/json"> blocks.
+ *
+ * One fetch, exact values, no LLM in the extraction path — the numbers come
+ * from the site's own serialized state, so they cannot be fabricated.
+ */
+
+import { fetchAndParse } from './_fetchAndParse.js';
+import { extractEmbeddedState } from '../../utils/embeddedState.js';
+import { selectJsonPath } from '../../utils/jsonPath.js';
+
+// Above this, an unscoped result is big enough to be a problem for the caller
+// (context window, transport) rather than just large. Warn — never truncate:
+// a half-serialized object is worse than a big one, and `path` already gives
+// the caller an exact way to ask for less.
+const LARGE_RESULT_BYTES = 256_000;
+
+/**
+ * @param {{ url: string, path?: string, user_agent?: string, respect_robots?: boolean }} params
+ */
+export async function extractEmbeddedStateHandler({ url, path, user_agent, respect_robots }) {
+  try {
+    // The raw `html` is used, not `$`: fetchAndParse strips <script> from the
+    // parsed tree by default, and every source here lives in a script tag.
+    const { html, finalUrl, warnings: fetchWarnings } = await fetchAndParse(url, {
+      userAgent: user_agent,
+      respectRobots: respect_robots,
+      tool: 'extract_embedded_state'
+    });
+
+    const state = extractEmbeddedState(html);
+    const warnings = [...fetchWarnings, ...state.warnings];
+
+    if (state.found.length === 0) {
+      warnings.push(
+        'No embedded state found. The page may render entirely on the client, or ship its data in a format this tool does not read.'
+      );
+    }
+
+    const data = path ? selectJsonPath(state.data, path) : state.data;
+    const bytes = Buffer.byteLength(JSON.stringify(data) ?? '');
+
+    if (!path && bytes > LARGE_RESULT_BYTES) {
+      const largest = state.found.reduce((a, b) => (b.bytes > a.bytes ? b : a));
+      warnings.push(
+        `Result is ${bytes} bytes; "${largest.name}" alone is ${largest.bytes}. Re-run with path to scope it, e.g. path:"${largest.name}.${Object.keys(state.data[largest.name])[0]}".`
+      );
+    }
+
+    return {
+      content: [{
+        type: 'text',
+        text: JSON.stringify({
+          url: finalUrl,
+          found: state.found,
+          path: path || null,
+          bytes,
+          data,
+          warnings
+        }, null, 2)
+      }]
+    };
+  } catch (error) {
+    return {
+      content: [{ type: 'text', text: `Failed to extract embedded state: ${error.message}` }],
+      isError: true
+    };
+  }
+}

package/src/tools/extract/extractStructured.js

@@ -11,6 +11,7 @@ import { LLMManager } from '../../core/llm/LLMManager.js';
 import { CRAWLFORGE_USER_AGENT } from '../../utils/fetchIdentity.js';
 import { fetchAndParse, flattenBodyText } from './_fetchAndParse.js';
 import { extractMainContent } from '../scrape/_mainContent.js';
+import { verifyNumericProvenance } from '../../utils/provenance.js';
 
 // Semantic element selectors for well-known field names, tried as a last
 // resort in the CSS fallback so common fields (e.g. "title") still resolve when
@@ -79,7 +80,8 @@ const ExtractStructuredSchema = z.object({
   fallbackToSelectors: z.boolean().optional().default(true),
   selectorHints: z.record(z.string()).optional(),
   respect_robots: z.boolean().optional(),
-  user_agent: z.string().optional()
+  user_agent: z.string().optional(),
+  verify_numbers: z.boolean().optional().default(true)
 });
 
 export class ExtractStructuredTool {
@@ -136,7 +138,7 @@ export class ExtractStructuredTool {
 
     try {
       const validated = ExtractStructuredSchema.parse(params);
-      const { url, schema, prompt, llmConfig, fallbackToSelectors, selectorHints, respect_robots, user_agent } = validated;
+      const { url, schema, prompt, llmConfig, fallbackToSelectors, selectorHints, respect_robots, user_agent, verify_numbers } = validated;
 
       // Step 1: Fetch and parse — shared helper strips scripts/styles/iframes/svgs
       const { html, $, textContent, warnings } = await fetchAndParse(url, {
@@ -180,6 +182,43 @@ export class ExtractStructuredTool {
         llmErrorMessage = llmError.message;
       }
 
+      // Step 3b (3.4): numeric provenance. Only the LLM path invents numbers —
+      // the CSS and keyword fallbacks can only return text they read off the
+      // page — so the guard is scoped to it.
+      //
+      // It is checked against the FULL source, never `mainContentText()`: on
+      // the Apple MacBook Air page Readability keeps the FAQ block and every
+      // price is left behind in an embedded JSON blob, so checking against what
+      // the model was shown would null every correct price.
+      let provenance = { enabled: false };
+      if (extractionResult && extractionMethod === 'llm' && verify_numbers) {
+        const checked = verifyNumericProvenance(extractionResult.data || {}, `${html}\n${textContent}`);
+        // The model's own `valid` flag described the data before the guard ran.
+        // A required field the guard nulled is not filled in any more, so that
+        // flag cannot stand or the response reports a fabrication as valid.
+        const nulledRequired = checked.unverified
+          .map((entry) => entry.path)
+          .filter((path) => (schema.required || []).includes(path));
+        extractionResult = {
+          ...extractionResult,
+          data: checked.data,
+          ...(nulledRequired.length > 0 ? {
+            valid: false,
+            validationErrors: [
+              ...(extractionResult.validationErrors || []),
+              ...nulledRequired.map((field) => `Field "${field}" was not found in the page source`)
+            ]
+          } : {})
+        };
+        provenance = {
+          enabled: true,
+          verified: checked.verified,
+          nulled: checked.nulled,
+          unverified: checked.unverified
+        };
+        if (checked.skipped) provenance.skipped = checked.skipped;
+      }
+
       // Step 4: CSS selector fallback if LLM unavailable or failed
       if (!extractionResult && fallbackToSelectors !== false) {
         // D1.4: no LLM configured and the schema demands more than 3 required
@@ -223,6 +262,12 @@ export class ExtractStructuredTool {
       if (llmErrorMessage) {
         extractionNotes.push(`LLM extraction failed: ${llmErrorMessage}`);
       }
+      if (provenance.nulled > 0) {
+        extractionNotes.push(
+          `Numeric provenance: ${provenance.nulled} value(s) the model returned are not in the page source and were replaced with null: ` +
+          provenance.unverified.map((u) => `${u.path}=${JSON.stringify(u.value)}`).join(', ')
+        );
+      }
 
       // A required field that came back missing or empty is a failed
       // extraction, not a successful one carrying a note: surface it at the
@@ -249,6 +294,7 @@ export class ExtractStructuredTool {
           errors: extractionResult.validationErrors || []
         },
         extractionNotes,
+        provenance,
         ...(warnings?.length ? { warnings } : {})
       };
 

package/src/tools/extract/extractWithLlm.js

@@ -10,6 +10,7 @@
 import { z } from 'zod';
 import { fetchAndParse } from './_fetchAndParse.js';
 import { ollamaBaseUrl, ollamaHeaders, selectOllamaModel } from '../../utils/ollamaConfig.js';
+import { verifyNumericProvenance } from '../../utils/provenance.js';
 // D1.3: SamplingClient for MCP sampling fallback (lazy — only imported if needed)
 let _SamplingClient = null;
 async function getSamplingClient() {
@@ -485,6 +486,7 @@ export class ExtractWithLlm {
    * @param {number}  [params.maxTokens] - Max output tokens (default 4096)
    * @param {boolean} [params.respect_robots] - Per-request robots.txt override
    * @param {string}  [params.user_agent]     - Per-request identity override
+   * @param {boolean} [params.verify_numbers] - Numeric provenance guard (default true)
    * @returns {Promise<Object>}
    */
   async execute(params) {
@@ -497,7 +499,8 @@ export class ExtractWithLlm {
       model: modelParam,
       maxTokens = 4096,
       respect_robots,
-      user_agent
+      user_agent,
+      verify_numbers = true
     } = params;
 
     // Validate: exactly one of url or content must be provided
@@ -530,18 +533,25 @@ export class ExtractWithLlm {
 
     // Step 1: Get text to extract from
     let text;
+    // What the provenance guard checks against. Deliberately wider than what
+    // the model is shown: the raw html carries numbers the flattened text does
+    // not (Apple's prices exist only inside an embedded JSON blob), and a value
+    // missing from `text` but present on the page must not be nulled.
+    let sourceForProvenance;
     let fetchWarnings = [];
     try {
       if (url) {
-        const { textContent, warnings } = await fetchAndParse(url, {
+        const { html, textContent, warnings } = await fetchAndParse(url, {
           respectRobots: respect_robots,
           userAgent: user_agent,
           tool: 'extract_with_llm'
         });
         text = textContent;
+        sourceForProvenance = `${html}\n${textContent}`;
         fetchWarnings = warnings || [];
       } else {
         text = content;
+        sourceForProvenance = content;
       }
     } catch (fetchErr) {
       return { success: false, error: `Failed to fetch content: ${fetchErr.message}` };
@@ -654,10 +664,27 @@ export class ExtractWithLlm {
       }
     }
 
+    // 3.4: numeric provenance. A number the model wrote that is nowhere in the
+    // page it was given was invented, so it comes back null with a reason
+    // rather than as a confident answer.
+    let provenance = { enabled: false };
+    if (verify_numbers) {
+      const checked = verifyNumericProvenance(parsed, sourceForProvenance);
+      parsed = checked.data;
+      provenance = {
+        enabled: true,
+        verified: checked.verified,
+        nulled: checked.nulled,
+        unverified: checked.unverified
+      };
+      if (checked.skipped) provenance.skipped = checked.skipped;
+    }
+
     // C3: surface truncation metadata so callers know the input was clipped
     const result = {
       success: true,
       data: parsed,
+      provenance,
       provider: resolvedModel === 'sampling' ? 'sampling' : provider,
       model: resolvedModel || model,
       usage

package/src/tools/templates/ScrapeTemplateTool.js

@@ -4,44 +4,113 @@
  * Usage pattern (D3.3):
  *   const tool = new ScrapeTemplateTool();
  *   const result = await tool.execute({ template: "github-repo", url: "https://github.com/user/repo" });
+ *
+ * Three ways in: a template id, `"auto"` (the registry picks the template from
+ * the URL and the response names the one it chose), and `"list"`. A list
+ * connector is driven by `params` rather than a URL and returns N entities from
+ * one call — the registry builds the URL, this tool fetches it.
  */
 
 import { TemplateRegistry } from 'crawlforge-extractors';
 import { safeFetch } from '../../utils/ssrfGuard.js';
 import { preflightFetch } from '../../utils/robotsGate.js';
 import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
+import { markPreflightRefusal } from '../../server/requestContext.js';
+
+/**
+ * A caller mistake caught before anything is fetched: no template matches the
+ * URL, a required list parameter is missing, a key-based connector has no key.
+ * We fetched nothing, so — like a robots refusal — it costs the caller nothing.
+ */
+function badRequest(message) {
+  markPreflightRefusal('BAD_REQUEST');
+  return new Error(message);
+}
 
 export class ScrapeTemplateTool {
-  constructor() {
-    this.registry = new TemplateRegistry();
+  /**
+   * @param {{ templates?: object[] }} [config] `templates` replaces the shipped
+   *   set; tests use it to exercise a connector shape nothing ships yet.
+   */
+  constructor(config = {}) {
+    this.registry = new TemplateRegistry(config?.templates);
+  }
+
+  /** The catalogue — no network. */
+  listTemplates() {
+    const templates = this.registry.list();
+    return { templates, count: templates.length };
   }
 
   /**
    * Execute the scrape_template tool.
-   * @param {{ template: string, url: string, timeout?: number,
-   *          user_agent?: string, respect_robots?: boolean }} params
+   * @param {{ template: string, url?: string, params?: object, timeout?: number,
+   *          user_agent?: string, respect_robots?: boolean }} request
    * @returns {Promise<object>}
    */
-  async execute({ template, url, timeout = 15000, user_agent, respect_robots }) {
-    // list mode — return available templates without scraping
-    if (template === 'list' || !url) {
-      return {
-        templates: this.registry.list(),
-        count: this.registry.list().length
-      };
+  async execute({ template, url, params, timeout = 15000, user_agent, respect_robots }) {
+    if (template === 'list') return this.listTemplates();
+
+    let templateId = template;
+    if (template === 'auto') {
+      if (!url) {
+        throw badRequest(
+          'template "auto" needs a url to detect from. Pass a url, or template:"list" to see every template.'
+        );
+      }
+      const detected = this.registry.detect(url);
+      if (!detected) {
+        throw badRequest(
+          `No template matches ${url}. Pass template:"list" to see every template and the URLs ` +
+          'each one handles, or name a template explicitly.'
+        );
+      }
+      templateId = detected.id;
+    } else if (!url && !params) {
+      // A template named with nothing to run it against still lists, as before.
+      return this.listTemplates();
     }
 
     // Validate template exists before making network call
-    const tpl = this.registry.get(template);
+    const tpl = this.registry.get(templateId);
     if (!tpl) {
       const available = this.registry.list().map(t => t.id).join(', ');
-      throw new Error(`Unknown template "${template}". Available templates: ${available}`);
+      throw new Error(`Unknown template "${templateId}". Available templates: ${available}`);
+    }
+
+    // A key-based connector is answered here or not at all: the registry never
+    // reads process.env, and a missing key must not reach the target as a 401.
+    let apiKey;
+    if (tpl.requiresApiKey) {
+      apiKey = process.env[tpl.credentialRef];
+      if (!apiKey) {
+        throw badRequest(
+          `Template "${templateId}" reads an API-keyed endpoint. Set ${tpl.credentialRef} in the ` +
+          'server environment and try again.'
+        );
+      }
     }
 
-    // A template may redirect its own fetch to a machine-readable endpoint
-    // (shopify-product reads /products/<handle>.json). Same host either way,
-    // so the SSRF guard below still applies.
-    const fetchUrl = template === 'list' ? url : (tpl.resolveUrl ? tpl.resolveUrl(url) : url);
+    // The URL actually fetched comes from one of three places. The robots gate
+    // below runs against that URL, never the caller's input — listUrl in
+    // particular reaches a host the caller never named.
+    let fetchUrl;
+    if (params && tpl.listUrl) {
+      try {
+        fetchUrl = tpl.listUrl(apiKey ? { ...params, apiKey } : params);
+      } catch (error) {
+        // listUrl throws naming the parameter it wanted: a caller mistake, not
+        // a fetch that failed.
+        throw badRequest(error.message);
+      }
+    } else if (!url) {
+      throw badRequest(`Template "${templateId}" is reached by url, not params. Pass a url.`);
+    } else {
+      // A template may redirect its own fetch to a machine-readable endpoint
+      // (shopify-product reads /products/<handle>.json). Same host either way,
+      // so the SSRF guard below still applies.
+      fetchUrl = tpl.resolveUrl ? tpl.resolveUrl(url) : url;
+    }
 
     // Robots gate + per-host politeness before any request to the target.
     const gate = await preflightFetch(fetchUrl, {
@@ -76,8 +145,28 @@ export class ScrapeTemplateTool {
       throw error;
     }
 
+    // What the response reports. A params-driven call named no URL, so the one
+    // we built is the one to report — and the one the extractor quotes in its
+    // own error messages. Except on a key-based connector, where that URL
+    // carries the key: it goes into listUrl and nowhere else.
+    const keyed = Boolean(tpl.requiresApiKey);
+    const reportedUrl = url ?? (keyed ? undefined : fetchUrl);
+    const reportedFetchUrl = keyed ? reportedUrl : fetchUrl;
+
+    let echoParams;
+    if (params) {
+      echoParams = { ...params };
+      delete echoParams.apiKey;
+    }
+
     // Run the template extractor
-    const result = await this.registry.run(template, html, url, fetchUrl);
+    const result = tpl.extractList
+      ? await this.registry.runList(templateId, html, { url: reportedUrl, params: echoParams })
+      : await this.registry.run(templateId, html, reportedUrl, reportedFetchUrl);
+
+    // run() stamps fetchedUrl itself; runList() does not.
+    if (tpl.extractList && reportedFetchUrl !== reportedUrl) result.fetchedUrl = reportedFetchUrl;
+
     return gate.warnings.length > 0 ? { ...result, warnings: gate.warnings } : result;
   }
 }