crawlforge-mcp-server 5.3.1 → 5.4.0

package/src/tools/extract/extractWithLlm.js

@@ -10,6 +10,7 @@
 import { z } from 'zod';
 import { fetchAndParse } from './_fetchAndParse.js';
 import { ollamaBaseUrl, ollamaHeaders, selectOllamaModel } from '../../utils/ollamaConfig.js';
+import { verifyNumericProvenance } from '../../utils/provenance.js';
 // D1.3: SamplingClient for MCP sampling fallback (lazy — only imported if needed)
 let _SamplingClient = null;
 async function getSamplingClient() {
@@ -485,6 +486,7 @@ export class ExtractWithLlm {
    * @param {number}  [params.maxTokens] - Max output tokens (default 4096)
    * @param {boolean} [params.respect_robots] - Per-request robots.txt override
    * @param {string}  [params.user_agent]     - Per-request identity override
+   * @param {boolean} [params.verify_numbers] - Numeric provenance guard (default true)
    * @returns {Promise<Object>}
    */
   async execute(params) {
@@ -497,7 +499,8 @@ export class ExtractWithLlm {
       model: modelParam,
       maxTokens = 4096,
       respect_robots,
-      user_agent
+      user_agent,
+      verify_numbers = true
     } = params;
 
     // Validate: exactly one of url or content must be provided
@@ -530,18 +533,25 @@ export class ExtractWithLlm {
 
     // Step 1: Get text to extract from
     let text;
+    // What the provenance guard checks against. Deliberately wider than what
+    // the model is shown: the raw html carries numbers the flattened text does
+    // not (Apple's prices exist only inside an embedded JSON blob), and a value
+    // missing from `text` but present on the page must not be nulled.
+    let sourceForProvenance;
     let fetchWarnings = [];
     try {
       if (url) {
-        const { textContent, warnings } = await fetchAndParse(url, {
+        const { html, textContent, warnings } = await fetchAndParse(url, {
           respectRobots: respect_robots,
           userAgent: user_agent,
           tool: 'extract_with_llm'
         });
         text = textContent;
+        sourceForProvenance = `${html}\n${textContent}`;
         fetchWarnings = warnings || [];
       } else {
         text = content;
+        sourceForProvenance = content;
       }
     } catch (fetchErr) {
       return { success: false, error: `Failed to fetch content: ${fetchErr.message}` };
@@ -654,10 +664,27 @@ export class ExtractWithLlm {
       }
     }
 
+    // 3.4: numeric provenance. A number the model wrote that is nowhere in the
+    // page it was given was invented, so it comes back null with a reason
+    // rather than as a confident answer.
+    let provenance = { enabled: false };
+    if (verify_numbers) {
+      const checked = verifyNumericProvenance(parsed, sourceForProvenance);
+      parsed = checked.data;
+      provenance = {
+        enabled: true,
+        verified: checked.verified,
+        nulled: checked.nulled,
+        unverified: checked.unverified
+      };
+      if (checked.skipped) provenance.skipped = checked.skipped;
+    }
+
     // C3: surface truncation metadata so callers know the input was clipped
     const result = {
       success: true,
       data: parsed,
+      provenance,
       provider: resolvedModel === 'sampling' ? 'sampling' : provider,
       model: resolvedModel || model,
       usage

package/src/tools/templates/ScrapeTemplateTool.js

@@ -4,44 +4,113 @@
  * Usage pattern (D3.3):
  *   const tool = new ScrapeTemplateTool();
  *   const result = await tool.execute({ template: "github-repo", url: "https://github.com/user/repo" });
+ *
+ * Three ways in: a template id, `"auto"` (the registry picks the template from
+ * the URL and the response names the one it chose), and `"list"`. A list
+ * connector is driven by `params` rather than a URL and returns N entities from
+ * one call — the registry builds the URL, this tool fetches it.
  */
 
 import { TemplateRegistry } from 'crawlforge-extractors';
 import { safeFetch } from '../../utils/ssrfGuard.js';
 import { preflightFetch } from '../../utils/robotsGate.js';
 import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
+import { markPreflightRefusal } from '../../server/requestContext.js';
+
+/**
+ * A caller mistake caught before anything is fetched: no template matches the
+ * URL, a required list parameter is missing, a key-based connector has no key.
+ * We fetched nothing, so — like a robots refusal — it costs the caller nothing.
+ */
+function badRequest(message) {
+  markPreflightRefusal('BAD_REQUEST');
+  return new Error(message);
+}
 
 export class ScrapeTemplateTool {
-  constructor() {
-    this.registry = new TemplateRegistry();
+  /**
+   * @param {{ templates?: object[] }} [config] `templates` replaces the shipped
+   *   set; tests use it to exercise a connector shape nothing ships yet.
+   */
+  constructor(config = {}) {
+    this.registry = new TemplateRegistry(config?.templates);
+  }
+
+  /** The catalogue — no network. */
+  listTemplates() {
+    const templates = this.registry.list();
+    return { templates, count: templates.length };
   }
 
   /**
    * Execute the scrape_template tool.
-   * @param {{ template: string, url: string, timeout?: number,
-   *          user_agent?: string, respect_robots?: boolean }} params
+   * @param {{ template: string, url?: string, params?: object, timeout?: number,
+   *          user_agent?: string, respect_robots?: boolean }} request
    * @returns {Promise<object>}
    */
-  async execute({ template, url, timeout = 15000, user_agent, respect_robots }) {
-    // list mode — return available templates without scraping
-    if (template === 'list' || !url) {
-      return {
-        templates: this.registry.list(),
-        count: this.registry.list().length
-      };
+  async execute({ template, url, params, timeout = 15000, user_agent, respect_robots }) {
+    if (template === 'list') return this.listTemplates();
+
+    let templateId = template;
+    if (template === 'auto') {
+      if (!url) {
+        throw badRequest(
+          'template "auto" needs a url to detect from. Pass a url, or template:"list" to see every template.'
+        );
+      }
+      const detected = this.registry.detect(url);
+      if (!detected) {
+        throw badRequest(
+          `No template matches ${url}. Pass template:"list" to see every template and the URLs ` +
+          'each one handles, or name a template explicitly.'
+        );
+      }
+      templateId = detected.id;
+    } else if (!url && !params) {
+      // A template named with nothing to run it against still lists, as before.
+      return this.listTemplates();
     }
 
     // Validate template exists before making network call
-    const tpl = this.registry.get(template);
+    const tpl = this.registry.get(templateId);
     if (!tpl) {
       const available = this.registry.list().map(t => t.id).join(', ');
-      throw new Error(`Unknown template "${template}". Available templates: ${available}`);
+      throw new Error(`Unknown template "${templateId}". Available templates: ${available}`);
+    }
+
+    // A key-based connector is answered here or not at all: the registry never
+    // reads process.env, and a missing key must not reach the target as a 401.
+    let apiKey;
+    if (tpl.requiresApiKey) {
+      apiKey = process.env[tpl.credentialRef];
+      if (!apiKey) {
+        throw badRequest(
+          `Template "${templateId}" reads an API-keyed endpoint. Set ${tpl.credentialRef} in the ` +
+          'server environment and try again.'
+        );
+      }
     }
 
-    // A template may redirect its own fetch to a machine-readable endpoint
-    // (shopify-product reads /products/<handle>.json). Same host either way,
-    // so the SSRF guard below still applies.
-    const fetchUrl = template === 'list' ? url : (tpl.resolveUrl ? tpl.resolveUrl(url) : url);
+    // The URL actually fetched comes from one of three places. The robots gate
+    // below runs against that URL, never the caller's input — listUrl in
+    // particular reaches a host the caller never named.
+    let fetchUrl;
+    if (params && tpl.listUrl) {
+      try {
+        fetchUrl = tpl.listUrl(apiKey ? { ...params, apiKey } : params);
+      } catch (error) {
+        // listUrl throws naming the parameter it wanted: a caller mistake, not
+        // a fetch that failed.
+        throw badRequest(error.message);
+      }
+    } else if (!url) {
+      throw badRequest(`Template "${templateId}" is reached by url, not params. Pass a url.`);
+    } else {
+      // A template may redirect its own fetch to a machine-readable endpoint
+      // (shopify-product reads /products/<handle>.json). Same host either way,
+      // so the SSRF guard below still applies.
+      fetchUrl = tpl.resolveUrl ? tpl.resolveUrl(url) : url;
+    }
 
     // Robots gate + per-host politeness before any request to the target.
     const gate = await preflightFetch(fetchUrl, {
@@ -76,8 +145,28 @@ export class ScrapeTemplateTool {
       throw error;
     }
 
+    // What the response reports. A params-driven call named no URL, so the one
+    // we built is the one to report — and the one the extractor quotes in its
+    // own error messages. Except on a key-based connector, where that URL
+    // carries the key: it goes into listUrl and nowhere else.
+    const keyed = Boolean(tpl.requiresApiKey);
+    const reportedUrl = url ?? (keyed ? undefined : fetchUrl);
+    const reportedFetchUrl = keyed ? reportedUrl : fetchUrl;
+
+    let echoParams;
+    if (params) {
+      echoParams = { ...params };
+      delete echoParams.apiKey;
+    }
+
     // Run the template extractor
-    const result = await this.registry.run(template, html, url, fetchUrl);
+    const result = tpl.extractList
+      ? await this.registry.runList(templateId, html, { url: reportedUrl, params: echoParams })
+      : await this.registry.run(templateId, html, reportedUrl, reportedFetchUrl);
+
+    // run() stamps fetchedUrl itself; runList() does not.
+    if (tpl.extractList && reportedFetchUrl !== reportedUrl) result.fetchedUrl = reportedFetchUrl;
+
     return gate.warnings.length > 0 ? { ...result, warnings: gate.warnings } : result;
   }
 }

package/src/utils/embeddedState.js

@@ -0,0 +1,304 @@
+/**
+ * embeddedState.js — find the JSON state a page already ships in its own HTML.
+ *
+ * Modern SPAs serialize the data their UI renders into the document: Next.js
+ * pages carry __NEXT_DATA__ or an RSC flight stream, Nuxt carries __NUXT__,
+ * Apollo/Redux apps carry __APOLLO_STATE__ / __INITIAL_STATE__ /
+ * __PRELOADED_STATE__. One fetch returns the exact values the site itself uses
+ * — no LLM in the extraction path, so no fabricated prices.
+ *
+ * Pure: HTML in, named payloads out. No fetching, no cheerio — every source
+ * lives inside a <script> tag, so one regex pass over script tags is enough
+ * and avoids a second full parse of a multi-megabyte document.
+ */
+
+/**
+ * Global variable assignments we look for, in the order they are reported.
+ * Each is matched with an optional window./self./globalThis. prefix.
+ * `name` is the path-safe key the caller addresses with a JSON path — the
+ * raw variable name has no dots, but keeping the two separate means a caller
+ * never has to guess how "self.__next_f" would be spelled in a path.
+ */
+const STATE_VARIABLES = [
+  { name: 'nuxt', variable: '__NUXT__' },
+  { name: 'apollo_state', variable: '__APOLLO_STATE__' },
+  { name: 'initial_state', variable: '__INITIAL_STATE__' },
+  { name: 'preloaded_state', variable: '__PRELOADED_STATE__' }
+];
+
+// Script bodies cannot contain a literal "</script", so a non-greedy match is
+// exact here — the same assumption every HTML parser makes.
+const SCRIPT_RE = /<script\b([^>]*)>([\s\S]*?)<\/script\s*>/gi;
+
+const HTML_COMMENT_RE = /<!--[\s\S]*?-->/g;
+
+// self.__next_f.push([1,"<chunk>"]) — Next.js App Router RSC flight chunks.
+const NEXT_F_PUSH_RE = /self\.__next_f\.push\(\s*\[\s*1\s*,\s*/g;
+
+/**
+ * Read an HTML attribute out of a raw tag's attribute string.
+ * @param {string} attrs
+ * @param {string} name
+ * @returns {string|null}
+ */
+function attr(attrs, name) {
+  const match = attrs.match(
+    new RegExp(`(?:^|\\s)${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s>]+))`, 'i')
+  );
+  if (!match) return null;
+  return match[1] ?? match[2] ?? match[3] ?? null;
+}
+
+/**
+ * Scan a double-quoted string literal starting at `start` and return its
+ * parsed value plus the index just past its closing quote.
+ * @param {string} text
+ * @param {number} start index of the opening quote
+ * @returns {{ value: string, end: number }|null}
+ */
+function readStringLiteral(text, start) {
+  if (text[start] !== '"') return null;
+  let i = start + 1;
+  while (i < text.length) {
+    const ch = text[i];
+    if (ch === '\\') { i += 2; continue; }
+    if (ch === '"') break;
+    i++;
+  }
+  if (i >= text.length) return null;
+  try {
+    return { value: JSON.parse(text.slice(start, i + 1)), end: i + 1 };
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Scan a balanced {...} / [...] literal starting at `start`, respecting string
+ * literals and escapes so braces inside strings don't end it early.
+ * @param {string} text
+ * @param {number} start
+ * @returns {string|null} the raw literal, or null if it never closes
+ */
+function readBracketedLiteral(text, start) {
+  const open = text[start];
+  const close = open === '{' ? '}' : open === '[' ? ']' : null;
+  if (!close) return null;
+  let depth = 0;
+  let i = start;
+  while (i < text.length) {
+    const ch = text[i];
+    if (ch === '"' || ch === "'") {
+      const quote = ch;
+      i++;
+      while (i < text.length && text[i] !== quote) {
+        if (text[i] === '\\') i++;
+        i++;
+      }
+    } else if (ch === open) {
+      depth++;
+    } else if (ch === close) {
+      depth--;
+      if (depth === 0) return text.slice(start, i + 1);
+    }
+    i++;
+  }
+  return null;
+}
+
+/**
+ * Split a concatenated RSC flight stream into its rows.
+ *
+ * The stream is a sequence of `<hexId>:<payload>\n` rows. Three payload shapes
+ * matter:
+ *   - `T<hexByteLength>,` — a length-prefixed text blob. The length is in
+ *     UTF-8 BYTES and INCLUDES the row's terminating newline, so the cursor
+ *     advances exactly that many bytes and no further. (Advancing one extra
+ *     character for a newline silently eats the first hex digit of the next
+ *     row id, turning row "14" into row "4" and overwriting an unrelated row —
+ *     verified against the live Healthgrades capture, where it produced seven
+ *     colliding ids.)
+ *   - `I[...]` / `HL[...]` — module and hint references. Not JSON; kept as the
+ *     raw string so the caller can still see which components a page loads.
+ *   - anything else — JSON, parsed.
+ *
+ * @param {string} stream
+ * @returns {Record<string, unknown>} row id -> value
+ */
+function parseFlightRows(stream) {
+  const rows = {};
+  let cursor = 0;
+
+  while (cursor < stream.length) {
+    const newline = stream.indexOf('\n', cursor);
+    const lineEnd = newline === -1 ? stream.length : newline;
+    const header = stream.slice(cursor, lineEnd).match(/^([0-9a-f]+):/i);
+    if (!header) {
+      // Not a row start: a chunk boundary landed mid-row, or the stream was
+      // truncated. Resync on the next line rather than giving up.
+      cursor = lineEnd + 1;
+      continue;
+    }
+
+    const id = header[1];
+    const payloadStart = cursor + header[0].length;
+    const payload = stream.slice(payloadStart, lineEnd);
+
+    const textRow = payload.match(/^T([0-9a-f]+),/i);
+    if (textRow) {
+      const blobStart = payloadStart + textRow[0].length;
+      const text = Buffer.from(stream.slice(blobStart), 'utf8')
+        .subarray(0, parseInt(textRow[1], 16))
+        .toString('utf8');
+      rows[id] = text;
+      cursor = blobStart + text.length;
+      continue;
+    }
+
+    try {
+      rows[id] = JSON.parse(payload);
+    } catch {
+      rows[id] = payload;
+    }
+    cursor = lineEnd + 1;
+  }
+
+  return rows;
+}
+
+/**
+ * Collect every `self.__next_f.push([1,"…"])` chunk in document order and
+ * concatenate them into the flight stream they encode.
+ * @param {string} html
+ * @returns {{ chunks: number, stream: string }}
+ */
+function readFlightStream(html) {
+  const parts = [];
+  NEXT_F_PUSH_RE.lastIndex = 0;
+  let match;
+  while ((match = NEXT_F_PUSH_RE.exec(html)) !== null) {
+    const literal = readStringLiteral(html, NEXT_F_PUSH_RE.lastIndex);
+    if (!literal) continue;
+    parts.push(literal.value);
+    NEXT_F_PUSH_RE.lastIndex = literal.end;
+  }
+  return { chunks: parts.length, stream: parts.join('') };
+}
+
+const serializedBytes = (value) => Buffer.byteLength(JSON.stringify(value) ?? '');
+
+/**
+ * Extract every embedded state payload a page carries.
+ *
+ * @param {string} rawHtml raw HTML — NOT a script-stripped document
+ * @returns {{ data: Record<string, unknown>, found: Array<{name: string, variable: string, bytes: number, note?: string}>, warnings: string[] }}
+ */
+export function extractEmbeddedState(rawHtml) {
+  const data = {};
+  const found = [];
+  const warnings = [];
+  const jsonScripts = [];
+
+  // Commented-out markup is not state the page renders from, and an opening
+  // <script> tag inside a comment would otherwise match through to the first
+  // real </script> — swallowing the genuine tag that follows it.
+  const html = rawHtml.replace(HTML_COMMENT_RE, '');
+
+  SCRIPT_RE.lastIndex = 0;
+  let script;
+  while ((script = SCRIPT_RE.exec(html)) !== null) {
+    const [, attrs, body] = script;
+    const type = (attr(attrs, 'type') || '').toLowerCase();
+    if (type !== 'application/json') continue;
+
+    const id = attr(attrs, 'id');
+    let parsed;
+    try {
+      parsed = JSON.parse(body);
+    } catch {
+      warnings.push(
+        `A <script type="application/json"${id ? ` id="${id}"` : ''}> block is not valid JSON; skipped.`
+      );
+      continue;
+    }
+
+    // __NEXT_DATA__ is a JSON script too. Reported under its own name so a
+    // caller addressing "next_data" gets it, and not duplicated into
+    // json_scripts — on a Next.js page that would double a multi-hundred-KB
+    // payload.
+    if (id === '__NEXT_DATA__') {
+      data.next_data = parsed;
+      found.push({
+        name: 'next_data',
+        variable: '__NEXT_DATA__',
+        bytes: serializedBytes(parsed)
+      });
+    } else {
+      jsonScripts.push({ id: id || null, data: parsed });
+    }
+  }
+
+  const flight = readFlightStream(html);
+  if (flight.chunks > 0) {
+    const rows = parseFlightRows(flight.stream);
+    data.next_f = rows;
+    found.push({
+      name: 'next_f',
+      variable: 'self.__next_f',
+      bytes: serializedBytes(rows),
+      note: `${flight.chunks} RSC flight chunks concatenated into ${Object.keys(rows).length} rows, keyed by row id`
+    });
+  }
+
+  for (const { name, variable } of STATE_VARIABLES) {
+    const assignment = html.match(
+      new RegExp(`(?:window|self|globalThis)?\\.?\\b${variable}\\s*=\\s*`)
+    );
+    if (!assignment) continue;
+
+    const valueStart = assignment.index + assignment[0].length;
+    const literal = readBracketedLiteral(html, valueStart);
+    let parsed;
+    if (literal !== null) {
+      try {
+        parsed = JSON.parse(literal);
+      } catch {
+        parsed = undefined;
+      }
+    }
+
+    if (parsed === undefined) {
+      // Nuxt 2 wraps its payload in an IIFE, and Nuxt 3 emits a bare JS object
+      // literal with unquoted keys. Neither is JSON and neither is worth
+      // eval()ing — say so instead of reporting a source we did not read.
+      warnings.push(
+        `${variable} is present but its value is not a JSON literal (a JS object literal or function-wrapped payload); not parsed.`
+      );
+      continue;
+    }
+
+    // Nuxt 3 emits `window.__NUXT__={}` and then fills it from a second,
+    // non-JSON statement. Reporting the empty object without saying so reads
+    // like "this page has no Nuxt state", which is the opposite of true.
+    if (Object.keys(parsed).length === 0) {
+      warnings.push(
+        `${variable} is present but assigned an empty ${Array.isArray(parsed) ? 'array' : 'object'}; the page fills it in from a later statement this tool does not evaluate.`
+      );
+    }
+
+    data[name] = parsed;
+    found.push({ name, variable, bytes: serializedBytes(parsed) });
+  }
+
+  if (jsonScripts.length > 0) {
+    data.json_scripts = jsonScripts;
+    found.push({
+      name: 'json_scripts',
+      variable: 'script[type="application/json"]',
+      bytes: serializedBytes(jsonScripts),
+      note: `${jsonScripts.length} block(s), each { id, data }`
+    });
+  }
+
+  return { data, found, warnings };
+}

package/src/utils/jsonLd.js

@@ -0,0 +1,122 @@
+/**
+ * JSON-LD parsing and schema.org type filtering.
+ *
+ * Pure: no fetching. Callers pass a loaded cheerio document (parse) or already
+ * parsed blocks (filter).
+ */
+
+/**
+ * schema.org descendants that a filter on the parent type must also match.
+ *
+ * Real pages publish the specific subtype and almost never the parent, so an
+ * exact-string @type match returns nothing on exactly the pages callers ask
+ * about: ticketmaster.com/discover/concerts emits MusicEvent (never Event),
+ * apple.com/shop/buy-mac/macbook-air emits AggregateOffer and BreadcrumbList
+ * (never Offer or ItemList). Each list is the transitive descendant set of the
+ * key in schema.org v30. Types outside this table are matched exactly.
+ */
+export const JSON_LD_SUBTYPES = {
+  ItemList: ['BreadcrumbList', 'HowToSection', 'HowToStep', 'OfferCatalog'],
+  Product: [
+    'DietarySupplement', 'Drug', 'IndividualProduct', 'ProductCollection',
+    'ProductGroup', 'ProductModel', 'SomeProducts', 'Vehicle', 'BusOrCoach',
+    'Car', 'Motorcycle', 'MotorizedBicycle'
+  ],
+  Offer: ['AggregateOffer', 'OfferForLease', 'OfferForPurchase'],
+  Event: [
+    'BusinessEvent', 'ChildrensEvent', 'ComedyEvent', 'CourseInstance',
+    'DanceEvent', 'DeliveryEvent', 'EducationEvent', 'EventSeries',
+    'ExhibitionEvent', 'Festival', 'FoodEvent', 'Hackathon', 'LiteraryEvent',
+    'MusicEvent', 'PublicationEvent', 'BroadcastEvent', 'OnDemandEvent',
+    'SaleEvent', 'ScreeningEvent', 'SocialEvent', 'SportsEvent', 'TheaterEvent',
+    'VisualArtsEvent'
+  ],
+  JobPosting: [],
+  RealEstateListing: []
+};
+
+// Lowercased filter name → set of lowercased @type values it accepts.
+const MATCH_SETS = new Map(
+  Object.entries(JSON_LD_SUBTYPES).map(([parent, subtypes]) => [
+    parent.toLowerCase(),
+    new Set([parent, ...subtypes].map((t) => t.toLowerCase()))
+  ])
+);
+
+/**
+ * Some publishers write @type as a full IRI ("https://schema.org/Product").
+ * @param {unknown} value
+ * @returns {string|null}
+ */
+function normalizeType(value) {
+  if (typeof value !== 'string') return null;
+  return value.replace(/^https?:\/\/schema\.org\//, '').trim().toLowerCase() || null;
+}
+
+/**
+ * Parse all JSON-LD blocks from the document. A malformed block is skipped so
+ * one bad block does not lose the good ones.
+ * @param {import('cheerio').CheerioAPI} $
+ * @returns {Array}
+ */
+export function parseJsonLd($) {
+  const results = [];
+  $('script[type="application/ld+json"]').each((_, el) => {
+    try {
+      const raw = $(el).html();
+      if (raw) results.push(JSON.parse(raw));
+    } catch {
+      // Skip invalid blocks
+    }
+  });
+  return results;
+}
+
+/**
+ * Collect the JSON-LD nodes matching the requested schema.org types.
+ *
+ * Nodes are found at any depth, so @graph wrappers, top-level arrays and types
+ * nested inside another node (an Offer inside an Event) are all reachable.
+ * @type itself may be a string or an array of strings.
+ *
+ * @param {Array} blocks - parsed JSON-LD blocks, as returned by parseJsonLd
+ * @param {string[]} types - schema.org type names to keep
+ * @returns {{ items: Array, counts: Record<string, number> }} matching nodes in
+ *   document order, and how many matched per requested type
+ */
+export function filterJsonLdByType(blocks, types) {
+  const wanted = types.map((requested) => ({
+    requested,
+    match: MATCH_SETS.get(requested.toLowerCase()) || new Set([requested.toLowerCase()])
+  }));
+  const counts = Object.fromEntries(types.map((t) => [t, 0]));
+  const items = [];
+
+  const visit = (node) => {
+    if (Array.isArray(node)) {
+      node.forEach(visit);
+      return;
+    }
+    if (!node || typeof node !== 'object') return;
+
+    const raw = node['@type'];
+    const nodeTypes = (Array.isArray(raw) ? raw : [raw]).map(normalizeType).filter(Boolean);
+    if (nodeTypes.length) {
+      let matched = false;
+      for (const { requested, match } of wanted) {
+        if (nodeTypes.some((t) => match.has(t))) {
+          counts[requested] += 1;
+          matched = true;
+        }
+      }
+      if (matched) items.push(node);
+    }
+
+    // Descend even into a matched node: its children may match another
+    // requested type (Ticketmaster nests each Offer inside its MusicEvent).
+    for (const value of Object.values(node)) visit(value);
+  };
+
+  blocks.forEach(visit);
+  return { items, counts };
+}