crawlforge-mcp-server 5.3.0 → 5.4.0

package/src/utils/embeddedState.js

@@ -0,0 +1,304 @@
+/**
+ * embeddedState.js — find the JSON state a page already ships in its own HTML.
+ *
+ * Modern SPAs serialize the data their UI renders into the document: Next.js
+ * pages carry __NEXT_DATA__ or an RSC flight stream, Nuxt carries __NUXT__,
+ * Apollo/Redux apps carry __APOLLO_STATE__ / __INITIAL_STATE__ /
+ * __PRELOADED_STATE__. One fetch returns the exact values the site itself uses
+ * — no LLM in the extraction path, so no fabricated prices.
+ *
+ * Pure: HTML in, named payloads out. No fetching, no cheerio — every source
+ * lives inside a <script> tag, so one regex pass over script tags is enough
+ * and avoids a second full parse of a multi-megabyte document.
+ */
+
+/**
+ * Global variable assignments we look for, in the order they are reported.
+ * Each is matched with an optional window./self./globalThis. prefix.
+ * `name` is the path-safe key the caller addresses with a JSON path — the
+ * raw variable name has no dots, but keeping the two separate means a caller
+ * never has to guess how "self.__next_f" would be spelled in a path.
+ */
+const STATE_VARIABLES = [
+  { name: 'nuxt', variable: '__NUXT__' },
+  { name: 'apollo_state', variable: '__APOLLO_STATE__' },
+  { name: 'initial_state', variable: '__INITIAL_STATE__' },
+  { name: 'preloaded_state', variable: '__PRELOADED_STATE__' }
+];
+
+// Script bodies cannot contain a literal "</script", so a non-greedy match is
+// exact here — the same assumption every HTML parser makes.
+const SCRIPT_RE = /<script\b([^>]*)>([\s\S]*?)<\/script\s*>/gi;
+
+const HTML_COMMENT_RE = /<!--[\s\S]*?-->/g;
+
+// self.__next_f.push([1,"<chunk>"]) — Next.js App Router RSC flight chunks.
+const NEXT_F_PUSH_RE = /self\.__next_f\.push\(\s*\[\s*1\s*,\s*/g;
+
+/**
+ * Read an HTML attribute out of a raw tag's attribute string.
+ * @param {string} attrs
+ * @param {string} name
+ * @returns {string|null}
+ */
+function attr(attrs, name) {
+  const match = attrs.match(
+    new RegExp(`(?:^|\\s)${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s>]+))`, 'i')
+  );
+  if (!match) return null;
+  return match[1] ?? match[2] ?? match[3] ?? null;
+}
+
+/**
+ * Scan a double-quoted string literal starting at `start` and return its
+ * parsed value plus the index just past its closing quote.
+ * @param {string} text
+ * @param {number} start index of the opening quote
+ * @returns {{ value: string, end: number }|null}
+ */
+function readStringLiteral(text, start) {
+  if (text[start] !== '"') return null;
+  let i = start + 1;
+  while (i < text.length) {
+    const ch = text[i];
+    if (ch === '\\') { i += 2; continue; }
+    if (ch === '"') break;
+    i++;
+  }
+  if (i >= text.length) return null;
+  try {
+    return { value: JSON.parse(text.slice(start, i + 1)), end: i + 1 };
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Scan a balanced {...} / [...] literal starting at `start`, respecting string
+ * literals and escapes so braces inside strings don't end it early.
+ * @param {string} text
+ * @param {number} start
+ * @returns {string|null} the raw literal, or null if it never closes
+ */
+function readBracketedLiteral(text, start) {
+  const open = text[start];
+  const close = open === '{' ? '}' : open === '[' ? ']' : null;
+  if (!close) return null;
+  let depth = 0;
+  let i = start;
+  while (i < text.length) {
+    const ch = text[i];
+    if (ch === '"' || ch === "'") {
+      const quote = ch;
+      i++;
+      while (i < text.length && text[i] !== quote) {
+        if (text[i] === '\\') i++;
+        i++;
+      }
+    } else if (ch === open) {
+      depth++;
+    } else if (ch === close) {
+      depth--;
+      if (depth === 0) return text.slice(start, i + 1);
+    }
+    i++;
+  }
+  return null;
+}
+
+/**
+ * Split a concatenated RSC flight stream into its rows.
+ *
+ * The stream is a sequence of `<hexId>:<payload>\n` rows. Three payload shapes
+ * matter:
+ *   - `T<hexByteLength>,` — a length-prefixed text blob. The length is in
+ *     UTF-8 BYTES and INCLUDES the row's terminating newline, so the cursor
+ *     advances exactly that many bytes and no further. (Advancing one extra
+ *     character for a newline silently eats the first hex digit of the next
+ *     row id, turning row "14" into row "4" and overwriting an unrelated row —
+ *     verified against the live Healthgrades capture, where it produced seven
+ *     colliding ids.)
+ *   - `I[...]` / `HL[...]` — module and hint references. Not JSON; kept as the
+ *     raw string so the caller can still see which components a page loads.
+ *   - anything else — JSON, parsed.
+ *
+ * @param {string} stream
+ * @returns {Record<string, unknown>} row id -> value
+ */
+function parseFlightRows(stream) {
+  const rows = {};
+  let cursor = 0;
+
+  while (cursor < stream.length) {
+    const newline = stream.indexOf('\n', cursor);
+    const lineEnd = newline === -1 ? stream.length : newline;
+    const header = stream.slice(cursor, lineEnd).match(/^([0-9a-f]+):/i);
+    if (!header) {
+      // Not a row start: a chunk boundary landed mid-row, or the stream was
+      // truncated. Resync on the next line rather than giving up.
+      cursor = lineEnd + 1;
+      continue;
+    }
+
+    const id = header[1];
+    const payloadStart = cursor + header[0].length;
+    const payload = stream.slice(payloadStart, lineEnd);
+
+    const textRow = payload.match(/^T([0-9a-f]+),/i);
+    if (textRow) {
+      const blobStart = payloadStart + textRow[0].length;
+      const text = Buffer.from(stream.slice(blobStart), 'utf8')
+        .subarray(0, parseInt(textRow[1], 16))
+        .toString('utf8');
+      rows[id] = text;
+      cursor = blobStart + text.length;
+      continue;
+    }
+
+    try {
+      rows[id] = JSON.parse(payload);
+    } catch {
+      rows[id] = payload;
+    }
+    cursor = lineEnd + 1;
+  }
+
+  return rows;
+}
+
+/**
+ * Collect every `self.__next_f.push([1,"…"])` chunk in document order and
+ * concatenate them into the flight stream they encode.
+ * @param {string} html
+ * @returns {{ chunks: number, stream: string }}
+ */
+function readFlightStream(html) {
+  const parts = [];
+  NEXT_F_PUSH_RE.lastIndex = 0;
+  let match;
+  while ((match = NEXT_F_PUSH_RE.exec(html)) !== null) {
+    const literal = readStringLiteral(html, NEXT_F_PUSH_RE.lastIndex);
+    if (!literal) continue;
+    parts.push(literal.value);
+    NEXT_F_PUSH_RE.lastIndex = literal.end;
+  }
+  return { chunks: parts.length, stream: parts.join('') };
+}
+
+const serializedBytes = (value) => Buffer.byteLength(JSON.stringify(value) ?? '');
+
+/**
+ * Extract every embedded state payload a page carries.
+ *
+ * @param {string} rawHtml raw HTML — NOT a script-stripped document
+ * @returns {{ data: Record<string, unknown>, found: Array<{name: string, variable: string, bytes: number, note?: string}>, warnings: string[] }}
+ */
+export function extractEmbeddedState(rawHtml) {
+  const data = {};
+  const found = [];
+  const warnings = [];
+  const jsonScripts = [];
+
+  // Commented-out markup is not state the page renders from, and an opening
+  // <script> tag inside a comment would otherwise match through to the first
+  // real </script> — swallowing the genuine tag that follows it.
+  const html = rawHtml.replace(HTML_COMMENT_RE, '');
+
+  SCRIPT_RE.lastIndex = 0;
+  let script;
+  while ((script = SCRIPT_RE.exec(html)) !== null) {
+    const [, attrs, body] = script;
+    const type = (attr(attrs, 'type') || '').toLowerCase();
+    if (type !== 'application/json') continue;
+
+    const id = attr(attrs, 'id');
+    let parsed;
+    try {
+      parsed = JSON.parse(body);
+    } catch {
+      warnings.push(
+        `A <script type="application/json"${id ? ` id="${id}"` : ''}> block is not valid JSON; skipped.`
+      );
+      continue;
+    }
+
+    // __NEXT_DATA__ is a JSON script too. Reported under its own name so a
+    // caller addressing "next_data" gets it, and not duplicated into
+    // json_scripts — on a Next.js page that would double a multi-hundred-KB
+    // payload.
+    if (id === '__NEXT_DATA__') {
+      data.next_data = parsed;
+      found.push({
+        name: 'next_data',
+        variable: '__NEXT_DATA__',
+        bytes: serializedBytes(parsed)
+      });
+    } else {
+      jsonScripts.push({ id: id || null, data: parsed });
+    }
+  }
+
+  const flight = readFlightStream(html);
+  if (flight.chunks > 0) {
+    const rows = parseFlightRows(flight.stream);
+    data.next_f = rows;
+    found.push({
+      name: 'next_f',
+      variable: 'self.__next_f',
+      bytes: serializedBytes(rows),
+      note: `${flight.chunks} RSC flight chunks concatenated into ${Object.keys(rows).length} rows, keyed by row id`
+    });
+  }
+
+  for (const { name, variable } of STATE_VARIABLES) {
+    const assignment = html.match(
+      new RegExp(`(?:window|self|globalThis)?\\.?\\b${variable}\\s*=\\s*`)
+    );
+    if (!assignment) continue;
+
+    const valueStart = assignment.index + assignment[0].length;
+    const literal = readBracketedLiteral(html, valueStart);
+    let parsed;
+    if (literal !== null) {
+      try {
+        parsed = JSON.parse(literal);
+      } catch {
+        parsed = undefined;
+      }
+    }
+
+    if (parsed === undefined) {
+      // Nuxt 2 wraps its payload in an IIFE, and Nuxt 3 emits a bare JS object
+      // literal with unquoted keys. Neither is JSON and neither is worth
+      // eval()ing — say so instead of reporting a source we did not read.
+      warnings.push(
+        `${variable} is present but its value is not a JSON literal (a JS object literal or function-wrapped payload); not parsed.`
+      );
+      continue;
+    }
+
+    // Nuxt 3 emits `window.__NUXT__={}` and then fills it from a second,
+    // non-JSON statement. Reporting the empty object without saying so reads
+    // like "this page has no Nuxt state", which is the opposite of true.
+    if (Object.keys(parsed).length === 0) {
+      warnings.push(
+        `${variable} is present but assigned an empty ${Array.isArray(parsed) ? 'array' : 'object'}; the page fills it in from a later statement this tool does not evaluate.`
+      );
+    }
+
+    data[name] = parsed;
+    found.push({ name, variable, bytes: serializedBytes(parsed) });
+  }
+
+  if (jsonScripts.length > 0) {
+    data.json_scripts = jsonScripts;
+    found.push({
+      name: 'json_scripts',
+      variable: 'script[type="application/json"]',
+      bytes: serializedBytes(jsonScripts),
+      note: `${jsonScripts.length} block(s), each { id, data }`
+    });
+  }
+
+  return { data, found, warnings };
+}

package/src/utils/jsonLd.js

@@ -0,0 +1,122 @@
+/**
+ * JSON-LD parsing and schema.org type filtering.
+ *
+ * Pure: no fetching. Callers pass a loaded cheerio document (parse) or already
+ * parsed blocks (filter).
+ */
+
+/**
+ * schema.org descendants that a filter on the parent type must also match.
+ *
+ * Real pages publish the specific subtype and almost never the parent, so an
+ * exact-string @type match returns nothing on exactly the pages callers ask
+ * about: ticketmaster.com/discover/concerts emits MusicEvent (never Event),
+ * apple.com/shop/buy-mac/macbook-air emits AggregateOffer and BreadcrumbList
+ * (never Offer or ItemList). Each list is the transitive descendant set of the
+ * key in schema.org v30. Types outside this table are matched exactly.
+ */
+export const JSON_LD_SUBTYPES = {
+  ItemList: ['BreadcrumbList', 'HowToSection', 'HowToStep', 'OfferCatalog'],
+  Product: [
+    'DietarySupplement', 'Drug', 'IndividualProduct', 'ProductCollection',
+    'ProductGroup', 'ProductModel', 'SomeProducts', 'Vehicle', 'BusOrCoach',
+    'Car', 'Motorcycle', 'MotorizedBicycle'
+  ],
+  Offer: ['AggregateOffer', 'OfferForLease', 'OfferForPurchase'],
+  Event: [
+    'BusinessEvent', 'ChildrensEvent', 'ComedyEvent', 'CourseInstance',
+    'DanceEvent', 'DeliveryEvent', 'EducationEvent', 'EventSeries',
+    'ExhibitionEvent', 'Festival', 'FoodEvent', 'Hackathon', 'LiteraryEvent',
+    'MusicEvent', 'PublicationEvent', 'BroadcastEvent', 'OnDemandEvent',
+    'SaleEvent', 'ScreeningEvent', 'SocialEvent', 'SportsEvent', 'TheaterEvent',
+    'VisualArtsEvent'
+  ],
+  JobPosting: [],
+  RealEstateListing: []
+};
+
+// Lowercased filter name → set of lowercased @type values it accepts.
+const MATCH_SETS = new Map(
+  Object.entries(JSON_LD_SUBTYPES).map(([parent, subtypes]) => [
+    parent.toLowerCase(),
+    new Set([parent, ...subtypes].map((t) => t.toLowerCase()))
+  ])
+);
+
+/**
+ * Some publishers write @type as a full IRI ("https://schema.org/Product").
+ * @param {unknown} value
+ * @returns {string|null}
+ */
+function normalizeType(value) {
+  if (typeof value !== 'string') return null;
+  return value.replace(/^https?:\/\/schema\.org\//, '').trim().toLowerCase() || null;
+}
+
+/**
+ * Parse all JSON-LD blocks from the document. A malformed block is skipped so
+ * one bad block does not lose the good ones.
+ * @param {import('cheerio').CheerioAPI} $
+ * @returns {Array}
+ */
+export function parseJsonLd($) {
+  const results = [];
+  $('script[type="application/ld+json"]').each((_, el) => {
+    try {
+      const raw = $(el).html();
+      if (raw) results.push(JSON.parse(raw));
+    } catch {
+      // Skip invalid blocks
+    }
+  });
+  return results;
+}
+
+/**
+ * Collect the JSON-LD nodes matching the requested schema.org types.
+ *
+ * Nodes are found at any depth, so @graph wrappers, top-level arrays and types
+ * nested inside another node (an Offer inside an Event) are all reachable.
+ * @type itself may be a string or an array of strings.
+ *
+ * @param {Array} blocks - parsed JSON-LD blocks, as returned by parseJsonLd
+ * @param {string[]} types - schema.org type names to keep
+ * @returns {{ items: Array, counts: Record<string, number> }} matching nodes in
+ *   document order, and how many matched per requested type
+ */
+export function filterJsonLdByType(blocks, types) {
+  const wanted = types.map((requested) => ({
+    requested,
+    match: MATCH_SETS.get(requested.toLowerCase()) || new Set([requested.toLowerCase()])
+  }));
+  const counts = Object.fromEntries(types.map((t) => [t, 0]));
+  const items = [];
+
+  const visit = (node) => {
+    if (Array.isArray(node)) {
+      node.forEach(visit);
+      return;
+    }
+    if (!node || typeof node !== 'object') return;
+
+    const raw = node['@type'];
+    const nodeTypes = (Array.isArray(raw) ? raw : [raw]).map(normalizeType).filter(Boolean);
+    if (nodeTypes.length) {
+      let matched = false;
+      for (const { requested, match } of wanted) {
+        if (nodeTypes.some((t) => match.has(t))) {
+          counts[requested] += 1;
+          matched = true;
+        }
+      }
+      if (matched) items.push(node);
+    }
+
+    // Descend even into a matched node: its children may match another
+    // requested type (Ticketmaster nests each Offer inside its MusicEvent).
+    for (const value of Object.values(node)) visit(value);
+  };
+
+  blocks.forEach(visit);
+  return { items, counts };
+}

package/src/utils/jsonPath.js

@@ -0,0 +1,80 @@
+/**
+ * jsonPath.js — the minimal subtree selector for extract_embedded_state.
+ *
+ * Deliberately not a JSONPath engine: dotted keys and array indexes only, no
+ * wildcards, filters, slices or recursive descent. That is enough to turn a
+ * multi-megabyte state blob into the branch a caller actually wants, and a
+ * caller who needs more can select a branch and filter it themselves.
+ *
+ * Syntax: `next_data.props.pageProps`, `json_scripts.0.data`, `next_f[1a]`,
+ * `a.b[0].c`. A key containing a literal "." cannot be addressed.
+ */
+
+/**
+ * Split a path into its segments.
+ * @param {string} path
+ * @returns {string[]}
+ */
+export function parseJsonPath(path) {
+  const segments = [];
+  for (const part of path.split('.')) {
+    // "a[0][1]" -> "a", "0", "1"
+    const [head, ...brackets] = part.split('[');
+    if (head !== '') segments.push(head);
+    for (const bracket of brackets) {
+      const key = bracket.endsWith(']') ? bracket.slice(0, -1) : bracket;
+      segments.push(key.replace(/^['"]|['"]$/g, ''));
+    }
+  }
+  return segments.filter((segment) => segment !== '');
+}
+
+/**
+ * Describe what a caller could have asked for at a dead end, so a typo comes
+ * back as a fixable message rather than an empty result.
+ * @param {unknown} value
+ * @returns {string}
+ */
+function describeOptions(value) {
+  if (Array.isArray(value)) return `array of length ${value.length}`;
+  if (value !== null && typeof value === 'object') {
+    const keys = Object.keys(value);
+    const shown = keys.slice(0, 25).join(', ');
+    return `available keys: ${shown}${keys.length > 25 ? `, … (${keys.length} total)` : ''}`;
+  }
+  return `a ${value === null ? 'null' : typeof value} value, which has no keys`;
+}
+
+/**
+ * Resolve a path against a parsed object.
+ *
+ * @param {unknown} root
+ * @param {string} path
+ * @returns {unknown}
+ * @throws {Error} when the path does not resolve — including the keys that
+ *   were available at the point it stopped
+ */
+export function selectJsonPath(root, path) {
+  const segments = parseJsonPath(path);
+  if (segments.length === 0) {
+    throw new Error(`Path "${path}" is empty`);
+  }
+
+  let current = root;
+  const walked = [];
+
+  for (const segment of segments) {
+    const container = current !== null && typeof current === 'object';
+    const key = Array.isArray(current) ? Number(segment) : segment;
+    if (!container || !(key in current)) {
+      const at = walked.length === 0 ? 'the result' : `"${walked.join('.')}"`;
+      throw new Error(
+        `Path "${path}" not found: ${at} has no "${segment}" (${describeOptions(current)})`
+      );
+    }
+    current = current[key];
+    walked.push(segment);
+  }
+
+  return current;
+}

package/src/utils/ollamaConfig.js

@@ -55,6 +55,32 @@ const PREFERRED_MODELS = [
   'qwen2.5:3b'
 ];
 
+/**
+ * Models measured fit to JUDGE claims — relevance to a topic, same-meaning
+ * grouping, and contradiction — as opposed to extracting fields. Measured
+ * 2026-08-28 by replaying a live deep_research run's own 136 claims through
+ * each installed model, three runs each:
+ *
+ *   gemma3:12b   0 false contradictions on 27 real pairs, 3/3 planted caught,
+ *                7-9 cross-source groups (the 4B model: 1-2 false, 0-1/3
+ *                caught, 1 group)
+ *   gemma3:4b    the extraction winner, but it scored "Playwright vs Selenium"
+ *                marketing 0.9 relevant to an anti-bot topic and put it in the
+ *                research summary
+ *   gemma4:31b   judged as cleanly as gemma3:12b but only with thinking turned
+ *                off — under the default it spends the whole token budget on
+ *                hidden reasoning and returns empty content — and it grouped so
+ *                strictly that consensus vanished. Not ranked.
+ *   gpt-oss:20b  empty content at these token budgets for the same reason,
+ *                and `think: false` makes it emit nothing at all. Not ranked.
+ *
+ * Membership here is what turns conflict detection on: a model that invents
+ * disagreement between sources that agree is worse than one that reports none,
+ * so a model absent from this list is never asked. When none is installed the
+ * judgement role falls through to the extraction ranking above.
+ */
+export const JUDGEMENT_MODELS = ['gemma3:12b'];
+
 /** Used only when Ollama cannot be reached, so the error names a real model. */
 export const FALLBACK_OLLAMA_MODEL = 'llama3.2';
 
@@ -102,9 +128,11 @@ export async function installedOllamaModels() {
  * instead would break anyone who has not pulled it, so the best *installed*
  * model is chosen, and an explicit OLLAMA_DEFAULT_MODEL always wins.
  *
+ * @param {'default'|'judgement'} [role] 'judgement' tries JUDGEMENT_MODELS
+ *   first and falls through to the extraction ranking when none is installed.
  * @returns {Promise<string>}
  */
-export async function selectOllamaModel() {
+export async function selectOllamaModel(role = 'default') {
   const explicit = process.env.OLLAMA_DEFAULT_MODEL;
   if (explicit) return explicit;
 
@@ -112,10 +140,16 @@ export async function selectOllamaModel() {
   if (installed.length === 0) return FALLBACK_OLLAMA_MODEL;
 
   const byBase = new Map(installed.map((name) => [baseName(name), name]));
-  for (const preferred of PREFERRED_MODELS) {
+  const ranking = role === 'judgement' ? [...JUDGEMENT_MODELS, ...PREFERRED_MODELS] : PREFERRED_MODELS;
+  for (const preferred of ranking) {
     const match = byBase.get(baseName(preferred));
     if (match) return match;
   }
   // Nothing recognised — use whatever is there rather than failing.
   return installed[0];
 }
+
+/** Whether a model name is one measured fit to judge contradictions. */
+export function isJudgementModel(name) {
+  return typeof name === 'string' && JUDGEMENT_MODELS.some((m) => baseName(m) === baseName(name));
+}