crawlforge-extractors 1.3.0 → 1.4.0

package/README.md

@@ -1,7 +1,8 @@
 # crawlforge-extractors
 
 Extraction logic shared by the [CrawlForge](https://www.crawlforge.dev) MCP server and REST API:
-site-specific scrape templates, response body reading, and structural fingerprinting.
+site-specific scrape templates, response body reading, structural fingerprinting, and
+embedded-state extraction.
 
 ## Why this package exists
 
@@ -134,6 +135,36 @@ A signature is the page's tag vocabulary plus its element-count-by-depth
 histogram — a few dozen keys, small enough to store next to a change-tracking
 baseline instead of keeping the whole DOM.
 
+### Reading a page's embedded state
+
+`extractEmbeddedState` returns the JSON a page already ships in its own HTML —
+`__NEXT_DATA__`, RSC flight chunks (`self.__next_f`), `__NUXT__`,
+`__APOLLO_STATE__`, `__INITIAL_STATE__`, `__PRELOADED_STATE__` and
+`<script type="application/json">` blocks. No LLM is involved, so the values are
+the site's own and cannot be fabricated.
+
+```js
+import { extractEmbeddedState, selectJsonPath } from 'crawlforge-extractors';
+
+const { data, found, warnings } = extractEmbeddedState(rawHtml);
+// found -> [{ name: 'next_data', variable: '__NEXT_DATA__', bytes: 439333 }]
+
+selectJsonPath(data, 'next_data.props.pageProps.events.0.name');
+```
+
+Pass the **raw** HTML. Every source lives in a `<script>` tag, so a document
+whose scripts have been stripped has nothing left to read.
+
+Payloads are never truncated — a half-serialized object is worse than a big
+one. `selectJsonPath` is how a caller asks for less: dotted keys and array
+indexes only, no wildcards, filters or recursive descent. A path that does not
+resolve throws naming the keys that *were* available at the point it stopped,
+so a typo comes back fixable rather than empty.
+
+A source that is present but is not JSON — Nuxt 2's IIFE wrapper, Nuxt 3's
+unquoted-key object literal — is reported in `warnings` unparsed. Nothing here
+calls `eval`.
+
 ## Templates
 
 **Pages and products.** `shopify-product` · `shopify-collection` ·

package/index.d.ts

@@ -176,4 +176,43 @@ export declare function structuralSimilarity(
   current: Partial<StructureSignature> | null | undefined
 ): number;
 
+/** One embedded-state payload a page carries, as reported in `found`. */
+export interface EmbeddedStateSource {
+  /** Path-safe key this payload is addressed by, e.g. "next_data". */
+  name: string;
+  /** The raw thing it was read from, e.g. "__NEXT_DATA__", "self.__next_f". */
+  variable: string;
+  /** Serialized size of this payload alone. */
+  bytes: number;
+  /** Present when the shape needs explaining (RSC rows, json_scripts blocks). */
+  note?: string;
+}
+
+export interface EmbeddedStateResult {
+  /** Payloads keyed by `name`; empty when the page ships no readable state. */
+  data: Record<string, unknown>;
+  found: EmbeddedStateSource[];
+  /** Sources seen but not parsed, and blocks that were not valid JSON. */
+  warnings: string[];
+}
+
+/**
+ * Find the JSON state a page already ships in its own HTML: __NEXT_DATA__,
+ * RSC flight chunks (self.__next_f), __NUXT__, __APOLLO_STATE__,
+ * __INITIAL_STATE__, __PRELOADED_STATE__ and <script type="application/json">.
+ *
+ * Pass the RAW html. A script-stripped document has nothing left to read.
+ */
+export declare function extractEmbeddedState(rawHtml: string): EmbeddedStateResult;
+
+/** Split a path into its segments. Dotted keys and array indexes only. */
+export declare function parseJsonPath(path: string): string[];
+
+/**
+ * Resolve a path against a parsed object. Not JSONPath: no wildcards, filters,
+ * slices or recursive descent. Throws naming the keys that were available at
+ * the point it stopped.
+ */
+export declare function selectJsonPath(root: unknown, path: string): unknown;
+
 export default TemplateRegistry;

package/index.js

@@ -25,3 +25,7 @@ export {
 } from './src/body.js';
 
 export { structureSignature, structuralSimilarity } from './src/structure.js';
+
+export { extractEmbeddedState } from './src/embeddedState.js';
+
+export { parseJsonPath, selectJsonPath } from './src/jsonPath.js';

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "crawlforge-extractors",
-  "version": "1.3.0",
-  "description": "Extraction logic shared by the CrawlForge MCP server and REST API — scrape templates, charset-correct capped body reading, and structural fingerprinting. One implementation, so the two surfaces cannot drift apart.",
+  "version": "1.4.0",
+  "description": "Extraction logic shared by the CrawlForge MCP server and REST API — scrape templates, charset-correct capped body reading, structural fingerprinting, and embedded-state extraction. One implementation, so the two surfaces cannot drift apart.",
   "type": "module",
   "main": "./index.js",
   "types": "./index.d.ts",

package/src/embeddedState.js

@@ -0,0 +1,304 @@
+/**
+ * embeddedState.js — find the JSON state a page already ships in its own HTML.
+ *
+ * Modern SPAs serialize the data their UI renders into the document: Next.js
+ * pages carry __NEXT_DATA__ or an RSC flight stream, Nuxt carries __NUXT__,
+ * Apollo/Redux apps carry __APOLLO_STATE__ / __INITIAL_STATE__ /
+ * __PRELOADED_STATE__. One fetch returns the exact values the site itself uses
+ * — no LLM in the extraction path, so no fabricated prices.
+ *
+ * Pure: HTML in, named payloads out. No fetching, no cheerio — every source
+ * lives inside a <script> tag, so one regex pass over script tags is enough
+ * and avoids a second full parse of a multi-megabyte document.
+ */
+
+/**
+ * Global variable assignments we look for, in the order they are reported.
+ * Each is matched with an optional window./self./globalThis. prefix.
+ * `name` is the path-safe key the caller addresses with a JSON path — the
+ * raw variable name has no dots, but keeping the two separate means a caller
+ * never has to guess how "self.__next_f" would be spelled in a path.
+ */
+const STATE_VARIABLES = [
+  { name: 'nuxt', variable: '__NUXT__' },
+  { name: 'apollo_state', variable: '__APOLLO_STATE__' },
+  { name: 'initial_state', variable: '__INITIAL_STATE__' },
+  { name: 'preloaded_state', variable: '__PRELOADED_STATE__' }
+];
+
+// Script bodies cannot contain a literal "</script", so a non-greedy match is
+// exact here — the same assumption every HTML parser makes.
+const SCRIPT_RE = /<script\b([^>]*)>([\s\S]*?)<\/script\s*>/gi;
+
+const HTML_COMMENT_RE = /<!--[\s\S]*?-->/g;
+
+// self.__next_f.push([1,"<chunk>"]) — Next.js App Router RSC flight chunks.
+const NEXT_F_PUSH_RE = /self\.__next_f\.push\(\s*\[\s*1\s*,\s*/g;
+
+/**
+ * Read an HTML attribute out of a raw tag's attribute string.
+ * @param {string} attrs
+ * @param {string} name
+ * @returns {string|null}
+ */
+function attr(attrs, name) {
+  const match = attrs.match(
+    new RegExp(`(?:^|\\s)${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s>]+))`, 'i')
+  );
+  if (!match) return null;
+  return match[1] ?? match[2] ?? match[3] ?? null;
+}
+
+/**
+ * Scan a double-quoted string literal starting at `start` and return its
+ * parsed value plus the index just past its closing quote.
+ * @param {string} text
+ * @param {number} start index of the opening quote
+ * @returns {{ value: string, end: number }|null}
+ */
+function readStringLiteral(text, start) {
+  if (text[start] !== '"') return null;
+  let i = start + 1;
+  while (i < text.length) {
+    const ch = text[i];
+    if (ch === '\\') { i += 2; continue; }
+    if (ch === '"') break;
+    i++;
+  }
+  if (i >= text.length) return null;
+  try {
+    return { value: JSON.parse(text.slice(start, i + 1)), end: i + 1 };
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Scan a balanced {...} / [...] literal starting at `start`, respecting string
+ * literals and escapes so braces inside strings don't end it early.
+ * @param {string} text
+ * @param {number} start
+ * @returns {string|null} the raw literal, or null if it never closes
+ */
+function readBracketedLiteral(text, start) {
+  const open = text[start];
+  const close = open === '{' ? '}' : open === '[' ? ']' : null;
+  if (!close) return null;
+  let depth = 0;
+  let i = start;
+  while (i < text.length) {
+    const ch = text[i];
+    if (ch === '"' || ch === "'") {
+      const quote = ch;
+      i++;
+      while (i < text.length && text[i] !== quote) {
+        if (text[i] === '\\') i++;
+        i++;
+      }
+    } else if (ch === open) {
+      depth++;
+    } else if (ch === close) {
+      depth--;
+      if (depth === 0) return text.slice(start, i + 1);
+    }
+    i++;
+  }
+  return null;
+}
+
+/**
+ * Split a concatenated RSC flight stream into its rows.
+ *
+ * The stream is a sequence of `<hexId>:<payload>\n` rows. Three payload shapes
+ * matter:
+ *   - `T<hexByteLength>,` — a length-prefixed text blob. The length is in
+ *     UTF-8 BYTES and INCLUDES the row's terminating newline, so the cursor
+ *     advances exactly that many bytes and no further. (Advancing one extra
+ *     character for a newline silently eats the first hex digit of the next
+ *     row id, turning row "14" into row "4" and overwriting an unrelated row —
+ *     verified against the live Healthgrades capture, where it produced seven
+ *     colliding ids.)
+ *   - `I[...]` / `HL[...]` — module and hint references. Not JSON; kept as the
+ *     raw string so the caller can still see which components a page loads.
+ *   - anything else — JSON, parsed.
+ *
+ * @param {string} stream
+ * @returns {Record<string, unknown>} row id -> value
+ */
+function parseFlightRows(stream) {
+  const rows = {};
+  let cursor = 0;
+
+  while (cursor < stream.length) {
+    const newline = stream.indexOf('\n', cursor);
+    const lineEnd = newline === -1 ? stream.length : newline;
+    const header = stream.slice(cursor, lineEnd).match(/^([0-9a-f]+):/i);
+    if (!header) {
+      // Not a row start: a chunk boundary landed mid-row, or the stream was
+      // truncated. Resync on the next line rather than giving up.
+      cursor = lineEnd + 1;
+      continue;
+    }
+
+    const id = header[1];
+    const payloadStart = cursor + header[0].length;
+    const payload = stream.slice(payloadStart, lineEnd);
+
+    const textRow = payload.match(/^T([0-9a-f]+),/i);
+    if (textRow) {
+      const blobStart = payloadStart + textRow[0].length;
+      const text = Buffer.from(stream.slice(blobStart), 'utf8')
+        .subarray(0, parseInt(textRow[1], 16))
+        .toString('utf8');
+      rows[id] = text;
+      cursor = blobStart + text.length;
+      continue;
+    }
+
+    try {
+      rows[id] = JSON.parse(payload);
+    } catch {
+      rows[id] = payload;
+    }
+    cursor = lineEnd + 1;
+  }
+
+  return rows;
+}
+
+/**
+ * Collect every `self.__next_f.push([1,"…"])` chunk in document order and
+ * concatenate them into the flight stream they encode.
+ * @param {string} html
+ * @returns {{ chunks: number, stream: string }}
+ */
+function readFlightStream(html) {
+  const parts = [];
+  NEXT_F_PUSH_RE.lastIndex = 0;
+  let match;
+  while ((match = NEXT_F_PUSH_RE.exec(html)) !== null) {
+    const literal = readStringLiteral(html, NEXT_F_PUSH_RE.lastIndex);
+    if (!literal) continue;
+    parts.push(literal.value);
+    NEXT_F_PUSH_RE.lastIndex = literal.end;
+  }
+  return { chunks: parts.length, stream: parts.join('') };
+}
+
+const serializedBytes = (value) => Buffer.byteLength(JSON.stringify(value) ?? '');
+
+/**
+ * Extract every embedded state payload a page carries.
+ *
+ * @param {string} rawHtml raw HTML — NOT a script-stripped document
+ * @returns {{ data: Record<string, unknown>, found: Array<{name: string, variable: string, bytes: number, note?: string}>, warnings: string[] }}
+ */
+export function extractEmbeddedState(rawHtml) {
+  const data = {};
+  const found = [];
+  const warnings = [];
+  const jsonScripts = [];
+
+  // Commented-out markup is not state the page renders from, and an opening
+  // <script> tag inside a comment would otherwise match through to the first
+  // real </script> — swallowing the genuine tag that follows it.
+  const html = rawHtml.replace(HTML_COMMENT_RE, '');
+
+  SCRIPT_RE.lastIndex = 0;
+  let script;
+  while ((script = SCRIPT_RE.exec(html)) !== null) {
+    const [, attrs, body] = script;
+    const type = (attr(attrs, 'type') || '').toLowerCase();
+    if (type !== 'application/json') continue;
+
+    const id = attr(attrs, 'id');
+    let parsed;
+    try {
+      parsed = JSON.parse(body);
+    } catch {
+      warnings.push(
+        `A <script type="application/json"${id ? ` id="${id}"` : ''}> block is not valid JSON; skipped.`
+      );
+      continue;
+    }
+
+    // __NEXT_DATA__ is a JSON script too. Reported under its own name so a
+    // caller addressing "next_data" gets it, and not duplicated into
+    // json_scripts — on a Next.js page that would double a multi-hundred-KB
+    // payload.
+    if (id === '__NEXT_DATA__') {
+      data.next_data = parsed;
+      found.push({
+        name: 'next_data',
+        variable: '__NEXT_DATA__',
+        bytes: serializedBytes(parsed)
+      });
+    } else {
+      jsonScripts.push({ id: id || null, data: parsed });
+    }
+  }
+
+  const flight = readFlightStream(html);
+  if (flight.chunks > 0) {
+    const rows = parseFlightRows(flight.stream);
+    data.next_f = rows;
+    found.push({
+      name: 'next_f',
+      variable: 'self.__next_f',
+      bytes: serializedBytes(rows),
+      note: `${flight.chunks} RSC flight chunks concatenated into ${Object.keys(rows).length} rows, keyed by row id`
+    });
+  }
+
+  for (const { name, variable } of STATE_VARIABLES) {
+    const assignment = html.match(
+      new RegExp(`(?:window|self|globalThis)?\\.?\\b${variable}\\s*=\\s*`)
+    );
+    if (!assignment) continue;
+
+    const valueStart = assignment.index + assignment[0].length;
+    const literal = readBracketedLiteral(html, valueStart);
+    let parsed;
+    if (literal !== null) {
+      try {
+        parsed = JSON.parse(literal);
+      } catch {
+        parsed = undefined;
+      }
+    }
+
+    if (parsed === undefined) {
+      // Nuxt 2 wraps its payload in an IIFE, and Nuxt 3 emits a bare JS object
+      // literal with unquoted keys. Neither is JSON and neither is worth
+      // eval()ing — say so instead of reporting a source we did not read.
+      warnings.push(
+        `${variable} is present but its value is not a JSON literal (a JS object literal or function-wrapped payload); not parsed.`
+      );
+      continue;
+    }
+
+    // Nuxt 3 emits `window.__NUXT__={}` and then fills it from a second,
+    // non-JSON statement. Reporting the empty object without saying so reads
+    // like "this page has no Nuxt state", which is the opposite of true.
+    if (Object.keys(parsed).length === 0) {
+      warnings.push(
+        `${variable} is present but assigned an empty ${Array.isArray(parsed) ? 'array' : 'object'}; the page fills it in from a later statement this tool does not evaluate.`
+      );
+    }
+
+    data[name] = parsed;
+    found.push({ name, variable, bytes: serializedBytes(parsed) });
+  }
+
+  if (jsonScripts.length > 0) {
+    data.json_scripts = jsonScripts;
+    found.push({
+      name: 'json_scripts',
+      variable: 'script[type="application/json"]',
+      bytes: serializedBytes(jsonScripts),
+      note: `${jsonScripts.length} block(s), each { id, data }`
+    });
+  }
+
+  return { data, found, warnings };
+}

package/src/jsonPath.js

@@ -0,0 +1,80 @@
+/**
+ * jsonPath.js — the minimal subtree selector for extract_embedded_state.
+ *
+ * Deliberately not a JSONPath engine: dotted keys and array indexes only, no
+ * wildcards, filters, slices or recursive descent. That is enough to turn a
+ * multi-megabyte state blob into the branch a caller actually wants, and a
+ * caller who needs more can select a branch and filter it themselves.
+ *
+ * Syntax: `next_data.props.pageProps`, `json_scripts.0.data`, `next_f[1a]`,
+ * `a.b[0].c`. A key containing a literal "." cannot be addressed.
+ */
+
+/**
+ * Split a path into its segments.
+ * @param {string} path
+ * @returns {string[]}
+ */
+export function parseJsonPath(path) {
+  const segments = [];
+  for (const part of path.split('.')) {
+    // "a[0][1]" -> "a", "0", "1"
+    const [head, ...brackets] = part.split('[');
+    if (head !== '') segments.push(head);
+    for (const bracket of brackets) {
+      const key = bracket.endsWith(']') ? bracket.slice(0, -1) : bracket;
+      segments.push(key.replace(/^['"]|['"]$/g, ''));
+    }
+  }
+  return segments.filter((segment) => segment !== '');
+}
+
+/**
+ * Describe what a caller could have asked for at a dead end, so a typo comes
+ * back as a fixable message rather than an empty result.
+ * @param {unknown} value
+ * @returns {string}
+ */
+function describeOptions(value) {
+  if (Array.isArray(value)) return `array of length ${value.length}`;
+  if (value !== null && typeof value === 'object') {
+    const keys = Object.keys(value);
+    const shown = keys.slice(0, 25).join(', ');
+    return `available keys: ${shown}${keys.length > 25 ? `, … (${keys.length} total)` : ''}`;
+  }
+  return `a ${value === null ? 'null' : typeof value} value, which has no keys`;
+}
+
+/**
+ * Resolve a path against a parsed object.
+ *
+ * @param {unknown} root
+ * @param {string} path
+ * @returns {unknown}
+ * @throws {Error} when the path does not resolve — including the keys that
+ *   were available at the point it stopped
+ */
+export function selectJsonPath(root, path) {
+  const segments = parseJsonPath(path);
+  if (segments.length === 0) {
+    throw new Error(`Path "${path}" is empty`);
+  }
+
+  let current = root;
+  const walked = [];
+
+  for (const segment of segments) {
+    const container = current !== null && typeof current === 'object';
+    const key = Array.isArray(current) ? Number(segment) : segment;
+    if (!container || !(key in current)) {
+      const at = walked.length === 0 ? 'the result' : `"${walked.join('.')}"`;
+      throw new Error(
+        `Path "${path}" not found: ${at} has no "${segment}" (${describeOptions(current)})`
+      );
+    }
+    current = current[key];
+    walked.push(segment);
+  }
+
+  return current;
+}