crawlforge-extractors 1.0.0 → 1.1.0

package/README.md

@@ -1,6 +1,7 @@
 # crawlforge-extractors
 
-Site-specific extraction logic shared by the [CrawlForge](https://www.crawlforge.dev) MCP server and REST API.
+Extraction logic shared by the [CrawlForge](https://www.crawlforge.dev) MCP server and REST API:
+site-specific scrape templates, response body reading, and structural fingerprinting.
 
 ## Why this package exists
 
@@ -15,6 +16,10 @@ extractors. The copies drifted, and nothing detected it:
 A customer would have found both before we did. One implementation removes the
 possibility rather than adding a check for it.
 
+The same reasoning brought in `readBody` and the structure signatures: both
+were behaviours one surface had and the other did not, for no reason anyone
+had decided.
+
 ## Scope
 
 Only pure, dependency-light logic belongs here: parse a body, return fields.
@@ -44,6 +49,42 @@ to `extract($)` with a cheerio document otherwise.
 A template that rejects a response as not its own throws — surface that to the
 caller as a bad request, not a server error.
 
+### Reading a response body
+
+`readBody` decodes with the body's real charset and refuses to buffer past a
+cap. Decoding everything as UTF-8 mangles the large share of the web still
+served as Shift_JIS, GBK or ISO-8859-1, and an uncapped read lets one oversized
+response exhaust a serverless function.
+
+```js
+import { readBody, BodyTooLargeError } from 'crawlforge-extractors';
+
+try {
+  const html = await readBody(response, { maxBytes: 10 * 1024 * 1024 });
+} catch (error) {
+  if (error instanceof BodyTooLargeError) {
+    // error.limit and error.size say what happened.
+  }
+}
+```
+
+It takes a `Response` the caller has already issued, not a URL — SSRF policy,
+host throttling and timeouts differ between the two surfaces and stay with them.
+
+### Comparing page structure
+
+```js
+import { structureSignature, structuralSimilarity } from 'crawlforge-extractors';
+
+const before = structureSignature(cheerio.load(oldHtml));
+const after = structureSignature(cheerio.load(newHtml));
+structuralSimilarity(before, after); // 0-1
+```
+
+A signature is the page's tag vocabulary plus its element-count-by-depth
+histogram — a few dozen keys, small enough to store next to a change-tracking
+baseline instead of keeping the whole DOM.
+
 ## Templates
 
 `shopify-product` · `amazon-product` · `linkedin-profile` · `github-repo` ·

package/index.d.ts

@@ -58,4 +58,53 @@ export declare class TemplateRegistry {
   run(id: string, body: string, url: string, fetchedUrl?: string): Promise<TemplateResult>;
 }
 
+/** Default cap on a buffered response body: 25 MB. */
+export declare const DEFAULT_MAX_BODY_BYTES: number;
+
+/** Thrown by readBody when a response exceeds the cap it was given. */
+export declare class BodyTooLargeError extends Error {
+  name: 'BodyTooLargeError';
+  /** The cap that was exceeded, in bytes. */
+  limit: number;
+  /** Declared or accumulated size that tripped it, in bytes. */
+  size: number;
+}
+
+/**
+ * Pick the charset to decode a body with: Content-Type, then a <meta charset>
+ * sniff of the opening bytes, then utf-8.
+ */
+export declare function detectCharset(response: Response, bytes: Uint8Array): string;
+
+/**
+ * Read a response body as text, capped and decoded with its real charset.
+ * Throws BodyTooLargeError past the cap.
+ */
+export declare function readBody(
+  response: Response,
+  options?: { maxBytes?: number }
+): Promise<string>;
+
+export interface StructureSignature {
+  /** Sorted, de-duplicated tag vocabulary. */
+  tags: string[];
+  /** Element count per nesting depth. */
+  depths: Record<string, number>;
+}
+
+/**
+ * Reduce a parsed document to a signature small enough to store. Pass `root`
+ * to fingerprint one subtree instead of the whole document.
+ */
+export declare function structureSignature(
+  $: CheerioDoc,
+  root?: ReturnType<CheerioDoc>
+): StructureSignature;
+
+/** Compare two signatures, 0-1. */
+export declare function structuralSimilarity(
+  baseline: Partial<StructureSignature> | null | undefined,
+  current: Partial<StructureSignature> | null | undefined
+): number;
+
 export default TemplateRegistry;

package/index.js

@@ -10,8 +10,18 @@
  *
  * Only logic that is pure and dependency-light belongs here: parse a body,
  * return fields. Fetching, billing, auth, caching and browser work stay with
- * whichever surface is calling.
+ * whichever surface is calling — which is why readBody takes a Response the
+ * caller has already issued rather than a URL.
  */
 
 export { TemplateRegistry, TEMPLATES } from './src/templates.js';
 export { TemplateRegistry as default } from './src/templates.js';
+
+export {
+  readBody,
+  detectCharset,
+  BodyTooLargeError,
+  DEFAULT_MAX_BODY_BYTES
+} from './src/body.js';
+
+export { structureSignature, structuralSimilarity } from './src/structure.js';

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "crawlforge-extractors",
-  "version": "1.0.0",
-  "description": "Site-specific extraction logic shared by the CrawlForge MCP server and REST API — one implementation, so the two surfaces cannot drift apart.",
+  "version": "1.1.0",
+  "description": "Extraction logic shared by the CrawlForge MCP server and REST API — scrape templates, charset-correct capped body reading, and structural fingerprinting. One implementation, so the two surfaces cannot drift apart.",
   "type": "module",
   "main": "./index.js",
   "types": "./index.d.ts",
@@ -18,7 +18,7 @@
   "dependencies": {
     "cheerio": "^1.1.2"
   },
-  "keywords": ["crawlforge", "scraping", "extraction", "templates"],
+  "keywords": ["crawlforge", "scraping", "extraction", "templates", "charset", "change-detection"],
   "license": "MIT",
   "repository": {
     "type": "git",

package/src/body.js

@@ -0,0 +1,113 @@
+/**
+ * Response body reading: decode with the body's real charset, and refuse to
+ * buffer more than the caller allows.
+ *
+ * Both surfaces issue their own requests — SSRF rules, host throttling and
+ * auth differ between them — so this takes a Response that has already come
+ * back and only reads it.
+ */
+
+export const DEFAULT_MAX_BODY_BYTES = 25 * 1024 * 1024;
+
+export class BodyTooLargeError extends Error {
+  constructor(message, { limit, size }) {
+    super(message);
+    this.name = 'BodyTooLargeError';
+    this.limit = limit;
+    this.size = size;
+  }
+}
+
+/**
+ * Determine the charset to decode a body with: the Content-Type header first,
+ * then a <meta charset> sniff of the opening bytes, defaulting to utf-8.
+ *
+ * @param {Response} response
+ * @param {Uint8Array} bytes
+ * @returns {string}
+ */
+export function detectCharset(response, bytes) {
+  const contentType = response.headers?.get?.('content-type') || '';
+  const headerMatch = /charset=["']?([\w-]+)/i.exec(contentType);
+  if (headerMatch) {
+    return headerMatch[1].trim().toLowerCase();
+  }
+
+  // <meta charset> tags must appear within the first 1024 bytes per the
+  // HTML5 spec's prescan algorithm; ASCII-range bytes decode identically
+  // under latin1 regardless of the document's real encoding.
+  const sniffLength = Math.min(bytes.byteLength, 1024);
+  const sniffText = new TextDecoder('latin1').decode(bytes.subarray(0, sniffLength));
+  const metaMatch =
+    /<meta[^>]+charset=["']?([\w-]+)/i.exec(sniffText) ||
+    /<meta[^>]+http-equiv=["']?content-type["']?[^>]*content=["'][^"']*charset=([\w-]+)/i.exec(sniffText);
+  if (metaMatch) {
+    return metaMatch[1].trim().toLowerCase();
+  }
+
+  return 'utf-8';
+}
+
+/**
+ * Read a response body as text, capped and charset-correct.
+ *
+ * @param {Response} response
+ * @param {{ maxBytes?: number }} [options]
+ * @returns {Promise<string>}
+ * @throws {BodyTooLargeError} when the body exceeds `maxBytes`
+ */
+export async function readBody(response, options = {}) {
+  const maxBytes = options.maxBytes ?? DEFAULT_MAX_BODY_BYTES;
+
+  // Cheapest rejection first. Servers may omit or lie about Content-Length;
+  // the streaming count below is what actually enforces the cap.
+  const declared = Number.parseInt(response.headers?.get?.('content-length') ?? '', 10);
+  if (Number.isFinite(declared) && declared > maxBytes) {
+    throw new BodyTooLargeError(
+      `Response body too large: Content-Length ${declared} exceeds limit of ${maxBytes} bytes`,
+      { limit: maxBytes, size: declared }
+    );
+  }
+
+  // Only the byte-count guard needs a stream. Responses that are already
+  // buffered (and test doubles) are read as-is so callers still get their text.
+  if (!response.body || typeof response.body.getReader !== 'function') {
+    return response.text();
+  }
+
+  const reader = response.body.getReader();
+  const chunks = [];
+  let totalBytes = 0;
+
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    totalBytes += value.byteLength;
+    if (totalBytes > maxBytes) {
+      reader.cancel();
+      throw new BodyTooLargeError(
+        `Response body too large: exceeded limit of ${maxBytes} bytes`,
+        { limit: maxBytes, size: totalBytes }
+      );
+    }
+    chunks.push(value);
+  }
+
+  // Reassemble in a single pass: totalBytes is already known, so this costs one
+  // allocation plus one copy per chunk rather than the O(n^2) of regrowing a
+  // buffer as each chunk arrives.
+  const merged = new Uint8Array(totalBytes);
+  let offset = 0;
+  for (const chunk of chunks) {
+    merged.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+
+  const charset = detectCharset(response, merged);
+  try {
+    return new TextDecoder(charset).decode(merged);
+  } catch {
+    // Unrecognized charset label — utf-8 beats discarding the body.
+    return new TextDecoder().decode(merged);
+  }
+}

package/src/structure.js

@@ -0,0 +1,108 @@
+/**
+ * Structural fingerprinting for change tracking.
+ *
+ * A signature is deliberately small — the tag vocabulary of a document plus
+ * how its elements are distributed by nesting depth — so a caller can store it
+ * next to a baseline (the REST API keeps baselines in Redis) instead of
+ * keeping the whole DOM around.
+ */
+
+/**
+ * Reduce a parsed document to a comparable structural signature.
+ *
+ * Pass `root` to fingerprint one subtree — a caller tracking a CSS selector
+ * wants a score for that region, not for edits elsewhere on the page. Depths
+ * stay absolute either way, which is harmless because both sides of a
+ * comparison are measured the same way.
+ *
+ * @param {import('cheerio').CheerioAPI} $
+ * @param {import('cheerio').Cheerio<any>} [root]
+ * @returns {{ tags: string[], depths: Record<string, number> }}
+ */
+export function structureSignature($, root) {
+  const tags = new Set();
+  const depths = {};
+
+  (root ? root.find('*') : $('*')).each((_, element) => {
+    if (!element.name) return;
+    tags.add(element.name);
+
+    // Walked off the node itself rather than through cheerio's parents(),
+    // which allocates a wrapper object and an array per element.
+    let depth = 0;
+    for (let parent = element.parent; parent; parent = parent.parent) depth++;
+    depths[depth] = (depths[depth] || 0) + 1;
+  });
+
+  return { tags: [...tags].sort(), depths };
+}
+
+/**
+ * Compare two signatures. 1 means the documents are built from the same tags
+ * in the same depth distribution; 0 means they share nothing.
+ *
+ * @param {{ tags?: string[], depths?: Record<string, number> }} baseline
+ * @param {{ tags?: string[], depths?: Record<string, number> }} current
+ * @returns {number} 0-1
+ */
+export function structuralSimilarity(baseline, current) {
+  if (!baseline || !current) return 0;
+
+  const baselineTags = baseline.tags ?? [];
+  const currentTags = current.tags ?? [];
+  if (baselineTags.length === 0 && currentTags.length === 0) return 1;
+  if (baselineTags.length === 0 || currentTags.length === 0) return 0;
+
+  const score =
+    (tagSimilarity(baselineTags, currentTags) +
+      depthSimilarity(baseline.depths, current.depths)) /
+    2;
+
+  // Clamp defensively — this is a 0-1 metric and must never leave that range.
+  return Math.max(0, Math.min(1, score));
+}
+
+/**
+ * Jaccard overlap of the two tag vocabularies. Both sides are de-duplicated
+ * first: intersecting a duplicate-laden list against a set union let repeated
+ * tags inflate the numerator, which is how this once returned 1.05.
+ */
+function tagSimilarity(baselineTags, currentTags) {
+  const baseline = new Set(baselineTags);
+  const current = new Set(currentTags);
+
+  let intersection = 0;
+  for (const tag of baseline) {
+    if (current.has(tag)) intersection++;
+  }
+
+  const union = baseline.size + current.size - intersection;
+  return union === 0 ? 1 : intersection / union;
+}
+
+/**
+ * Weighted Jaccard over the depth histograms: how much of the two documents'
+ * element mass sits at the same nesting depth.
+ *
+ * This half used to compare nothing. `hierarchy` was initialised to {} and
+ * never written, so the comparison was `0 === 0` and returned a constant 1 —
+ * which pinned every structural score at (tagSimilarity + 1) / 2 and meant a
+ * page could never score below 0.5 however much its structure changed.
+ */
+function depthSimilarity(baseline, current) {
+  if (!baseline || !current) return 0;
+
+  const depths = new Set([...Object.keys(baseline), ...Object.keys(current)]);
+  if (depths.size === 0) return 1;
+
+  let shared = 0;
+  let total = 0;
+  for (const depth of depths) {
+    const a = baseline[depth] || 0;
+    const b = current[depth] || 0;
+    shared += Math.min(a, b);
+    total += Math.max(a, b);
+  }
+
+  return total === 0 ? 1 : shared / total;
+}