crawlforge-extractors 1.7.0 → 1.9.0

package/README.md

@@ -208,6 +208,35 @@ const verdict = documentVerdict(
 `detectChallengePage` is the vendor check alone. Both are pure: the caller
 keeps or drops the content, and decides what to try next.
 
+### Finding the units that match a query
+
+`segmentUnits` cuts the markdown a scrape returned into sentences, table rows
+and fenced code blocks; `rankUnits` scores them against a query with BM25 and
+returns the best few verbatim. It is how a caller gets the one table row that
+answers "enterprise price per month" without paying to read the whole page,
+and without a model paraphrasing it — nothing here can say something the page
+does not.
+
+```js
+import { segmentUnits, rankUnits } from 'crawlforge-extractors';
+
+const units = segmentUnits(markdown);
+const best = rankUnits(units, 'enterprise price per month', { maxUnits: 5 });
+// [{ text: 'Enterprise is priced at $499 per month, billed annually, and includes a dedicated success engineer.',
+//    kind: 'sentence', offset: 815, length: 99, heading: 'Enterprise', score: 6.612 },
+//  { text: '| Price per month | $29 | $99 | $499 |',
+//    kind: 'table_row', offset: 257, length: 38, heading: 'Compare plans', score: 5.809 }, …]
+```
+
+Offsets are JS string indexes into the exact string passed in, and every unit
+keeps `markdown.slice(offset, offset + length) === text`: trimming and list
+markers move the offset, nothing rewrites the text. Headings are not units;
+each unit carries the heading above it. The sentence splitter shares the MCP
+server's terminator rules — `。！？` and the danda split on a zero-width
+boundary, and an ASCII period does not split after `Dr.`, `e.g.`, `Node.js`
+or `3.14`. A CJK query matches by character bigrams, so no segmenter is
+needed.
+
 ## Templates
 
 **Pages and products.** `shopify-product` · `shopify-collection` ·

package/index.d.ts

@@ -324,3 +324,101 @@ export declare function documentVerdict(
 
 /** A document with this much text or less and an error title is a placeholder. */
 export declare const SOFT_ERROR_MAX_CHARS: number;
+
+/** One scoreable piece of a page's markdown: a sentence, a table row or a fenced code block. */
+export interface HighlightUnit {
+  /** Verbatim from the markdown: `markdown.slice(offset, offset + length) === text`. */
+  text: string;
+  kind: 'sentence' | 'table_row' | 'code_block';
+  /** JS string index into the markdown segmentUnits was given — not a byte offset. */
+  offset: number;
+  length: number;
+  /** The nearest heading above the unit, without its `#` marks; null before the first heading. */
+  heading: string | null;
+}
+
+export interface RankedHighlightUnit extends HighlightUnit {
+  /** BM25 with the phrase boost applied and the heading's terms counted at half weight, rounded to 3 decimals; always above `minScore`. */
+  score: number;
+}
+
+/**
+ * Cut the markdown a scrape returned into sentences, table rows and fenced
+ * code blocks, each with its offset into that same string. Headings are not
+ * units; they label the units that follow. List and quote markers are
+ * skipped by moving the offset, never by rewriting the text. The sentence
+ * splitter carries the MCP server's rules: 。！？ and the danda split on a
+ * zero-width boundary, and an ASCII period does not split after an
+ * abbreviation, a word with internal periods, a decimal or an initial.
+ */
+export declare function segmentUnits(markdown: string): HighlightUnit[];
+
+/**
+ * The units that answer a query, best first: BM25 over the units as the
+ * corpus, each unit inheriting its heading's terms at half weight, ×1.5 when
+ * a unit contains the whole query. Units scoring at or below `minScore` (default 0: no
+ * term in common) are dropped; at most `maxUnits` (default 10, clamped to at
+ * least 1) come back, ties broken by offset. The input is not modified.
+ */
+export declare function rankUnits(
+  units: HighlightUnit[],
+  query: string,
+  options?: { maxUnits?: number; minScore?: number }
+): RankedHighlightUnit[];
+
+/** The four entity classes `redactPii` finds with regex alone, no model. */
+export declare const REGEX_ENTITIES: readonly ['EMAIL', 'PHONE', 'FINANCIAL', 'SECRET'];
+
+/**
+ * The entity classes regex cannot find. `redactPii` never handles these — a
+ * caller wanting them routes the text through its own model pass and is
+ * charged separately for it.
+ */
+export declare const MODEL_ONLY_ENTITIES: readonly ['PERSON', 'LOCATION'];
+
+/** The style `redactPii` uses when the caller names none: `<EMAIL>` and friends. */
+export declare const DEFAULT_REPLACE_STYLE: 'tag';
+
+/** What a redaction pass changed: a count per entity class, and the total. */
+export interface PiiRedaction {
+  /** Per-class replacement counts. A class with no hits is omitted, not zero. */
+  entities: Record<string, number>;
+  /** Total replacements made across every class. */
+  count: number;
+}
+
+/**
+ * Replace the personal and secret data in a string, in one pass, with no model.
+ *
+ * Detectors contribute spans over the original text and the string is rebuilt
+ * once at the end, so overlaps are resolved a single time and nothing is
+ * counted twice. Accept order is SECRET, EMAIL, FINANCIAL, PHONE: a labelled
+ * credential beats whatever its value looks like, and a card number can never
+ * afterwards be re-read as a phone number.
+ *
+ * Precision is preferred to recall throughout — this runs on arbitrary page
+ * text, where a false positive silently destroys content the caller paid to
+ * scrape. Card numbers must pass Luhn and IBANs mod-97; prices, dates,
+ * version numbers and long ids are left alone.
+ *
+ * Markdown-safe. `scrape` returns markdown by default and turndown escapes it,
+ * writing `_` as `\_` and a line-leading `-` as `\-`, so the EMAIL local part
+ * and the `api_key` SECRET label match through that escape: an escaped address
+ * is redacted as ONE span (`simon\_lacey@example.com` becomes `<EMAIL>`, never
+ * `simon\<EMAIL>` with a count of 1 beside the surviving name). The escape is
+ * tolerated, never removed — text outside a replaced span comes back
+ * byte-identical, offsets intact. No other detector needs the tolerance.
+ *
+ * `entities` names are upper-cased and intersected with `REGEX_ENTITIES`.
+ * Passing `undefined`, a non-array or an EMPTY array means all four, because a
+ * missing selection should fail towards more redaction — but an array that
+ * leaves nothing behind (`['PERSON']`) redacts NOTHING rather than falling
+ * back to all. An unknown `replaceStyle` is treated as `'tag'`. SECRET keeps
+ * its label and replaces only the value in every style, `'remove'` included.
+ *
+ * Never throws: a non-string `text` comes back unchanged with a zero count.
+ */
+export declare function redactPii(
+  text: string,
+  options?: { entities?: string[]; replaceStyle?: 'tag' | 'mask' | 'remove' }
+): { text: string; redaction: PiiRedaction };

package/index.js

@@ -33,3 +33,7 @@ export { parseJsonPath, selectJsonPath } from './src/jsonPath.js';
 export { shopifyProductFromJsonLd } from './src/shopifyJsonLd.js';
 
 export { detectChallengePage, documentVerdict, SOFT_ERROR_MAX_CHARS } from './src/blockedPage.js';
+
+export { segmentUnits, rankUnits } from './src/highlights.js';
+
+export { redactPii, REGEX_ENTITIES, MODEL_ONLY_ENTITIES, DEFAULT_REPLACE_STYLE } from './src/pii.js';

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "crawlforge-extractors",
-  "version": "1.7.0",
-  "description": "Extraction logic shared by the CrawlForge MCP server and REST API — scrape templates, charset-correct capped body reading, structural fingerprinting, and embedded-state extraction. One implementation, so the two surfaces cannot drift apart.",
+  "version": "1.9.0",
+  "description": "Extraction logic shared by the CrawlForge MCP server and REST API — scrape templates, charset-correct capped body reading, structural fingerprinting, embedded-state extraction, and query-scoped highlights. One implementation, so the two surfaces cannot drift apart.",
   "type": "module",
   "main": "./index.js",
   "types": "./index.d.ts",

package/src/highlights.js

@@ -0,0 +1,342 @@
+/**
+ * highlights.js — the units of a scraped page that answer a query, verbatim.
+ *
+ * A caller asking "what does the enterprise plan cost" does not want the
+ * whole markdown of a pricing page in its context window, and it does not
+ * want a model's paraphrase of it either — the first is expensive, the
+ * second can be wrong. This module is the extractive middle: cut the
+ * markdown a scrape already returned into units (sentences, table rows,
+ * fenced code blocks), score each against the query with BM25, and hand
+ * back the top few exactly as they appear on the page, with character
+ * offsets into that same string so the caller can quote with a locator.
+ * Both surfaces run it, so a highlight is the same highlight over MCP and
+ * REST.
+ *
+ * Everything here is a pure function over the markdown string. Offsets are
+ * JS string indexes, and the invariant every unit keeps is
+ * `markdown.slice(offset, offset + length) === text`. Trimming moves the
+ * offsets; nothing rewrites the text.
+ *
+ * The sentence splitter ports the MCP server's sentenceUtils.js rules: the
+ * CJK / fullwidth / Devanagari terminators split on a zero-width boundary
+ * and are never judged by the ASCII checks; an ASCII `.` followed by
+ * whitespace does not split after an abbreviation (Dr., etc.), a word with
+ * internal periods (Node.js, e.g.), a decimal (3.14) or a single-letter
+ * initial. The one departure from the server: those four checks only guard
+ * `.`, because "What is Node.js? It is a runtime." is two sentences.
+ */
+
+const CJK_TERMINATORS = '。．！？；।॥';
+const ASCII_TERMINATORS = '.!?';
+// Closing punctuation a terminator may carry with it: "…end."), 'so.', and
+// the emphasis markers of a bold FAQ question — "**Can I cancel?** Yes."
+const CLOSERS = '"\')\\]”’»*_';
+
+const ABBREVIATIONS = new Set([
+  'mr', 'mrs', 'ms', 'dr', 'prof', 'sr', 'jr', 'st', 'ave', 'blvd',
+  'vs', 'etc', 'inc', 'ltd', 'corp', 'dept', 'univ', 'assn',
+  'approx', 'appt', 'apt', 'est', 'min', 'max',
+  'govt', 'lib', 'misc', 'natl', 'intl',
+  'jan', 'feb', 'mar', 'apr', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec',
+  'mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun',
+  'fig', 'eq', 'ref', 'vol', 'no', 'pp', 'ed', 'rev',
+  'e', 'i' // e.g. and i.e.
+]);
+
+const MIN_UNIT_CHARS = 2;
+
+const FENCE = /^(`{3,}|~{3,})/;
+const FENCE_CLOSER = /^(?:`+|~+)$/;
+const HEADING = /^#{1,6}(?:\s+(.*?))?\s*#*\s*$/;
+const TABLE_DELIMITER = /^\|?[\s:|-]*-[\s:|-]*\|?$/;
+const HORIZONTAL_RULE = /^(?:-\s*){3,}$|^(?:\*\s*){3,}$|^(?:_\s*){3,}$/;
+const HTML_COMMENT_LINE = /^<!--[\s\S]*-->$/;
+const IMAGE_ONLY_LINE = /^!\[[^\]]*\]\([^)]*\)$/;
+const BLOCKQUOTE_MARKER = /^>\s?/;
+const LIST_MARKER = /^(?:[-*+]|\d{1,9}[.)])\s+/;
+
+function isSpace(ch) {
+  return /\s/.test(ch);
+}
+
+/** Move [start, end) inward past whitespace at both ends. */
+function trimRange(markdown, start, end) {
+  while (start < end && isSpace(markdown[start])) start++;
+  while (end > start && isSpace(markdown[end - 1])) end--;
+  return [start, end];
+}
+
+function hasEnoughText(text) {
+  let count = 0;
+  for (const ch of text) {
+    if (!isSpace(ch) && ++count >= MIN_UNIT_CHARS) return true;
+  }
+  return false;
+}
+
+/**
+ * Cut the markdown a scrape returned into the units a query can be scored
+ * against. Headings are not units — they label the units that follow.
+ *
+ * @param {string} markdown
+ * @returns {Array<{ text: string, kind: 'sentence' | 'table_row' | 'code_block', offset: number, length: number, heading: string | null }>}
+ */
+export function segmentUnits(markdown) {
+  if (typeof markdown !== 'string' || markdown.length === 0) return [];
+
+  const units = [];
+  let heading = null;
+
+  function push(kind, start, end) {
+    [start, end] = trimRange(markdown, start, end);
+    const text = markdown.slice(start, end);
+    if (!hasEnoughText(text)) return;
+    units.push({ text, kind, offset: start, length: end - start, heading });
+  }
+
+  // A paragraph is a contiguous range: consecutive prose lines, the first
+  // one's list or quote marker already skipped. Line breaks inside it are
+  // whitespace, so a sentence may carry a "\n" verbatim.
+  let paragraphStart = -1;
+  let paragraphEnd = -1;
+  function flushParagraph() {
+    if (paragraphStart >= 0) splitSentences(markdown, paragraphStart, paragraphEnd, push);
+    paragraphStart = -1;
+  }
+
+  let fence = null; // { marker, contentStart }
+
+  const length = markdown.length;
+  let lineStart = 0;
+  while (lineStart <= length) {
+    let lineEnd = markdown.indexOf('\n', lineStart);
+    if (lineEnd === -1) lineEnd = length;
+    const [s, e] = trimRange(markdown, lineStart, lineEnd);
+    const line = markdown.slice(s, e);
+    let match;
+
+    if (fence) {
+      if (line[0] === fence.marker[0] && line.length >= fence.marker.length && FENCE_CLOSER.test(line)) {
+        push('code_block', fence.contentStart, Math.max(fence.contentStart, lineStart));
+        fence = null;
+      }
+    } else if (line === '') {
+      flushParagraph();
+    } else if ((match = FENCE.exec(line))) {
+      flushParagraph();
+      fence = { marker: match[1], contentStart: Math.min(lineEnd + 1, length) };
+    } else if ((match = HEADING.exec(line))) {
+      flushParagraph();
+      heading = (match[1] || '').trim() || null;
+    } else if (line[0] === '|') {
+      flushParagraph();
+      if (!TABLE_DELIMITER.test(line)) push('table_row', s, e);
+    } else if (HORIZONTAL_RULE.test(line) || HTML_COMMENT_LINE.test(line) || IMAGE_ONLY_LINE.test(line)) {
+      flushParagraph();
+    } else {
+      let start = s;
+      let marked = false;
+      while ((match = BLOCKQUOTE_MARKER.exec(markdown.slice(start, e)))) {
+        start += match[0].length;
+        marked = true;
+      }
+      if ((match = LIST_MARKER.exec(markdown.slice(start, e)))) {
+        start += match[0].length;
+        marked = true;
+      }
+      // A marker starts its own paragraph: two list items are two units,
+      // and "- " never lands inside a sentence's text.
+      if (marked) flushParagraph();
+      if (paragraphStart < 0) paragraphStart = start;
+      paragraphEnd = e;
+    }
+
+    lineStart = lineEnd + 1;
+  }
+
+  // A fence nobody closed: the rest of the document is that block.
+  if (fence) push('code_block', fence.contentStart, length);
+  flushParagraph();
+
+  return units;
+}
+
+/**
+ * Split the paragraph at [start, end) into sentences, calling `emit` with
+ * each one's range. Whitespace between sentences belongs to neither.
+ */
+function splitSentences(markdown, start, end, emit) {
+  let sentenceStart = start;
+  let i = start;
+  while (i < end) {
+    const ch = markdown[i];
+    if (CJK_TERMINATORS.includes(ch)) {
+      // Unambiguous, and CJK text puts no whitespace after them.
+      emit('sentence', sentenceStart, i + 1);
+      sentenceStart = i + 1;
+      i++;
+      continue;
+    }
+    if (!ASCII_TERMINATORS.includes(ch)) {
+      i++;
+      continue;
+    }
+    // Consume the run: "?!", "...", then any closing quote or bracket.
+    let j = i;
+    while (j < end && ASCII_TERMINATORS.includes(markdown[j])) j++;
+    while (j < end && CLOSERS.includes(markdown[j])) j++;
+    if (j < end && !isSpace(markdown[j])) {
+      // "Node.js", "3.14", "e.g." — a period glued to the next word.
+      i = j;
+      continue;
+    }
+    if (ch === '.' && isFalseStop(markdown, sentenceStart, i)) {
+      i = j;
+      continue;
+    }
+    emit('sentence', sentenceStart, j);
+    sentenceStart = j;
+    i = j;
+  }
+  if (sentenceStart < end) emit('sentence', sentenceStart, end);
+}
+
+/**
+ * Whether the period at `dot` ends an abbreviation, a word with internal
+ * periods, a decimal or a single-letter initial — the server's four checks,
+ * applied to the whitespace-delimited word before the period.
+ */
+function isFalseStop(markdown, from, dot) {
+  let wordStart = dot;
+  while (wordStart > from && !isSpace(markdown[wordStart - 1])) wordStart--;
+  const word = markdown.slice(wordStart, dot);
+  if (word === '') return false;
+  if (ABBREVIATIONS.has(word.toLowerCase().replace(/[^a-z]/g, ''))) return true;
+  if (/\w\.\w/.test(word)) return true;
+  if (/\d\.\d/.test(word)) return true;
+  return /^[A-Z]$/.test(word);
+}
+
+// Letters and digits of the scripts that put no spaces between words (Han,
+// Hiragana, Katakana, Hangul) form one kind of run; every other letter,
+// digit or combining mark forms the other — marks, because a Devanagari
+// vowel sign is a mark, and without them "कीमत" is two fragments.
+// Script_Extensions keeps "ー" and "々" with their runs; the letter/number
+// class keeps "、" and "。" out of them.
+const CJK_CHAR = '[\\p{scx=Han}\\p{scx=Hiragana}\\p{scx=Katakana}\\p{scx=Hangul}]';
+const TOKEN = new RegExp(`(?:(?=${CJK_CHAR})[\\p{L}\\p{N}])+|(?:(?!${CJK_CHAR})[\\p{L}\\p{N}\\p{M}])+`, 'gu');
+const CJK_START = new RegExp(`^${CJK_CHAR}`, 'u');
+const SUFFIXES = ['ing', 'ed', 'es', 's'];
+
+/**
+ * Lowercase word tokens. A CJK run becomes character bigrams so a query in
+ * one of those scripts can match without a segmenter; a light suffix strip
+ * puts "pricing", "prices" and "price" on one stem ("pric").
+ * @param {string} text
+ * @returns {string[]}
+ */
+function tokenize(text) {
+  const tokens = [];
+  for (const [run] of text.toLowerCase().matchAll(TOKEN)) {
+    if (CJK_START.test(run)) {
+      const chars = Array.from(run);
+      if (chars.length === 1) tokens.push(run);
+      for (let i = 0; i + 1 < chars.length; i++) tokens.push(chars[i] + chars[i + 1]);
+    } else {
+      tokens.push(stem(run));
+    }
+  }
+  return tokens;
+}
+
+function stem(token) {
+  if (token.length <= 4) return token;
+  for (const suffix of SUFFIXES) {
+    if (token.endsWith(suffix)) {
+      token = token.slice(0, -suffix.length);
+      break;
+    }
+  }
+  return token.length > 4 && token.endsWith('e') ? token.slice(0, -1) : token;
+}
+
+const K1 = 1.2;
+const B = 0.75;
+const PHRASE_BOOST = 1.5;
+// A unit inherits the terms of the heading it sits under, at half the weight
+// of a term in its own text: on a card-style pricing page the plan name is
+// the heading and "$83/month" is the unit, and without the heading the price
+// line shares nothing with "professional plan price per month". Heading
+// terms count toward a unit's term frequency only — not toward document
+// frequency, or the plan name would look common and lose its weight.
+const HEADING_WEIGHT = 0.5;
+// BM25's length normalisation rewards short documents, and a page's shortest
+// units are its buttons: "Choose Professional" outranked every price line on
+// a live pricing page. A unit is scored as if it had at least this many
+// tokens, so a two-word call to action carries no length advantage.
+const MIN_DOC_LENGTH = 4;
+
+/**
+ * The units that answer a query, best first: BM25 over the units as the
+ * corpus, with each unit inheriting its heading's terms at half weight, and a
+ * phrase boost when a unit contains the whole query. The input units are not
+ * touched; every returned unit is a new object.
+ *
+ * @template {{ text: string, heading?: string | null, offset: number }} U
+ * @param {U[]} units
+ * @param {string} query
+ * @param {{ maxUnits?: number, minScore?: number }} [options]
+ * @returns {Array<U & { score: number }>}
+ */
+export function rankUnits(units, query, { maxUnits = 10, minScore = 0 } = {}) {
+  if (!Array.isArray(units) || units.length === 0 || typeof query !== 'string') return [];
+  const phrase = query.trim().toLowerCase();
+  if (phrase === '') return [];
+  const terms = [...new Set(tokenize(phrase))];
+  if (terms.length === 0) return [];
+
+  const limit = Number.isFinite(maxUnits) ? Math.max(1, Math.floor(maxUnits)) : 10;
+  const floor = Number.isFinite(minScore) ? minScore : 0;
+
+  const docs = units.map((unit) => {
+    const counts = new Map();
+    const tokens = tokenize(String(unit.text ?? ''));
+    for (const token of tokens) counts.set(token, (counts.get(token) || 0) + 1);
+    const own = new Set(counts.keys());
+    let length = tokens.length;
+    if (unit.heading) {
+      for (const token of new Set(tokenize(String(unit.heading)))) {
+        if (own.has(token)) continue;
+        counts.set(token, HEADING_WEIGHT);
+        length += HEADING_WEIGHT;
+      }
+    }
+    return { counts, own, length: Math.max(length, MIN_DOC_LENGTH) };
+  });
+  const n = docs.length;
+  const avgdl = docs.reduce((sum, doc) => sum + doc.length, 0) / n || 1;
+  const idf = new Map(terms.map((term) => {
+    const df = docs.reduce((count, doc) => count + (doc.own.has(term) ? 1 : 0), 0);
+    return [term, Math.log(1 + (n - df + 0.5) / (df + 0.5))];
+  }));
+
+  const ranked = [];
+  units.forEach((unit, index) => {
+    const doc = docs[index];
+    let score = 0;
+    for (const term of terms) {
+      const tf = doc.counts.get(term);
+      if (!tf) continue;
+      score += idf.get(term) * (tf * (K1 + 1)) / (tf + K1 * (1 - B + B * doc.length / avgdl));
+    }
+    if (score === 0) return;
+    if (String(unit.text).toLowerCase().includes(phrase)) score *= PHRASE_BOOST;
+    // Round before the threshold, so a score handed back as minScore means
+    // what the caller saw.
+    score = Math.round(score * 1000) / 1000;
+    if (score > floor) ranked.push({ ...unit, score });
+  });
+
+  ranked.sort((a, b) => b.score - a.score || a.offset - b.offset);
+  return ranked.slice(0, limit);
+}

package/src/pii.js

@@ -0,0 +1,359 @@
+/**
+ * pii.js — redact the personal data a scraped page carries, before that text
+ * reaches a model, a log line or a customer's context window.
+ *
+ * A scrape returns whatever the page holds, and pages hold support-inbox
+ * addresses, staff phone lists, a checkout form's card number and — in an
+ * error page or a leaked config block — an API key. The MCP server already
+ * scrubbed the last of those out of its own logs (`secretMask.js`
+ * `redactSecretsFromString`); nothing scrubbed the page text it hands back.
+ * That heuristic moves here so both surfaces run one implementation, and it
+ * arrives with three more entity classes beside it.
+ *
+ * Everything in this module is pure regex over a string: no network, no
+ * model, no cheerio, deterministic. That is the point. A model-backed pass
+ * for PERSON and LOCATION exists on the calling side, is opt-in, and is
+ * priced separately; this module never invokes it. If a caller asks for
+ * PERSON or LOCATION here they are ignored in silence — `MODEL_ONLY_ENTITIES`
+ * is exported so the caller can route them, and so the two surfaces read one
+ * declaration of which entities a regex cannot decide.
+ *
+ * ## Precision over recall, deliberately
+ *
+ * This runs over arbitrary page text that a customer paid to scrape. A false
+ * positive silently destroys real content — a redacted price is worse than an
+ * un-redacted phone number, because the customer can see the second one and
+ * cannot recover the first. Every detector here is therefore built to be
+ * conservative and to fail by *not* matching:
+ *
+ * - a bare run of digits is never a phone number. A phone must carry a `+`
+ *   country code or a real separator-bearing NANP shape;
+ * - a card number must pass **Luhn**, and an IBAN must pass **mod-97**. Both
+ *   checks exist to reject the coincidence, not to validate the account;
+ * - a candidate that a check rejects is dropped, not retried shorter. Missing
+ *   one number is the cheap failure.
+ *
+ * What it deliberately does NOT catch, so nobody reports these as bugs:
+ *
+ * - **PERSON, LOCATION** — model-only, by definition. No regex knows that
+ *   "Paris Hilton" is not a hotel in France.
+ * - **7-digit local phone numbers** (`555-1234`) — indistinguishable from a
+ *   part number, a date range or a score at scale.
+ * - **non-NANP domestic phone numbers written without a `+`** (`020 7123
+ *   4567`, `0912-345-678`) — a leading 0 area code is the shape of far too
+ *   many identifiers. Written in E.164 (`+44 20 7123 4567`) they are caught.
+ * - **lowercase IBANs** — IBANs are printed uppercase; accepting lowercase
+ *   would make every long word a candidate.
+ * - **national ID numbers, passport numbers, dates of birth, addresses,
+ *   licence plates** — all of them are locale-specific digit runs with no
+ *   check digit worth trusting.
+ * - **secrets with no label** — a bare `sk-live-...` in prose is not matched.
+ *   The SECRET heuristic needs `Bearer `, or an `api_key` / `x-api-key` /
+ *   `password` / `secret` / `token` label followed by `:` or `=`.
+ * - **markdown escapes other than `\\_` and a line-leading `\\-`** — see the
+ *   next section. Those two are the only escaped characters that land inside
+ *   a token this module has to match through; the rest (`\\*`, `` \\` ``,
+ *   `\\[`, `\\]`) are not legal in an address or a label in the first place,
+ *   so admitting them would widen the false-positive surface for nothing.
+ *
+ * ## Markdown escaping
+ *
+ * `scrape` returns markdown by default, and turndown escapes the characters
+ * markdown gives meaning to: `\\`, `*`, `` ` ``, `[`, `]` and `_` anywhere in
+ * a text node, and `-`, `+ `, `=`, `#`, `>` and `~~~` only at the start of a
+ * line. So `simon_lacey@example.com` reaches this module written
+ * `simon\\_lacey@example.com`, and the naive pattern matches only from the
+ * `_` onwards.
+ *
+ * That is worse than a miss. `simon\\<EMAIL>` reports `count: 1` — a
+ * successful redaction — while the given name is still sitting in the text.
+ * A redaction report that overstates what it redacted is the one failure this
+ * feature cannot have; a clean miss at least says `count: 0`. So the EMAIL
+ * local part and the `api_key` label tolerate the escape, and an escaped run
+ * matches as ONE span: `simon\\_lacey@example.com` becomes `<EMAIL>` entire.
+ *
+ * The escape is tolerated, never removed. Nothing here unescapes and
+ * re-escapes the text — the caller's string comes back with spans replaced
+ * and every character outside a span byte-identical.
+ *
+ * The other detectors were checked against turndown's own escape table and
+ * need no tolerance, which a round-trip test pins: PHONE and FINANCIAL use
+ * space, `.` and `-` as separators, and `-` is escaped only as the first
+ * character of a line, where a phone or card number begins with a digit, `+`
+ * or `(`. IBANs are letters, digits and spaces. `Bearer`, `x-api-key`,
+ * `password`, `secret` and `token` contain nothing escapable — only
+ * `api_key` does. And secret *values* never needed it: `\\S+` already matches
+ * a backslash, so `token=sk\\_live\\_abc` was always redacted whole. Code
+ * blocks are not escaped at all — turndown indents them instead.
+ *
+ * ## Detector order
+ *
+ * All detectors run over the ORIGINAL string and contribute spans; the string
+ * is rebuilt once at the end. Spans are accepted in this order, and a span
+ * that overlaps an already-accepted one is dropped — so nothing is counted or
+ * replaced twice, and a card number claimed by FINANCIAL can never afterwards
+ * be re-read as a PHONE:
+ *
+ *   1. SECRET      — a labelled credential wins over whatever its value looks
+ *                    like (`password: ops@example.com` is a SECRET, not an
+ *                    EMAIL).
+ *   2. EMAIL       — before the numeric detectors, so digits inside an address
+ *                    are not mistaken for an account number.
+ *   3. FINANCIAL   — cards and IBANs before PHONE, so `4242 4242 4242 4242`
+ *                    can never be read as a phone number.
+ *   4. PHONE       — the least specific shape, so it goes last.
+ *
+ * Within PHONE the E.164 patterns run before the NANP one, so `+1 555 123
+ * 4567` is one match and not a NANP match with a stray `+1` in front.
+ *
+ * ## The SECRET invariant
+ *
+ * SECRET keeps its label and replaces only the value — `Bearer <SECRET>`,
+ * `api_key=<SECRET>` — in every replace style, including `remove` (which
+ * leaves `Bearer `). That is not cosmetic: the MCP server's `maskError`
+ * depends on the label surviving so an error message still says *which*
+ * credential the request carried. Every other entity replaces the whole match.
+ */
+
+/** The entity classes this module decides with a regex. */
+export const REGEX_ENTITIES = Object.freeze(['EMAIL', 'PHONE', 'FINANCIAL', 'SECRET']);
+
+/**
+ * The entity classes a regex cannot decide. `redactPii` ignores these; a
+ * caller that offers them routes them to its own model pass.
+ */
+export const MODEL_ONLY_ENTITIES = Object.freeze(['PERSON', 'LOCATION']);
+
+/** The replace style used when the caller does not choose one. */
+export const DEFAULT_REPLACE_STYLE = 'tag';
+
+/** The `mask` style's replacement — the same constant secretMask.js uses. */
+const MASK = '[REDACTED]';
+
+// ---------------------------------------------------------------------------
+// Patterns
+// ---------------------------------------------------------------------------
+
+// A standard local@domain with a real TLD. The TLD floor of two letters is
+// what keeps "user@localhost" and "@handle" out.
+//
+// The local part also admits the pair \_ (and \-), which is how turndown
+// writes an underscore, or a line-leading hyphen, in markdown — see the
+// markdown-escaping section of the header. It is the PAIR, not a bare
+// backslash in the character class: the escape can only appear where the
+// character it escapes was already legal, so the local part still cannot run
+// across a space or any other boundary. The domain needs no such tolerance —
+// turndown escapes nothing that is legal in one.
+const EMAIL =
+  /(?:[A-Za-z0-9._%+-]|\\[_-])+@[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?)*\.[A-Za-z]{2,24}\b/g;
+
+// E.164 written as one run: +15551234567. The digit count is checked in code
+// (ITU allows at most 15), not in the quantifier, so the rule is readable.
+const PHONE_E164_RUN = /(?<![\w+])\+\d{8,15}(?!\w)/g;
+
+// E.164 written in groups: +44 20 7123 4567, +1 (555) 123-4567, +1-555-123-4567.
+// Every group after the country code must be introduced by a separator or be
+// parenthesised, which is what stops "+1" swallowing the number beside it.
+const PHONE_E164_GROUPED = /(?<![\w+])\+\d{1,3}(?:[ .-]?\(\d{1,4}\)|[ .-]\d{1,4}){1,5}(?!\w)/g;
+
+// NANP with separators, parenthesised: (555) 123-4567, 1 (555) 123-4567.
+const PHONE_NANP_PARENS = /(?<![\w.,-])(?:1[ .-]?)?\([2-9]\d{2}\)[ .]?\d{3}[ .-]\d{4}(?![\w-])/g;
+
+// NANP with separators, plain: 555-123-4567, 555.123.4567, 555 123 4567,
+// 1-555-123-4567. Three things carry the precision here. The [2-9] on the area
+// code is a real NANP rule and rejects 012/123/100 and the other
+// counting-sequence lookalikes (the exchange code cannot take the same rule:
+// 555-123-4567, the number every document uses, has 123 there). The
+// backreference makes the separator the same character throughout, which is
+// how a real number is written and how "1,234.567 8901" is not. And the
+// leading (?<![\w.,-]) keeps this out of ISBNs, dates, versions, grouped money
+// and hyphenated identifiers while the trailing (?![\w-]) keeps it out of
+// longer digit runs.
+const PHONE_NANP_PLAIN = /(?<![\w.,-])(?:1[ .-])?[2-9]\d{2}([ .-])\d{3}\1\d{4}(?![\w-])/g;
+
+// A payment card: 13-19 digits, either contiguous or in consistent groups of
+// 2-6 separated by one repeated space or hyphen (\1 is the backreference that
+// enforces "the same separator throughout"). Groups of at least two digits
+// are what stops a table row of single digits reading as an account number.
+// Luhn does the rest. The lookarounds keep it out of decimals.
+const CARD = /(?<![\w-])(?<!\d\.)(?:\d{13,19}|\d{2,6}(?:([ -])\d{2,6})(?:\1\d{2,6})*)(?![\w-])(?!\.\d)/g;
+
+// An IBAN: country code, two check digits, then the account part either
+// contiguous or in the conventional groups of four. mod-97 gates it.
+const IBAN = /(?<![A-Za-z0-9])[A-Z]{2}\d{2}(?:[A-Z0-9]{11,30}|(?: [A-Z0-9]{4})+(?: [A-Z0-9]{1,3})?)(?![A-Za-z0-9])/g;
+
+// The secret heuristics, ported verbatim from the MCP server's
+// secretMask.js redactSecretsFromString, in its original order. Group 1 is
+// the label and is preserved; the span replaced is everything after it.
+const SECRET_PATTERNS = [
+  /(Bearer\s+)\S+/gi,
+  /(api(?:\\?[_-])?key\s*[:=]\s*)\S+/gi,
+  /(x-api-key\s*[:=]\s*)\S+/gi,
+  /(password\s*[:=]\s*)\S+/gi,
+  /(secret\s*[:=]\s*)\S+/gi,
+  /(token\s*[:=]\s*)\S+/gi
+];
+
+// ---------------------------------------------------------------------------
+// Checks
+// ---------------------------------------------------------------------------
+
+/**
+ * Luhn. 13-19 digits, and never a run of one repeated digit — 0000000000000000
+ * satisfies Luhn and is a placeholder, not a card.
+ * @param {string} candidate
+ * @returns {boolean}
+ */
+function luhnOk(candidate) {
+  const digits = candidate.replace(/\D/g, '');
+  if (digits.length < 13 || digits.length > 19) return false;
+  if (/^(\d)\1*$/.test(digits)) return false;
+  let sum = 0;
+  let double = false;
+  for (let i = digits.length - 1; i >= 0; i--) {
+    let d = digits.charCodeAt(i) - 48;
+    if (double) {
+      d *= 2;
+      if (d > 9) d -= 9;
+    }
+    sum += d;
+    double = !double;
+  }
+  return sum % 10 === 0;
+}
+
+/**
+ * ISO 13616 mod-97: move the first four characters to the end, map A-Z to
+ * 10-35, and the remainder of the resulting number modulo 97 must be 1.
+ * Computed digit by digit so no value ever leaves the safe integer range.
+ * @param {string} candidate
+ * @returns {boolean}
+ */
+function ibanOk(candidate) {
+  const compact = candidate.replace(/ /g, '');
+  if (compact.length < 15 || compact.length > 34) return false;
+  const rearranged = compact.slice(4) + compact.slice(0, 4);
+  let remainder = 0;
+  for (const ch of rearranged) {
+    const value = ch >= 'A' ? ch.charCodeAt(0) - 55 : ch.charCodeAt(0) - 48;
+    if (value < 0 || value > 35) return false;
+    remainder = (remainder * (value > 9 ? 100 : 10) + value) % 97;
+  }
+  return remainder === 1;
+}
+
+/** E.164 allows 8-15 digits in total, country code included. */
+function e164Ok(candidate) {
+  const digits = candidate.replace(/\D/g, '').length;
+  return digits >= 8 && digits <= 15;
+}
+
+// ---------------------------------------------------------------------------
+// Detectors, in the order their spans are accepted
+// ---------------------------------------------------------------------------
+
+const DETECTORS = [
+  { entity: 'SECRET', patterns: SECRET_PATTERNS.map(re => ({ re, keepsLabel: true })) },
+  { entity: 'EMAIL', patterns: [{ re: EMAIL }] },
+  { entity: 'FINANCIAL', patterns: [{ re: CARD, check: luhnOk }, { re: IBAN, check: ibanOk }] },
+  {
+    entity: 'PHONE',
+    patterns: [
+      { re: PHONE_E164_RUN, check: e164Ok },
+      { re: PHONE_E164_GROUPED, check: e164Ok },
+      { re: PHONE_NANP_PARENS },
+      { re: PHONE_NANP_PLAIN }
+    ]
+  }
+];
+
+/**
+ * Every span the enabled detectors claim, in detector order, with overlaps
+ * resolved in favour of whichever detector ran first. Returned sorted by
+ * start so the caller can rebuild the string in one pass.
+ * @param {string} text
+ * @param {Set<string>} entities
+ * @returns {Array<{ start: number, end: number, entity: string }>}
+ */
+function collectSpans(text, entities) {
+  const accepted = [];
+  for (const detector of DETECTORS) {
+    if (!entities.has(detector.entity)) continue;
+    for (const { re, check, keepsLabel } of detector.patterns) {
+      re.lastIndex = 0;
+      let match;
+      while ((match = re.exec(text)) !== null) {
+        if (match[0].length === 0) {
+          re.lastIndex++;
+          continue;
+        }
+        const end = match.index + match[0].length;
+        const start = keepsLabel ? match.index + match[1].length : match.index;
+        if (start >= end) continue;
+        if (check && !check(text.slice(start, end))) continue;
+        if (accepted.some(span => start < span.end && span.start < end)) continue;
+        accepted.push({ start, end, entity: detector.entity });
+      }
+    }
+  }
+  return accepted.sort((a, b) => a.start - b.start);
+}
+
+/**
+ * @param {string} entity
+ * @param {'tag'|'mask'|'remove'} style
+ * @returns {string}
+ */
+function replacementFor(entity, style) {
+  if (style === 'mask') return MASK;
+  if (style === 'remove') return '';
+  return `<${entity}>`;
+}
+
+/**
+ * Redact the entities this module can decide with a regex.
+ *
+ * `entities` narrows the run to its intersection with REGEX_ENTITIES. A name
+ * this module does not handle — PERSON and LOCATION included — is dropped in
+ * silence, and a selection that leaves nothing behind redacts nothing: a
+ * caller asking only for PERSON wants its own model pass, not everything.
+ * Anything that is not a non-empty array means all of REGEX_ENTITIES, because
+ * a missing selection should fail towards more redaction, not less.
+ *
+ * `replaceStyle` is 'tag' (`<EMAIL>`), 'mask' (`[REDACTED]`) or 'remove' (the
+ * empty string); anything else is treated as 'tag'. SECRET keeps its label in
+ * every style — see the module header.
+ *
+ * Never throws. A non-string `text` comes back untouched with a zero count.
+ *
+ * @param {string} text
+ * @param {{ entities?: string[], replaceStyle?: 'tag'|'mask'|'remove' }} [options]
+ * @returns {{ text: string, redaction: { entities: Record<string, number>, count: number } }}
+ */
+export function redactPii(text, { entities, replaceStyle } = {}) {
+  if (typeof text !== 'string' || text.length === 0) {
+    return { text, redaction: { entities: {}, count: 0 } };
+  }
+
+  const requested = Array.isArray(entities) && entities.length > 0
+    ? entities.filter(name => typeof name === 'string').map(name => name.toUpperCase())
+    : REGEX_ENTITIES;
+  const wanted = new Set(requested.filter(name => REGEX_ENTITIES.includes(name)));
+  if (wanted.size === 0) return { text, redaction: { entities: {}, count: 0 } };
+
+  const spans = collectSpans(text, wanted);
+  if (spans.length === 0) return { text, redaction: { entities: {}, count: 0 } };
+
+  const style = replaceStyle === 'mask' || replaceStyle === 'remove' ? replaceStyle : DEFAULT_REPLACE_STYLE;
+  const counts = {};
+  let out = '';
+  let cursor = 0;
+  for (const span of spans) {
+    out += text.slice(cursor, span.start) + replacementFor(span.entity, style);
+    counts[span.entity] = (counts[span.entity] || 0) + 1;
+    cursor = span.end;
+  }
+  out += text.slice(cursor);
+
+  return { text: out, redaction: { entities: counts, count: spans.length } };
+}