crawlforge-extractors 1.6.5 → 1.8.0

package/README.md

@@ -186,6 +186,57 @@ JSON-LD carries no per-variant stock count, compare-at price or option names,
 so those fields are null. Pass the page URL after redirects: a `/collections/`
 URL is reported in `reason` as a retired handle.
 
+### Recognising a blocked page
+
+`documentVerdict` says what a fetched document is: the page, a bot-wall
+interstitial (Cloudflare, Amazon, DataDome, PerimeterX, Akamai, Vercel), an
+HTTP error page, an empty shell, or a short error-titled placeholder. A wall
+arrives as HTTP 200 with a title and prose of its own, so a fetch that only
+checks the status reports it as a success — producthunt.com came back
+`success: true, title: "Just a moment..."` for three regression rounds.
+
+```js
+import { documentVerdict } from 'crawlforge-extractors';
+
+const verdict = documentVerdict(
+  { url: response.url, status: response.status, title, text, html },
+  { fetcher: 'a plain fetch', rendered: false, contentReturned: false }
+);
+// { success: false, status: 200, blocked: { vendor: 'cloudflare', evidence: 'title "Just a moment..."' }, error: '…' }
+```
+
+`detectChallengePage` is the vendor check alone. Both are pure: the caller
+keeps or drops the content, and decides what to try next.
+
+### Finding the units that match a query
+
+`segmentUnits` cuts the markdown a scrape returned into sentences, table rows
+and fenced code blocks; `rankUnits` scores them against a query with BM25 and
+returns the best few verbatim. It is how a caller gets the one table row that
+answers "enterprise price per month" without paying to read the whole page,
+and without a model paraphrasing it — nothing here can say something the page
+does not.
+
+```js
+import { segmentUnits, rankUnits } from 'crawlforge-extractors';
+
+const units = segmentUnits(markdown);
+const best = rankUnits(units, 'enterprise price per month', { maxUnits: 5 });
+// [{ text: 'Enterprise is priced at $499 per month, billed annually, and includes a dedicated success engineer.',
+//    kind: 'sentence', offset: 815, length: 99, heading: 'Enterprise', score: 6.612 },
+//  { text: '| Price per month | $29 | $99 | $499 |',
+//    kind: 'table_row', offset: 257, length: 38, heading: 'Compare plans', score: 5.809 }, …]
+```
+
+Offsets are JS string indexes into the exact string passed in, and every unit
+keeps `markdown.slice(offset, offset + length) === text`: trimming and list
+markers move the offset, nothing rewrites the text. Headings are not units;
+each unit carries the heading above it. The sentence splitter shares the MCP
+server's terminator rules — `。！？` and the danda split on a zero-width
+boundary, and an ASCII period does not split after `Dr.`, `e.g.`, `Node.js`
+or `3.14`. A CJK query matches by character bigrams, so no segmenter is
+needed.
+
 ## Templates
 
 **Pages and products.** `shopify-product` · `shopify-collection` ·

package/index.d.ts

@@ -273,3 +273,95 @@ export declare function shopifyProductFromJsonLd(
 ): { found: true; data: ShopifyJsonLdProduct } | { found: false; reason: string };
 
 export default TemplateRegistry;
+
+/** The bot-defence vendor whose interstitial a document is, and what gave it away. */
+export interface ChallengeVerdict {
+  vendor: 'cloudflare' | 'amazon' | 'datadome' | 'perimeterx' | 'akamai' | 'vercel';
+  evidence: string;
+}
+
+/**
+ * Recognise a bot-wall interstitial served as a page (HTTP 200, a title, some
+ * prose and a challenge script). A title match is definitive; a script or
+ * form marker is definitive only on a short page, because a real page can
+ * legitimately embed a Turnstile widget.
+ */
+export declare function detectChallengePage(page: {
+  title?: string;
+  html?: string;
+  text?: string;
+}): ChallengeVerdict | null;
+
+export interface DocumentVerdict {
+  success: boolean;
+  /** The navigation's HTTP status when the caller had one, else null. */
+  status: number | null;
+  /** Present on every failure: what the document is and what to do. */
+  error?: string;
+  /** Present when the failure is a challenge wall. */
+  blocked?: ChallengeVerdict;
+}
+
+/**
+ * What a fetched document is: the page, a challenge wall, an HTTP error page,
+ * an empty shell, or a short error-titled placeholder. The defaults describe
+ * a stealth-browser caller: a browser `rendered` the document, `fetcher`
+ * names it in the messages, `waitedMs` is the extra render wait it gave an
+ * empty document, and the failure result still carries the content
+ * (`contentReturned`). A plain fetch passes `rendered: false` and
+ * `contentReturned: false`.
+ */
+export declare function documentVerdict(
+  scraped: { url?: string; title?: string; text?: string; html?: string; status?: number | null },
+  options?: {
+    waitedMs?: number;
+    allowEmpty?: boolean;
+    fetcher?: string;
+    rendered?: boolean;
+    contentReturned?: boolean;
+  }
+): DocumentVerdict;
+
+/** A document with this much text or less and an error title is a placeholder. */
+export declare const SOFT_ERROR_MAX_CHARS: number;
+
+/** One scoreable piece of a page's markdown: a sentence, a table row or a fenced code block. */
+export interface HighlightUnit {
+  /** Verbatim from the markdown: `markdown.slice(offset, offset + length) === text`. */
+  text: string;
+  kind: 'sentence' | 'table_row' | 'code_block';
+  /** JS string index into the markdown segmentUnits was given — not a byte offset. */
+  offset: number;
+  length: number;
+  /** The nearest heading above the unit, without its `#` marks; null before the first heading. */
+  heading: string | null;
+}
+
+export interface RankedHighlightUnit extends HighlightUnit {
+  /** BM25 with the phrase boost applied and the heading's terms counted at half weight, rounded to 3 decimals; always above `minScore`. */
+  score: number;
+}
+
+/**
+ * Cut the markdown a scrape returned into sentences, table rows and fenced
+ * code blocks, each with its offset into that same string. Headings are not
+ * units; they label the units that follow. List and quote markers are
+ * skipped by moving the offset, never by rewriting the text. The sentence
+ * splitter carries the MCP server's rules: 。！？ and the danda split on a
+ * zero-width boundary, and an ASCII period does not split after an
+ * abbreviation, a word with internal periods, a decimal or an initial.
+ */
+export declare function segmentUnits(markdown: string): HighlightUnit[];
+
+/**
+ * The units that answer a query, best first: BM25 over the units as the
+ * corpus, each unit inheriting its heading's terms at half weight, ×1.5 when
+ * a unit contains the whole query. Units scoring at or below `minScore` (default 0: no
+ * term in common) are dropped; at most `maxUnits` (default 10, clamped to at
+ * least 1) come back, ties broken by offset. The input is not modified.
+ */
+export declare function rankUnits(
+  units: HighlightUnit[],
+  query: string,
+  options?: { maxUnits?: number; minScore?: number }
+): RankedHighlightUnit[];

package/index.js

@@ -31,3 +31,7 @@ export { extractEmbeddedState } from './src/embeddedState.js';
 export { parseJsonPath, selectJsonPath } from './src/jsonPath.js';
 
 export { shopifyProductFromJsonLd } from './src/shopifyJsonLd.js';
+
+export { detectChallengePage, documentVerdict, SOFT_ERROR_MAX_CHARS } from './src/blockedPage.js';
+
+export { segmentUnits, rankUnits } from './src/highlights.js';

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "crawlforge-extractors",
-  "version": "1.6.5",
-  "description": "Extraction logic shared by the CrawlForge MCP server and REST API — scrape templates, charset-correct capped body reading, structural fingerprinting, and embedded-state extraction. One implementation, so the two surfaces cannot drift apart.",
+  "version": "1.8.0",
+  "description": "Extraction logic shared by the CrawlForge MCP server and REST API — scrape templates, charset-correct capped body reading, structural fingerprinting, embedded-state extraction, and query-scoped highlights. One implementation, so the two surfaces cannot drift apart.",
   "type": "module",
   "main": "./index.js",
   "types": "./index.d.ts",

package/src/blockedPage.js

@@ -0,0 +1,168 @@
+/**
+ * blockedPage.js — decide whether a fetched document is the page or a wall.
+ *
+ * Cloudflare, Amazon, DataDome, PerimeterX, Akamai and Vercel all answer a
+ * blocked request with HTTP 200 and a page of their own: a title, some prose
+ * and a challenge script. Reported as a successful scrape, that page hides
+ * the block — producthunt.com came back "success:true, title: Just a
+ * moment..." for three regression rounds (R10 Q1 → R15, 2026-09-04). The
+ * MCP server's stealth path learned to name these in 5.6.2 and to name HTTP
+ * error pages and short error-titled placeholders in 5.6.9; the plain
+ * `scrape` path on both surfaces never looked. The tables live here so the
+ * two surfaces reach one verdict.
+ *
+ * A title match is definitive; a script or form marker is definitive only
+ * on a short page, because a real page can legitimately embed a Turnstile
+ * widget.
+ */
+
+const SHORT_PAGE_CHARS = 4000;
+
+const CHALLENGES = [
+  {
+    vendor: 'cloudflare',
+    title: /^just a moment/i,
+    markers: /challenges\.cloudflare\.com|cf-chl-|_cf_chl_opt|cf_chl_rc_|window\._cf_chl/i,
+    evidence: 'a Cloudflare challenge script'
+  },
+  {
+    vendor: 'amazon',
+    // The robot check is the only Amazon page whose form posts to validateCaptcha.
+    definitive: /action="[^"]*validateCaptcha/i,
+    evidence: 'the validateCaptcha form'
+  },
+  {
+    vendor: 'datadome',
+    markers: /captcha-delivery\.com\/captcha|geo\.captcha-delivery\.com|dd\.captcha/i,
+    evidence: 'a DataDome captcha frame'
+  },
+  {
+    vendor: 'perimeterx',
+    markers: /px-captcha|_pxCaptcha|human-challenge/i,
+    evidence: 'a PerimeterX / HUMAN challenge element'
+  },
+  {
+    vendor: 'akamai',
+    title: /^access denied/i,
+    markers: /errors\.edgesuite\.net/i,
+    evidence: 'an Akamai access-denied page'
+  },
+  {
+    // Vercel's Attack Challenge Mode answers with HTTP 429, an
+    // x-vercel-mitigated: challenge header and a JavaScript interstitial
+    // titled "Vercel Security Checkpoint" (lesswrong.com, hashicorp.com,
+    // bombas.com, R17 2026-09-04). Chromium solves it and reloads; camoufox
+    // was left on the interstitial, which then read as a successful scrape.
+    vendor: 'vercel',
+    title: /^vercel security checkpoint/i,
+    markers: /vercel\.link\/security-checkpoint|_vercel\/challenge|x-vercel-challenge-token/i,
+    evidence: 'a Vercel Security Checkpoint page'
+  }
+];
+
+/**
+ * @param {{ title?: string, html?: string, text?: string }} page
+ * @returns {{ vendor: string, evidence: string } | null}
+ */
+export function detectChallengePage({ title = '', html = '', text = '' } = {}) {
+  const cleanTitle = (title || '').trim();
+  const visible = (text || '').replace(/\s+/g, ' ').trim();
+  const shortPage = visible.length < SHORT_PAGE_CHARS;
+  for (const challenge of CHALLENGES) {
+    if (challenge.title && challenge.title.test(cleanTitle)) {
+      return { vendor: challenge.vendor, evidence: `title "${cleanTitle}"` };
+    }
+    if (challenge.definitive && challenge.definitive.test(html)) {
+      return { vendor: challenge.vendor, evidence: challenge.evidence };
+    }
+    if (shortPage && challenge.markers && challenge.markers.test(html)) {
+      return { vendor: challenge.vendor, evidence: `${challenge.evidence} on a ${visible.length}-character page` };
+    }
+  }
+  return null;
+}
+
+// A document this short with one of these titles is an error placeholder,
+// not a page. Real pages with these words in a longer title (a news story
+// about an outage) carry far more text than the cap.
+const ERROR_TITLE = /^(?:(?:\d{3}\s*[-–—|:]\s*)?(?:error(?: page)?|access denied|forbidden|(?:page )?not found|service unavailable|internal server error|bad gateway|something went wrong|oops!?[^\n]{0,60}))$/i;
+export const SOFT_ERROR_MAX_CHARS = 1500;
+
+/**
+ * What a fetched document is: the page, a challenge wall, an HTTP error
+ * page, an empty shell, or a short error-titled placeholder. The content is
+ * for the caller to keep or drop — this only says what it is.
+ *
+ * The defaults describe the MCP server's stealth path, the original caller:
+ * a browser `rendered` the document, `fetcher` names it in the messages,
+ * `waitedMs` is the extra render wait it gave an empty document, and the
+ * failure result still carries the content (`contentReturned`). A plain
+ * fetch passes `rendered: false` (its empty shell or placeholder cannot be
+ * waited out — only a browser paints it) and `contentReturned: false` (it
+ * drops the document on a failure).
+ *
+ * @param {{ url?: string, title?: string, text?: string, html?: string, status?: number|null }} scraped
+ * @param {{ waitedMs?: number, allowEmpty?: boolean, fetcher?: string, rendered?: boolean, contentReturned?: boolean }} [options]
+ * @returns {{ success: boolean, status: number|null, error?: string, blocked?: { vendor: string, evidence: string } }}
+ */
+export function documentVerdict(scraped, { waitedMs = 0, allowEmpty = false, fetcher = 'the stealth browser', rendered = true, contentReturned = true } = {}) {
+  const status = Number.isInteger(scraped?.status) ? scraped.status : null;
+  const url = scraped?.url || '';
+  const title = String(scraped?.title || '').trim();
+  const text = String(scraped?.text || '').trim();
+
+  const challenge = detectChallengePage(scraped || {});
+  if (challenge) {
+    return {
+      success: false,
+      status,
+      blocked: challenge,
+      error: `${challenge.vendor} served a challenge page instead of the content (${challenge.evidence}); ${fetcher} did not pass it.`
+    };
+  }
+
+  if (status !== null && status >= 400) {
+    const why = status === 403
+      ? 'A 403 with no challenge vendor on the page is an IP-reputation or WAF block; the site will not serve this network.'
+      : status === 404
+        ? 'The site says the URL does not exist — check the path.'
+        : status === 429
+          ? 'The site is rate-limiting this network; wait before retrying.'
+          : 'Retry later; the server, not the page, failed.';
+    return {
+      success: false,
+      status,
+      error: `HTTP ${status}: ${url} answered with an error page${title ? ` titled "${title}"` : ''}, not the resource${contentReturned ? '; the content returned is that page' : ''}. ${why}`
+    };
+  }
+
+  if (!title && !text) {
+    if (allowEmpty) return { success: true, status };
+    const reached = fetcher.charAt(0).toUpperCase() + fetcher.slice(1);
+    const bytes = (scraped?.html || '').length;
+    return {
+      success: false,
+      status,
+      error: rendered
+        ? `${reached} reached ${url} but the document rendered no title and no text` +
+          ` after ${waitedMs}ms of extra wait (${bytes} bytes of HTML).` +
+          ' A JavaScript-rendered page needs a longer wait_for; an empty document means the server sent nothing to render.'
+        : `${reached} reached ${url} but the document has no title and no text (${bytes} bytes of HTML).` +
+          ' The page is rendered by JavaScript or the server sent an empty shell; only a browser renders it.'
+    };
+  }
+
+  if (text.length < SOFT_ERROR_MAX_CHARS && ERROR_TITLE.test(title)) {
+    return {
+      success: false,
+      status,
+      error:
+        `${url} rendered an error page titled "${title}" (${text.length} characters of text) instead of the resource` +
+        (rendered
+          ? ' — a soft block or an application error. Retry later, or with a longer wait_for if the site paints content after a placeholder.'
+          : ' — a soft block or an application error. Retry later; if the site paints content after a placeholder, only a browser renders it.')
+    };
+  }
+
+  return { success: true, status };
+}

package/src/highlights.js

@@ -0,0 +1,342 @@
+/**
+ * highlights.js — the units of a scraped page that answer a query, verbatim.
+ *
+ * A caller asking "what does the enterprise plan cost" does not want the
+ * whole markdown of a pricing page in its context window, and it does not
+ * want a model's paraphrase of it either — the first is expensive, the
+ * second can be wrong. This module is the extractive middle: cut the
+ * markdown a scrape already returned into units (sentences, table rows,
+ * fenced code blocks), score each against the query with BM25, and hand
+ * back the top few exactly as they appear on the page, with character
+ * offsets into that same string so the caller can quote with a locator.
+ * Both surfaces run it, so a highlight is the same highlight over MCP and
+ * REST.
+ *
+ * Everything here is a pure function over the markdown string. Offsets are
+ * JS string indexes, and the invariant every unit keeps is
+ * `markdown.slice(offset, offset + length) === text`. Trimming moves the
+ * offsets; nothing rewrites the text.
+ *
+ * The sentence splitter ports the MCP server's sentenceUtils.js rules: the
+ * CJK / fullwidth / Devanagari terminators split on a zero-width boundary
+ * and are never judged by the ASCII checks; an ASCII `.` followed by
+ * whitespace does not split after an abbreviation (Dr., etc.), a word with
+ * internal periods (Node.js, e.g.), a decimal (3.14) or a single-letter
+ * initial. The one departure from the server: those four checks only guard
+ * `.`, because "What is Node.js? It is a runtime." is two sentences.
+ */
+
+const CJK_TERMINATORS = '。．！？；।॥';
+const ASCII_TERMINATORS = '.!?';
+// Closing punctuation a terminator may carry with it: "…end."), 'so.', and
+// the emphasis markers of a bold FAQ question — "**Can I cancel?** Yes."
+const CLOSERS = '"\')\\]”’»*_';
+
+const ABBREVIATIONS = new Set([
+  'mr', 'mrs', 'ms', 'dr', 'prof', 'sr', 'jr', 'st', 'ave', 'blvd',
+  'vs', 'etc', 'inc', 'ltd', 'corp', 'dept', 'univ', 'assn',
+  'approx', 'appt', 'apt', 'est', 'min', 'max',
+  'govt', 'lib', 'misc', 'natl', 'intl',
+  'jan', 'feb', 'mar', 'apr', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec',
+  'mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun',
+  'fig', 'eq', 'ref', 'vol', 'no', 'pp', 'ed', 'rev',
+  'e', 'i' // e.g. and i.e.
+]);
+
+const MIN_UNIT_CHARS = 2;
+
+const FENCE = /^(`{3,}|~{3,})/;
+const FENCE_CLOSER = /^(?:`+|~+)$/;
+const HEADING = /^#{1,6}(?:\s+(.*?))?\s*#*\s*$/;
+const TABLE_DELIMITER = /^\|?[\s:|-]*-[\s:|-]*\|?$/;
+const HORIZONTAL_RULE = /^(?:-\s*){3,}$|^(?:\*\s*){3,}$|^(?:_\s*){3,}$/;
+const HTML_COMMENT_LINE = /^<!--[\s\S]*-->$/;
+const IMAGE_ONLY_LINE = /^!\[[^\]]*\]\([^)]*\)$/;
+const BLOCKQUOTE_MARKER = /^>\s?/;
+const LIST_MARKER = /^(?:[-*+]|\d{1,9}[.)])\s+/;
+
+function isSpace(ch) {
+  return /\s/.test(ch);
+}
+
+/** Move [start, end) inward past whitespace at both ends. */
+function trimRange(markdown, start, end) {
+  while (start < end && isSpace(markdown[start])) start++;
+  while (end > start && isSpace(markdown[end - 1])) end--;
+  return [start, end];
+}
+
+function hasEnoughText(text) {
+  let count = 0;
+  for (const ch of text) {
+    if (!isSpace(ch) && ++count >= MIN_UNIT_CHARS) return true;
+  }
+  return false;
+}
+
+/**
+ * Cut the markdown a scrape returned into the units a query can be scored
+ * against. Headings are not units — they label the units that follow.
+ *
+ * @param {string} markdown
+ * @returns {Array<{ text: string, kind: 'sentence' | 'table_row' | 'code_block', offset: number, length: number, heading: string | null }>}
+ */
+export function segmentUnits(markdown) {
+  if (typeof markdown !== 'string' || markdown.length === 0) return [];
+
+  const units = [];
+  let heading = null;
+
+  function push(kind, start, end) {
+    [start, end] = trimRange(markdown, start, end);
+    const text = markdown.slice(start, end);
+    if (!hasEnoughText(text)) return;
+    units.push({ text, kind, offset: start, length: end - start, heading });
+  }
+
+  // A paragraph is a contiguous range: consecutive prose lines, the first
+  // one's list or quote marker already skipped. Line breaks inside it are
+  // whitespace, so a sentence may carry a "\n" verbatim.
+  let paragraphStart = -1;
+  let paragraphEnd = -1;
+  function flushParagraph() {
+    if (paragraphStart >= 0) splitSentences(markdown, paragraphStart, paragraphEnd, push);
+    paragraphStart = -1;
+  }
+
+  let fence = null; // { marker, contentStart }
+
+  const length = markdown.length;
+  let lineStart = 0;
+  while (lineStart <= length) {
+    let lineEnd = markdown.indexOf('\n', lineStart);
+    if (lineEnd === -1) lineEnd = length;
+    const [s, e] = trimRange(markdown, lineStart, lineEnd);
+    const line = markdown.slice(s, e);
+    let match;
+
+    if (fence) {
+      if (line[0] === fence.marker[0] && line.length >= fence.marker.length && FENCE_CLOSER.test(line)) {
+        push('code_block', fence.contentStart, Math.max(fence.contentStart, lineStart));
+        fence = null;
+      }
+    } else if (line === '') {
+      flushParagraph();
+    } else if ((match = FENCE.exec(line))) {
+      flushParagraph();
+      fence = { marker: match[1], contentStart: Math.min(lineEnd + 1, length) };
+    } else if ((match = HEADING.exec(line))) {
+      flushParagraph();
+      heading = (match[1] || '').trim() || null;
+    } else if (line[0] === '|') {
+      flushParagraph();
+      if (!TABLE_DELIMITER.test(line)) push('table_row', s, e);
+    } else if (HORIZONTAL_RULE.test(line) || HTML_COMMENT_LINE.test(line) || IMAGE_ONLY_LINE.test(line)) {
+      flushParagraph();
+    } else {
+      let start = s;
+      let marked = false;
+      while ((match = BLOCKQUOTE_MARKER.exec(markdown.slice(start, e)))) {
+        start += match[0].length;
+        marked = true;
+      }
+      if ((match = LIST_MARKER.exec(markdown.slice(start, e)))) {
+        start += match[0].length;
+        marked = true;
+      }
+      // A marker starts its own paragraph: two list items are two units,
+      // and "- " never lands inside a sentence's text.
+      if (marked) flushParagraph();
+      if (paragraphStart < 0) paragraphStart = start;
+      paragraphEnd = e;
+    }
+
+    lineStart = lineEnd + 1;
+  }
+
+  // A fence nobody closed: the rest of the document is that block.
+  if (fence) push('code_block', fence.contentStart, length);
+  flushParagraph();
+
+  return units;
+}
+
+/**
+ * Split the paragraph at [start, end) into sentences, calling `emit` with
+ * each one's range. Whitespace between sentences belongs to neither.
+ */
+function splitSentences(markdown, start, end, emit) {
+  let sentenceStart = start;
+  let i = start;
+  while (i < end) {
+    const ch = markdown[i];
+    if (CJK_TERMINATORS.includes(ch)) {
+      // Unambiguous, and CJK text puts no whitespace after them.
+      emit('sentence', sentenceStart, i + 1);
+      sentenceStart = i + 1;
+      i++;
+      continue;
+    }
+    if (!ASCII_TERMINATORS.includes(ch)) {
+      i++;
+      continue;
+    }
+    // Consume the run: "?!", "...", then any closing quote or bracket.
+    let j = i;
+    while (j < end && ASCII_TERMINATORS.includes(markdown[j])) j++;
+    while (j < end && CLOSERS.includes(markdown[j])) j++;
+    if (j < end && !isSpace(markdown[j])) {
+      // "Node.js", "3.14", "e.g." — a period glued to the next word.
+      i = j;
+      continue;
+    }
+    if (ch === '.' && isFalseStop(markdown, sentenceStart, i)) {
+      i = j;
+      continue;
+    }
+    emit('sentence', sentenceStart, j);
+    sentenceStart = j;
+    i = j;
+  }
+  if (sentenceStart < end) emit('sentence', sentenceStart, end);
+}
+
+/**
+ * Whether the period at `dot` ends an abbreviation, a word with internal
+ * periods, a decimal or a single-letter initial — the server's four checks,
+ * applied to the whitespace-delimited word before the period.
+ */
+function isFalseStop(markdown, from, dot) {
+  let wordStart = dot;
+  while (wordStart > from && !isSpace(markdown[wordStart - 1])) wordStart--;
+  const word = markdown.slice(wordStart, dot);
+  if (word === '') return false;
+  if (ABBREVIATIONS.has(word.toLowerCase().replace(/[^a-z]/g, ''))) return true;
+  if (/\w\.\w/.test(word)) return true;
+  if (/\d\.\d/.test(word)) return true;
+  return /^[A-Z]$/.test(word);
+}
+
+// Letters and digits of the scripts that put no spaces between words (Han,
+// Hiragana, Katakana, Hangul) form one kind of run; every other letter,
+// digit or combining mark forms the other — marks, because a Devanagari
+// vowel sign is a mark, and without them "कीमत" is two fragments.
+// Script_Extensions keeps "ー" and "々" with their runs; the letter/number
+// class keeps "、" and "。" out of them.
+const CJK_CHAR = '[\\p{scx=Han}\\p{scx=Hiragana}\\p{scx=Katakana}\\p{scx=Hangul}]';
+const TOKEN = new RegExp(`(?:(?=${CJK_CHAR})[\\p{L}\\p{N}])+|(?:(?!${CJK_CHAR})[\\p{L}\\p{N}\\p{M}])+`, 'gu');
+const CJK_START = new RegExp(`^${CJK_CHAR}`, 'u');
+const SUFFIXES = ['ing', 'ed', 'es', 's'];
+
+/**
+ * Lowercase word tokens. A CJK run becomes character bigrams so a query in
+ * one of those scripts can match without a segmenter; a light suffix strip
+ * puts "pricing", "prices" and "price" on one stem ("pric").
+ * @param {string} text
+ * @returns {string[]}
+ */
+function tokenize(text) {
+  const tokens = [];
+  for (const [run] of text.toLowerCase().matchAll(TOKEN)) {
+    if (CJK_START.test(run)) {
+      const chars = Array.from(run);
+      if (chars.length === 1) tokens.push(run);
+      for (let i = 0; i + 1 < chars.length; i++) tokens.push(chars[i] + chars[i + 1]);
+    } else {
+      tokens.push(stem(run));
+    }
+  }
+  return tokens;
+}
+
+function stem(token) {
+  if (token.length <= 4) return token;
+  for (const suffix of SUFFIXES) {
+    if (token.endsWith(suffix)) {
+      token = token.slice(0, -suffix.length);
+      break;
+    }
+  }
+  return token.length > 4 && token.endsWith('e') ? token.slice(0, -1) : token;
+}
+
+const K1 = 1.2;
+const B = 0.75;
+const PHRASE_BOOST = 1.5;
+// A unit inherits the terms of the heading it sits under, at half the weight
+// of a term in its own text: on a card-style pricing page the plan name is
+// the heading and "$83/month" is the unit, and without the heading the price
+// line shares nothing with "professional plan price per month". Heading
+// terms count toward a unit's term frequency only — not toward document
+// frequency, or the plan name would look common and lose its weight.
+const HEADING_WEIGHT = 0.5;
+// BM25's length normalisation rewards short documents, and a page's shortest
+// units are its buttons: "Choose Professional" outranked every price line on
+// a live pricing page. A unit is scored as if it had at least this many
+// tokens, so a two-word call to action carries no length advantage.
+const MIN_DOC_LENGTH = 4;
+
+/**
+ * The units that answer a query, best first: BM25 over the units as the
+ * corpus, with each unit inheriting its heading's terms at half weight, and a
+ * phrase boost when a unit contains the whole query. The input units are not
+ * touched; every returned unit is a new object.
+ *
+ * @template {{ text: string, heading?: string | null, offset: number }} U
+ * @param {U[]} units
+ * @param {string} query
+ * @param {{ maxUnits?: number, minScore?: number }} [options]
+ * @returns {Array<U & { score: number }>}
+ */
+export function rankUnits(units, query, { maxUnits = 10, minScore = 0 } = {}) {
+  if (!Array.isArray(units) || units.length === 0 || typeof query !== 'string') return [];
+  const phrase = query.trim().toLowerCase();
+  if (phrase === '') return [];
+  const terms = [...new Set(tokenize(phrase))];
+  if (terms.length === 0) return [];
+
+  const limit = Number.isFinite(maxUnits) ? Math.max(1, Math.floor(maxUnits)) : 10;
+  const floor = Number.isFinite(minScore) ? minScore : 0;
+
+  const docs = units.map((unit) => {
+    const counts = new Map();
+    const tokens = tokenize(String(unit.text ?? ''));
+    for (const token of tokens) counts.set(token, (counts.get(token) || 0) + 1);
+    const own = new Set(counts.keys());
+    let length = tokens.length;
+    if (unit.heading) {
+      for (const token of new Set(tokenize(String(unit.heading)))) {
+        if (own.has(token)) continue;
+        counts.set(token, HEADING_WEIGHT);
+        length += HEADING_WEIGHT;
+      }
+    }
+    return { counts, own, length: Math.max(length, MIN_DOC_LENGTH) };
+  });
+  const n = docs.length;
+  const avgdl = docs.reduce((sum, doc) => sum + doc.length, 0) / n || 1;
+  const idf = new Map(terms.map((term) => {
+    const df = docs.reduce((count, doc) => count + (doc.own.has(term) ? 1 : 0), 0);
+    return [term, Math.log(1 + (n - df + 0.5) / (df + 0.5))];
+  }));
+
+  const ranked = [];
+  units.forEach((unit, index) => {
+    const doc = docs[index];
+    let score = 0;
+    for (const term of terms) {
+      const tf = doc.counts.get(term);
+      if (!tf) continue;
+      score += idf.get(term) * (tf * (K1 + 1)) / (tf + K1 * (1 - B + B * doc.length / avgdl));
+    }
+    if (score === 0) return;
+    if (String(unit.text).toLowerCase().includes(phrase)) score *= PHRASE_BOOST;
+    // Round before the threshold, so a score handed back as minScore means
+    // what the caller saw.
+    score = Math.round(score * 1000) / 1000;
+    if (score > floor) ranked.push({ ...unit, score });
+  });
+
+  ranked.sort((a, b) => b.score - a.score || a.offset - b.offset);
+  return ranked.slice(0, limit);
+}