crawlforge-extractors 1.7.0 → 1.8.0

package/README.md

@@ -208,6 +208,35 @@ const verdict = documentVerdict(
 `detectChallengePage` is the vendor check alone. Both are pure: the caller
 keeps or drops the content, and decides what to try next.
 
+### Finding the units that match a query
+
+`segmentUnits` cuts the markdown a scrape returned into sentences, table rows
+and fenced code blocks; `rankUnits` scores them against a query with BM25 and
+returns the best few verbatim. It is how a caller gets the one table row that
+answers "enterprise price per month" without paying to read the whole page,
+and without a model paraphrasing it — nothing here can say something the page
+does not.
+
+```js
+import { segmentUnits, rankUnits } from 'crawlforge-extractors';
+
+const units = segmentUnits(markdown);
+const best = rankUnits(units, 'enterprise price per month', { maxUnits: 5 });
+// [{ text: 'Enterprise is priced at $499 per month, billed annually, and includes a dedicated success engineer.',
+//    kind: 'sentence', offset: 815, length: 99, heading: 'Enterprise', score: 6.612 },
+//  { text: '| Price per month | $29 | $99 | $499 |',
+//    kind: 'table_row', offset: 257, length: 38, heading: 'Compare plans', score: 5.809 }, …]
+```
+
+Offsets are JS string indexes into the exact string passed in, and every unit
+keeps `markdown.slice(offset, offset + length) === text`: trimming and list
+markers move the offset, nothing rewrites the text. Headings are not units;
+each unit carries the heading above it. The sentence splitter shares the MCP
+server's terminator rules — `。！？` and the danda split on a zero-width
+boundary, and an ASCII period does not split after `Dr.`, `e.g.`, `Node.js`
+or `3.14`. A CJK query matches by character bigrams, so no segmenter is
+needed.
+
 ## Templates
 
 **Pages and products.** `shopify-product` · `shopify-collection` ·

package/index.d.ts

@@ -324,3 +324,44 @@ export declare function documentVerdict(
 
 /** A document with this much text or less and an error title is a placeholder. */
 export declare const SOFT_ERROR_MAX_CHARS: number;
+
+/** One scoreable piece of a page's markdown: a sentence, a table row or a fenced code block. */
+export interface HighlightUnit {
+  /** Verbatim from the markdown: `markdown.slice(offset, offset + length) === text`. */
+  text: string;
+  kind: 'sentence' | 'table_row' | 'code_block';
+  /** JS string index into the markdown segmentUnits was given — not a byte offset. */
+  offset: number;
+  length: number;
+  /** The nearest heading above the unit, without its `#` marks; null before the first heading. */
+  heading: string | null;
+}
+
+export interface RankedHighlightUnit extends HighlightUnit {
+  /** BM25 with the phrase boost applied and the heading's terms counted at half weight, rounded to 3 decimals; always above `minScore`. */
+  score: number;
+}
+
+/**
+ * Cut the markdown a scrape returned into sentences, table rows and fenced
+ * code blocks, each with its offset into that same string. Headings are not
+ * units; they label the units that follow. List and quote markers are
+ * skipped by moving the offset, never by rewriting the text. The sentence
+ * splitter carries the MCP server's rules: 。！？ and the danda split on a
+ * zero-width boundary, and an ASCII period does not split after an
+ * abbreviation, a word with internal periods, a decimal or an initial.
+ */
+export declare function segmentUnits(markdown: string): HighlightUnit[];
+
+/**
+ * The units that answer a query, best first: BM25 over the units as the
+ * corpus, each unit inheriting its heading's terms at half weight, ×1.5 when
+ * a unit contains the whole query. Units scoring at or below `minScore` (default 0: no
+ * term in common) are dropped; at most `maxUnits` (default 10, clamped to at
+ * least 1) come back, ties broken by offset. The input is not modified.
+ */
+export declare function rankUnits(
+  units: HighlightUnit[],
+  query: string,
+  options?: { maxUnits?: number; minScore?: number }
+): RankedHighlightUnit[];

package/index.js

@@ -33,3 +33,5 @@ export { parseJsonPath, selectJsonPath } from './src/jsonPath.js';
 export { shopifyProductFromJsonLd } from './src/shopifyJsonLd.js';
 
 export { detectChallengePage, documentVerdict, SOFT_ERROR_MAX_CHARS } from './src/blockedPage.js';
+
+export { segmentUnits, rankUnits } from './src/highlights.js';

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "crawlforge-extractors",
-  "version": "1.7.0",
-  "description": "Extraction logic shared by the CrawlForge MCP server and REST API — scrape templates, charset-correct capped body reading, structural fingerprinting, and embedded-state extraction. One implementation, so the two surfaces cannot drift apart.",
+  "version": "1.8.0",
+  "description": "Extraction logic shared by the CrawlForge MCP server and REST API — scrape templates, charset-correct capped body reading, structural fingerprinting, embedded-state extraction, and query-scoped highlights. One implementation, so the two surfaces cannot drift apart.",
   "type": "module",
   "main": "./index.js",
   "types": "./index.d.ts",

package/src/highlights.js

@@ -0,0 +1,342 @@
+/**
+ * highlights.js — the units of a scraped page that answer a query, verbatim.
+ *
+ * A caller asking "what does the enterprise plan cost" does not want the
+ * whole markdown of a pricing page in its context window, and it does not
+ * want a model's paraphrase of it either — the first is expensive, the
+ * second can be wrong. This module is the extractive middle: cut the
+ * markdown a scrape already returned into units (sentences, table rows,
+ * fenced code blocks), score each against the query with BM25, and hand
+ * back the top few exactly as they appear on the page, with character
+ * offsets into that same string so the caller can quote with a locator.
+ * Both surfaces run it, so a highlight is the same highlight over MCP and
+ * REST.
+ *
+ * Everything here is a pure function over the markdown string. Offsets are
+ * JS string indexes, and the invariant every unit keeps is
+ * `markdown.slice(offset, offset + length) === text`. Trimming moves the
+ * offsets; nothing rewrites the text.
+ *
+ * The sentence splitter ports the MCP server's sentenceUtils.js rules: the
+ * CJK / fullwidth / Devanagari terminators split on a zero-width boundary
+ * and are never judged by the ASCII checks; an ASCII `.` followed by
+ * whitespace does not split after an abbreviation (Dr., etc.), a word with
+ * internal periods (Node.js, e.g.), a decimal (3.14) or a single-letter
+ * initial. The one departure from the server: those four checks only guard
+ * `.`, because "What is Node.js? It is a runtime." is two sentences.
+ */
+
+const CJK_TERMINATORS = '。．！？；।॥';
+const ASCII_TERMINATORS = '.!?';
+// Closing punctuation a terminator may carry with it: "…end."), 'so.', and
+// the emphasis markers of a bold FAQ question — "**Can I cancel?** Yes."
+const CLOSERS = '"\')\\]”’»*_';
+
+const ABBREVIATIONS = new Set([
+  'mr', 'mrs', 'ms', 'dr', 'prof', 'sr', 'jr', 'st', 'ave', 'blvd',
+  'vs', 'etc', 'inc', 'ltd', 'corp', 'dept', 'univ', 'assn',
+  'approx', 'appt', 'apt', 'est', 'min', 'max',
+  'govt', 'lib', 'misc', 'natl', 'intl',
+  'jan', 'feb', 'mar', 'apr', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec',
+  'mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun',
+  'fig', 'eq', 'ref', 'vol', 'no', 'pp', 'ed', 'rev',
+  'e', 'i' // e.g. and i.e.
+]);
+
+const MIN_UNIT_CHARS = 2;
+
+const FENCE = /^(`{3,}|~{3,})/;
+const FENCE_CLOSER = /^(?:`+|~+)$/;
+const HEADING = /^#{1,6}(?:\s+(.*?))?\s*#*\s*$/;
+const TABLE_DELIMITER = /^\|?[\s:|-]*-[\s:|-]*\|?$/;
+const HORIZONTAL_RULE = /^(?:-\s*){3,}$|^(?:\*\s*){3,}$|^(?:_\s*){3,}$/;
+const HTML_COMMENT_LINE = /^<!--[\s\S]*-->$/;
+const IMAGE_ONLY_LINE = /^!\[[^\]]*\]\([^)]*\)$/;
+const BLOCKQUOTE_MARKER = /^>\s?/;
+const LIST_MARKER = /^(?:[-*+]|\d{1,9}[.)])\s+/;
+
+function isSpace(ch) {
+  return /\s/.test(ch);
+}
+
+/** Move [start, end) inward past whitespace at both ends. */
+function trimRange(markdown, start, end) {
+  while (start < end && isSpace(markdown[start])) start++;
+  while (end > start && isSpace(markdown[end - 1])) end--;
+  return [start, end];
+}
+
+function hasEnoughText(text) {
+  let count = 0;
+  for (const ch of text) {
+    if (!isSpace(ch) && ++count >= MIN_UNIT_CHARS) return true;
+  }
+  return false;
+}
+
+/**
+ * Cut the markdown a scrape returned into the units a query can be scored
+ * against. Headings are not units — they label the units that follow.
+ *
+ * @param {string} markdown
+ * @returns {Array<{ text: string, kind: 'sentence' | 'table_row' | 'code_block', offset: number, length: number, heading: string | null }>}
+ */
+export function segmentUnits(markdown) {
+  if (typeof markdown !== 'string' || markdown.length === 0) return [];
+
+  const units = [];
+  let heading = null;
+
+  function push(kind, start, end) {
+    [start, end] = trimRange(markdown, start, end);
+    const text = markdown.slice(start, end);
+    if (!hasEnoughText(text)) return;
+    units.push({ text, kind, offset: start, length: end - start, heading });
+  }
+
+  // A paragraph is a contiguous range: consecutive prose lines, the first
+  // one's list or quote marker already skipped. Line breaks inside it are
+  // whitespace, so a sentence may carry a "\n" verbatim.
+  let paragraphStart = -1;
+  let paragraphEnd = -1;
+  function flushParagraph() {
+    if (paragraphStart >= 0) splitSentences(markdown, paragraphStart, paragraphEnd, push);
+    paragraphStart = -1;
+  }
+
+  let fence = null; // { marker, contentStart }
+
+  const length = markdown.length;
+  let lineStart = 0;
+  while (lineStart <= length) {
+    let lineEnd = markdown.indexOf('\n', lineStart);
+    if (lineEnd === -1) lineEnd = length;
+    const [s, e] = trimRange(markdown, lineStart, lineEnd);
+    const line = markdown.slice(s, e);
+    let match;
+
+    if (fence) {
+      if (line[0] === fence.marker[0] && line.length >= fence.marker.length && FENCE_CLOSER.test(line)) {
+        push('code_block', fence.contentStart, Math.max(fence.contentStart, lineStart));
+        fence = null;
+      }
+    } else if (line === '') {
+      flushParagraph();
+    } else if ((match = FENCE.exec(line))) {
+      flushParagraph();
+      fence = { marker: match[1], contentStart: Math.min(lineEnd + 1, length) };
+    } else if ((match = HEADING.exec(line))) {
+      flushParagraph();
+      heading = (match[1] || '').trim() || null;
+    } else if (line[0] === '|') {
+      flushParagraph();
+      if (!TABLE_DELIMITER.test(line)) push('table_row', s, e);
+    } else if (HORIZONTAL_RULE.test(line) || HTML_COMMENT_LINE.test(line) || IMAGE_ONLY_LINE.test(line)) {
+      flushParagraph();
+    } else {
+      let start = s;
+      let marked = false;
+      while ((match = BLOCKQUOTE_MARKER.exec(markdown.slice(start, e)))) {
+        start += match[0].length;
+        marked = true;
+      }
+      if ((match = LIST_MARKER.exec(markdown.slice(start, e)))) {
+        start += match[0].length;
+        marked = true;
+      }
+      // A marker starts its own paragraph: two list items are two units,
+      // and "- " never lands inside a sentence's text.
+      if (marked) flushParagraph();
+      if (paragraphStart < 0) paragraphStart = start;
+      paragraphEnd = e;
+    }
+
+    lineStart = lineEnd + 1;
+  }
+
+  // A fence nobody closed: the rest of the document is that block.
+  if (fence) push('code_block', fence.contentStart, length);
+  flushParagraph();
+
+  return units;
+}
+
+/**
+ * Split the paragraph at [start, end) into sentences, calling `emit` with
+ * each one's range. Whitespace between sentences belongs to neither.
+ */
+function splitSentences(markdown, start, end, emit) {
+  let sentenceStart = start;
+  let i = start;
+  while (i < end) {
+    const ch = markdown[i];
+    if (CJK_TERMINATORS.includes(ch)) {
+      // Unambiguous, and CJK text puts no whitespace after them.
+      emit('sentence', sentenceStart, i + 1);
+      sentenceStart = i + 1;
+      i++;
+      continue;
+    }
+    if (!ASCII_TERMINATORS.includes(ch)) {
+      i++;
+      continue;
+    }
+    // Consume the run: "?!", "...", then any closing quote or bracket.
+    let j = i;
+    while (j < end && ASCII_TERMINATORS.includes(markdown[j])) j++;
+    while (j < end && CLOSERS.includes(markdown[j])) j++;
+    if (j < end && !isSpace(markdown[j])) {
+      // "Node.js", "3.14", "e.g." — a period glued to the next word.
+      i = j;
+      continue;
+    }
+    if (ch === '.' && isFalseStop(markdown, sentenceStart, i)) {
+      i = j;
+      continue;
+    }
+    emit('sentence', sentenceStart, j);
+    sentenceStart = j;
+    i = j;
+  }
+  if (sentenceStart < end) emit('sentence', sentenceStart, end);
+}
+
+/**
+ * Whether the period at `dot` ends an abbreviation, a word with internal
+ * periods, a decimal or a single-letter initial — the server's four checks,
+ * applied to the whitespace-delimited word before the period.
+ */
+function isFalseStop(markdown, from, dot) {
+  let wordStart = dot;
+  while (wordStart > from && !isSpace(markdown[wordStart - 1])) wordStart--;
+  const word = markdown.slice(wordStart, dot);
+  if (word === '') return false;
+  if (ABBREVIATIONS.has(word.toLowerCase().replace(/[^a-z]/g, ''))) return true;
+  if (/\w\.\w/.test(word)) return true;
+  if (/\d\.\d/.test(word)) return true;
+  return /^[A-Z]$/.test(word);
+}
+
+// Letters and digits of the scripts that put no spaces between words (Han,
+// Hiragana, Katakana, Hangul) form one kind of run; every other letter,
+// digit or combining mark forms the other — marks, because a Devanagari
+// vowel sign is a mark, and without them "कीमत" is two fragments.
+// Script_Extensions keeps "ー" and "々" with their runs; the letter/number
+// class keeps "、" and "。" out of them.
+const CJK_CHAR = '[\\p{scx=Han}\\p{scx=Hiragana}\\p{scx=Katakana}\\p{scx=Hangul}]';
+const TOKEN = new RegExp(`(?:(?=${CJK_CHAR})[\\p{L}\\p{N}])+|(?:(?!${CJK_CHAR})[\\p{L}\\p{N}\\p{M}])+`, 'gu');
+const CJK_START = new RegExp(`^${CJK_CHAR}`, 'u');
+const SUFFIXES = ['ing', 'ed', 'es', 's'];
+
+/**
+ * Lowercase word tokens. A CJK run becomes character bigrams so a query in
+ * one of those scripts can match without a segmenter; a light suffix strip
+ * puts "pricing", "prices" and "price" on one stem ("pric").
+ * @param {string} text
+ * @returns {string[]}
+ */
+function tokenize(text) {
+  const tokens = [];
+  for (const [run] of text.toLowerCase().matchAll(TOKEN)) {
+    if (CJK_START.test(run)) {
+      const chars = Array.from(run);
+      if (chars.length === 1) tokens.push(run);
+      for (let i = 0; i + 1 < chars.length; i++) tokens.push(chars[i] + chars[i + 1]);
+    } else {
+      tokens.push(stem(run));
+    }
+  }
+  return tokens;
+}
+
+function stem(token) {
+  if (token.length <= 4) return token;
+  for (const suffix of SUFFIXES) {
+    if (token.endsWith(suffix)) {
+      token = token.slice(0, -suffix.length);
+      break;
+    }
+  }
+  return token.length > 4 && token.endsWith('e') ? token.slice(0, -1) : token;
+}
+
+const K1 = 1.2;
+const B = 0.75;
+const PHRASE_BOOST = 1.5;
+// A unit inherits the terms of the heading it sits under, at half the weight
+// of a term in its own text: on a card-style pricing page the plan name is
+// the heading and "$83/month" is the unit, and without the heading the price
+// line shares nothing with "professional plan price per month". Heading
+// terms count toward a unit's term frequency only — not toward document
+// frequency, or the plan name would look common and lose its weight.
+const HEADING_WEIGHT = 0.5;
+// BM25's length normalisation rewards short documents, and a page's shortest
+// units are its buttons: "Choose Professional" outranked every price line on
+// a live pricing page. A unit is scored as if it had at least this many
+// tokens, so a two-word call to action carries no length advantage.
+const MIN_DOC_LENGTH = 4;
+
+/**
+ * The units that answer a query, best first: BM25 over the units as the
+ * corpus, with each unit inheriting its heading's terms at half weight, and a
+ * phrase boost when a unit contains the whole query. The input units are not
+ * touched; every returned unit is a new object.
+ *
+ * @template {{ text: string, heading?: string | null, offset: number }} U
+ * @param {U[]} units
+ * @param {string} query
+ * @param {{ maxUnits?: number, minScore?: number }} [options]
+ * @returns {Array<U & { score: number }>}
+ */
+export function rankUnits(units, query, { maxUnits = 10, minScore = 0 } = {}) {
+  if (!Array.isArray(units) || units.length === 0 || typeof query !== 'string') return [];
+  const phrase = query.trim().toLowerCase();
+  if (phrase === '') return [];
+  const terms = [...new Set(tokenize(phrase))];
+  if (terms.length === 0) return [];
+
+  const limit = Number.isFinite(maxUnits) ? Math.max(1, Math.floor(maxUnits)) : 10;
+  const floor = Number.isFinite(minScore) ? minScore : 0;
+
+  const docs = units.map((unit) => {
+    const counts = new Map();
+    const tokens = tokenize(String(unit.text ?? ''));
+    for (const token of tokens) counts.set(token, (counts.get(token) || 0) + 1);
+    const own = new Set(counts.keys());
+    let length = tokens.length;
+    if (unit.heading) {
+      for (const token of new Set(tokenize(String(unit.heading)))) {
+        if (own.has(token)) continue;
+        counts.set(token, HEADING_WEIGHT);
+        length += HEADING_WEIGHT;
+      }
+    }
+    return { counts, own, length: Math.max(length, MIN_DOC_LENGTH) };
+  });
+  const n = docs.length;
+  const avgdl = docs.reduce((sum, doc) => sum + doc.length, 0) / n || 1;
+  const idf = new Map(terms.map((term) => {
+    const df = docs.reduce((count, doc) => count + (doc.own.has(term) ? 1 : 0), 0);
+    return [term, Math.log(1 + (n - df + 0.5) / (df + 0.5))];
+  }));
+
+  const ranked = [];
+  units.forEach((unit, index) => {
+    const doc = docs[index];
+    let score = 0;
+    for (const term of terms) {
+      const tf = doc.counts.get(term);
+      if (!tf) continue;
+      score += idf.get(term) * (tf * (K1 + 1)) / (tf + K1 * (1 - B + B * doc.length / avgdl));
+    }
+    if (score === 0) return;
+    if (String(unit.text).toLowerCase().includes(phrase)) score *= PHRASE_BOOST;
+    // Round before the threshold, so a score handed back as minScore means
+    // what the caller saw.
+    score = Math.round(score * 1000) / 1000;
+    if (score > floor) ranked.push({ ...unit, score });
+  });
+
+  ranked.sort((a, b) => b.score - a.score || a.offset - b.offset);
+  return ranked.slice(0, limit);
+}