@startsimpli/integrations 0.2.18 → 0.2.20

package/src/corpus/fixtures.ts

@@ -43,7 +43,7 @@ import {
   toSourceDocument,
 } from './entities';
 import type { TenantEntity, TenantRelationship } from './entities';
-import { splitSections } from './text';
+import { splitSections, stem } from './text';
 import type {
   CorpusApi,
   CorpusPage,
@@ -571,11 +571,125 @@ export const FIXTURE_SOURCES: SourceFreshness[] = [
   },
 ];
 
+// ---------------------------------------------------------------------------
+//  the tenant's OWN search semantics
+// ---------------------------------------------------------------------------
+
+/**
+ * `websearch_to_tsquery('english', term) @@ to_tsvector('english', text)`, close
+ * enough to fail the same way (bd startsim-768w.38.36).
+ *
+ * WHY A FIXTURE HAS TO MODEL THIS AT ALL. Both fixtures below used to be MORE
+ * PERMISSIVE than the tenant — one OR'd whitespace-split substrings, the other
+ * ignored `?search=` entirely — so a client that sent a whole English question
+ * as the search term passed every test in this package and retrieved NOTHING on
+ * the live tenant, where the same term is a conjunction of stemmed words. A
+ * fixture server that is easier to satisfy than the real one does not fail
+ * safe; it fails silently, and then it ships.
+ *
+ * The two properties it exists to reproduce, because they are the two that
+ * broke:
+ *
+ *   AND, not OR. `websearch_to_tsquery` joins bare words with `&`, so a
+ *   document must contain EVERY content word of the query. Adding a word to a
+ *   question narrows; it never widens.
+ *
+ *   STEMMING AND STOPWORDS. The `english` configuration folds `blocked` and
+ *   `block` to one lexeme and deletes `what`/`is`/`the` outright — so a query
+ *   made only of stopwords compiles to an EMPTY tsquery, which matches no row
+ *   at all rather than every row.
+ *
+ * It also honours the syntax `websearch_to_tsquery` gives a person: `"quoted
+ * phrases"`, a bare `or` for disjunction, and a leading `-` to exclude.
+ *
+ * THE STEMMER IS THE PACKAGE'S OWN (`./text`), the same Porter implementation
+ * the ranker and the extraction dedupe use. Postgres's `english` is snowball,
+ * which is Porter2; using a third approximation here would let the fixture
+ * disagree with production for reasons that have nothing to do with the code
+ * under test — which is exactly the class of bug this file now exists to catch.
+ */
+
+/** The `english` configuration's stoplist, near enough. */
+const TSQUERY_STOPWORDS = new Set(
+  ('a about all am an and any are as at be been but by can did do does for from had has have he her his how i if in is it its me my not of on or our she so that the their them then there these they this to us was we were what when where which who why will with you your')
+    .split(' ')
+    .map((word) => stem(word)),
+);
+
+interface TsQueryItem {
+  /** One or more folded lexemes; more than one means a quoted phrase. */
+  lexemes: string[];
+  negated: boolean;
+}
+
+/** Parse the term the way `websearch_to_tsquery` does: AND groups, `or` between. */
+function parseTsQuery(term: string): TsQueryItem[][] {
+  const groups: TsQueryItem[][] = [[]];
+  const tokens = term.match(/"[^"]*"|\S+/g) ?? [];
+  for (const raw of tokens) {
+    if (/^-?or$/i.test(raw)) {
+      groups.push([]);
+      continue;
+    }
+    const negated = raw.startsWith('-');
+    const value = (negated ? raw.slice(1) : raw).replace(/^"|"$/g, '');
+    const lexemes = (value.match(/[a-z0-9]+/gi) ?? [])
+      .map((word) => stem(word))
+      .filter((lexeme) => lexeme && !TSQUERY_STOPWORDS.has(lexeme));
+    if (lexemes.length === 0) continue; // a stopword compiles to nothing at all
+    groups.at(-1)!.push({ lexemes, negated });
+  }
+  return groups.filter((group) => group.length > 0);
+}
+
+/** Does `text` satisfy the tsquery `term` compiles to? */
+export function matchesTenantSearch(term: string, text: string): boolean {
+  const groups = parseTsQuery(term);
+  // An EMPTY tsquery matches no row in Postgres. It does not match every row,
+  // which is the assumption a client makes right up until it ships.
+  if (groups.length === 0) return false;
+  const lexemes = (text.match(/[a-z0-9]+/gi) ?? []).map((word) => stem(word));
+  const present = new Set(lexemes);
+  const hasPhrase = (phrase: string[]) =>
+    phrase.length === 1
+      ? present.has(phrase[0])
+      : lexemes.some((_, index) => phrase.every((lexeme, offset) => lexemes[index + offset] === lexeme));
+  return groups.some((group) =>
+    group.every((item) => (item.negated ? !hasPhrase(item.lexemes) : hasPhrase(item.lexemes))),
+  );
+}
+
+/**
+ * Which search a fixture "server" performs.
+ *
+ * `postgres_fulltext` is what the live tenant runs and is what a retrieval test
+ * has to be written against. The permissive default is kept only so suites that
+ * predate the tenant learning to search keep testing what they were written to
+ * test; new retrieval work should ask for the real one.
+ */
+export type FixtureSearchBackend = 'permissive' | 'postgres_fulltext';
+
+/**
+ * Every string VALUE in a `data` blob, the way `jsonb_to_tsvector(cfg, data,
+ * '["string"]')` indexes it — values, never keys, so `?search=title` does not
+ * match every record in the tenant.
+ */
+function jsonStrings(value: unknown): string {
+  if (typeof value === 'string') return value;
+  if (Array.isArray(value)) return value.map(jsonStrings).join(' ');
+  if (value && typeof value === 'object') {
+    return Object.values(value as Record<string, unknown>).map(jsonStrings).join(' ');
+  }
+  return '';
+}
+
 // ---------------------------------------------------------------------------
 //  a CorpusApi over the fixtures
 // ---------------------------------------------------------------------------
 
 export interface FixtureCorpusOptions {
+  /** Which `?search=` semantics this "server" applies. See `FixtureSearchBackend`. */
+  searchBackend?: FixtureSearchBackend;
   documents?: SourceDocument[];
   moneyEvents?: MoneyEvent[];
   parties?: Party[];
@@ -632,9 +746,19 @@ export function createFixtureCorpusApi(options: FixtureCorpusOptions = {}): Corp
     defaultPageSize = 5,
     scope: serverScope,
     leakScope = false,
+    searchBackend = 'permissive',
     onQuery,
   } = options;
 
+  const matchesSearch = (term: string, text: string) =>
+    searchBackend === 'postgres_fulltext'
+      ? matchesTenantSearch(term, text)
+      : term
+          .toLowerCase()
+          .split(/\s+/)
+          .filter(Boolean)
+          .some((needle) => text.toLowerCase().includes(needle));
+
   const inScope = <T extends { scope: CorpusScope }>(rows: T[]): T[] => {
     if (leakScope || !serverScope) return rows;
     return rows.filter(
@@ -670,11 +794,7 @@ export function createFixtureCorpusApi(options: FixtureCorpusOptions = {}): Corp
       }
       if (query.search) {
         applied.push('search');
-        const needles = query.search.toLowerCase().split(/\s+/).filter(Boolean);
-        rows = rows.filter((document) => {
-          const text = haystack(document);
-          return needles.some((needle) => text.includes(needle));
-        });
+        rows = rows.filter((document) => matchesSearch(query.search!, haystack(document)));
       }
 
       rows = [...rows].sort((a, b) => (b.occurredAt ?? '').localeCompare(a.occurredAt ?? ''));
@@ -715,11 +835,12 @@ export function createFixtureCorpusApi(options: FixtureCorpusOptions = {}): Corp
       }
       if (query.search) {
         applied.push('search');
-        const needles = query.search.toLowerCase().split(/\s+/).filter(Boolean);
-        rows = rows.filter((event) => {
-          const text = `${event.subtype} ${event.status} ${event.description ?? ''} ${event.party?.name ?? ''}`.toLowerCase();
-          return needles.some((needle) => text.includes(needle));
-        });
+        rows = rows.filter((event) =>
+          matchesSearch(
+            query.search!,
+            `${event.subtype} ${event.status} ${event.description ?? ''} ${event.party?.name ?? ''}`,
+          ),
+        );
       }
       return paginate(rows, query.page ?? 1, query.pageSize ?? 20, applied);
     },
@@ -743,8 +864,12 @@ export function createFixtureCorpusApi(options: FixtureCorpusOptions = {}): Corp
  * block was lost (bd startsim-768w.38.22). It reproduces the tenant's real
  * behaviour, including the parts that are inconvenient:
  *
- *   • `?type=` is the ONLY filter honoured — `search`, `source` and `party` are
- *     accepted and ignored, exactly as `EntityViewSet` does today;
+ *   • `?type=` is honoured, and `?search=` is honoured ONLY when the caller asks
+ *     for `searchBackend: 'postgres_fulltext'` — the tenant has learned to
+ *     search since this was written (bd startsim-768w.38.12) and a retrieval
+ *     test must be written against the real semantics, but the suites that
+ *     predate it are still testing what they were written to test. `source` and
+ *     `party` are still accepted and ignored, exactly as `EntityViewSet` does;
  *   • `page_size` is IGNORED (`PageNumberPagination` with no
  *     `page_size_query_param`), so a caller asking for 200 gets `pageSize`;
  *   • `/api/v1/relationships/` returns every edge in the org, unfiltered,
@@ -760,6 +885,8 @@ export function createFixtureCorpusHttp(options: {
   pageSize?: number;
   /** Endpoints to fail, so the fail-soft paths are exercised. */
   missing?: string[];
+  /** Which `?search=` semantics this "tenant" applies. See `FixtureSearchBackend`. */
+  searchBackend?: FixtureSearchBackend;
   onRequest?: (path: string, params: Record<string, unknown>) => void;
 } = {}) {
   const entities = options.entities ?? [
@@ -803,8 +930,19 @@ export function createFixtureCorpusHttp(options: {
           return found as T;
         }
         const type = params.type ? String(params.type) : '';
-        // `search`/`source`/`party` are deliberately NOT applied.
-        const rows = type ? entities.filter((entity) => entity.entity_type === type) : entities;
+        // `source`/`party` are deliberately NOT applied. `search` is, but only
+        // for a caller that asked for the tenant's real backend.
+        let rows = type ? entities.filter((entity) => entity.entity_type === type) : entities;
+        const term = params.search ? String(params.search) : '';
+        if (term && options.searchBackend === 'postgres_fulltext') {
+          // The vector is built from `name` and every STRING VALUE in `data`,
+          // exactly like the generated column — and from the data the server
+          // WOULD send, so a leaking fixture's search leaks too. The client is
+          // not allowed to be safe only because the server was.
+          rows = rows.filter((entity) =>
+            matchesTenantSearch(term, `${entity.name ?? ''} ${jsonStrings(entity.data)}`),
+          );
+        }
         return slice(rows, params) as T;
       }
       throw Object.assign(new Error(`${path} responded 404`), { status: 404 });

package/src/corpus/index.ts

@@ -73,12 +73,14 @@ export type { CorpusTypeKeys, TenantEntity, TenantRelationship } from './entitie
 export {
   enforceScope,
   isInScope,
+  questionTerms,
   rankDocuments,
   retrieveForQuestion,
+  searchTermFor,
   snippetFor,
   tokenize,
 } from './retrieval';
-export type { RetrieveOptions, ScopeFilterResult } from './retrieval';
+export type { QuestionTerms, RetrieveOptions, ScopeFilterResult } from './retrieval';
 
 export {
   answerSystemPrompt,

package/src/corpus/retrieval.ts

@@ -29,12 +29,21 @@
  * what was dropped is surfaced rather than swallowed, because a non-zero drop is
  * a bug somewhere upstream that someone needs to see.
  */
+import { stem } from './text';
 import type { CorpusScope, RetrievedDocument, SourceDocument } from './types';
 
 /**
  * Terms too common to carry meaning. Deliberately small: an aggressive stoplist
  * silently deletes the user's question. `contract`, `invoice` and every domain
  * word stay in, because they are exactly what distinguishes one record here.
+ *
+ * The second block is the QUESTION vocabulary — the interrogatives, auxiliaries
+ * and discourse verbs that appear because somebody asked rather than because
+ * they meant something ("what did we decide", "can you tell me", "should we").
+ * They matter for the same reason as the rest: a term in this list is a term the
+ * server is not asked to match, and every word sent is a word that has to be
+ * found. Domain nouns are never in here — `decision` stays, `decide` goes,
+ * because one is a thing a record is about and the other is how a person asks.
  */
 const STOPWORDS = new Set([
   'a', 'about', 'all', 'am', 'an', 'and', 'any', 'are', 'as', 'at', 'be', 'been',
@@ -43,6 +52,16 @@ const STOPWORDS = new Set([
   'on', 'or', 'our', 'she', 'so', 'that', 'the', 'their', 'them', 'then', 'there',
   'these', 'they', 'this', 'to', 'us', 'was', 'we', 'were', 'what', 'when',
   'where', 'which', 'who', 'why', 'will', 'with', 'you', 'your',
+  // asked-not-meant
+  'again', 'also', 'anything', 'been', 'being', 'could', 'currently', 'decide',
+  'decided', 'doing', 'done', 'ever', 'everything', 'get', 'give', 'going',
+  'gonna', 'got', 'hey', 'im', 'into', 'just', 'know', 'lately', 'let', 'like',
+  'll', 'look', 'many', 'may', 'might', 'much', 'must', 'need', 'now', 'ok',
+  'okay', 'only', 'out', 'over', 'own', 'please', 'pull', 'really', 'remind',
+  're', 'said', 'same', 'say', 'says', 'see', 'shall', 'should', 'show', 'some',
+  'something', 'still', 'such', 'sure', 'tell', 'thing', 'things', 'think',
+  'told', 'up', 've', 've', 'very', 'want', 'wanna', 'whats', 'whom', 'whose',
+  'would', 'yet',
 ]);
 
 /** Lowercase word tokens, 2+ chars, stopwords removed. */
@@ -52,6 +71,113 @@ export function tokenize(text: string): string[] {
     .filter((token) => token.length >= 2 && !STOPWORDS.has(token));
 }
 
+// ---------------------------------------------------------------------------
+//  a question -> a query
+// ---------------------------------------------------------------------------
+
+/**
+ * THE FIX FOR bd startsim-768w.38.36, and the argument for its shape.
+ *
+ * The tenant compiles `?search=` with `websearch_to_tsquery('english', …)`,
+ * which joins bare words with `&`. Handing it a whole English question therefore
+ * asks for a document containing EVERY content word of that question — and on a
+ * corpus of a few thousand records, essentially no such document exists. The
+ * live measurement: `?search=parking` returned 2 records, `?search=parking
+ * allocation` returned 0, and every natural-language question returned 0 while a
+ * single rare word still answered.
+ *
+ * WHAT WAS CHOSEN: ONE request, a DISJUNCTION of the question's content words,
+ * ordered by the server's own `ts_rank_cd`, narrowed by the ranker below.
+ *
+ * Why not tier it — conjunction first, disjunction only when that returns
+ * nothing? It is one more round trip on exactly the questions that are already
+ * failing, and it puts a discontinuity in the middle of the product: adding one
+ * word to a question would silently move you between two different rankings for
+ * reasons no user can see. The deeper reason is that the two stages have
+ * different jobs. RETRIEVAL'S JOB IS RECALL — get the plausible records into the
+ * process. RANKING'S JOB IS PRECISION — decide which of them the model sees. A
+ * candidate set that is too narrow can never be rescued by a better ranker; one
+ * that is too broad always can, and the ranker for it already exists.
+ *
+ * Why not drop `?search=` and rank a recent window instead? Because that is the
+ * failure mode in the other direction: it answers every question from whatever
+ * happened to be recent, and it stops being possible at all the moment the
+ * corpus outgrows one page. The disjunction keeps the server doing the work it
+ * is good at — a stemmed, indexed, ranked scan of the whole corpus.
+ *
+ * PRECISION IS NOT TRADED AWAY. A one-word question compiles to exactly that one
+ * word, unchanged, so the rare-term case behaves identically to before. The
+ * widening only ever applies where a conjunction was going to return nothing.
+ *
+ * The syntax a person may already be typing survives: a `"quoted phrase"` stays
+ * quoted (the server phrase-matches it) and a `-word` stays excluded.
+ */
+export interface QuestionTerms {
+  /** Content words to match on, lowercased and deduplicated. */
+  include: string[];
+  /** Words the asker explicitly excluded with a leading `-`. */
+  exclude: string[];
+  /** Quoted phrases, kept verbatim so the server can phrase-match them. */
+  phrases: string[];
+}
+
+/**
+ * How many terms are worth sending. A question longer than this is a paragraph,
+ * and every extra disjunct widens the candidate set without adding signal —
+ * the ranker's coverage factor is what separates them once they are here.
+ */
+const MAX_QUERY_TERMS = 16;
+
+export function questionTerms(question: string): QuestionTerms {
+  const include: string[] = [];
+  const exclude: string[] = [];
+  const phrases: string[] = [];
+  const seen = new Set<string>();
+
+  for (const raw of question.match(/-?"[^"]*"|\S+/g) ?? []) {
+    const negated = raw.startsWith('-');
+    const value = negated ? raw.slice(1) : raw;
+    const quoted = value.startsWith('"') && value.endsWith('"') && value.length > 2;
+    const words = tokenize(quoted ? value.slice(1, -1) : value);
+    if (words.length === 0) continue;
+    if (quoted && !negated && words.length > 1) phrases.push(words.join(' '));
+    for (const word of words) {
+      const bucket = negated ? exclude : include;
+      const key = `${negated ? '-' : ''}${word}`;
+      if (seen.has(key)) continue;
+      seen.add(key);
+      bucket.push(word);
+    }
+  }
+  return { include: include.slice(0, MAX_QUERY_TERMS), exclude, phrases };
+}
+
+/**
+ * The `?search=` value for a question — `''` when there is nothing worth asking.
+ *
+ * An EMPTY term is returned rather than a term made of stopwords, because
+ * `websearch_to_tsquery` compiles a stopword-only query to an EMPTY tsquery and
+ * an empty tsquery matches NO row. "What is going on?" would ask the server for
+ * nothing and be told, accurately, that there is nothing. Omitting the parameter
+ * instead pulls the recent window, which is what that question means.
+ *
+ * Exclusions are sent AND re-applied by the ranker. `websearch_to_tsquery` has
+ * no parentheses, so `a or b -c` compiles to `a | b & !c` — the exclusion binds
+ * to the last disjunct only. That is a superset of what was asked for, never a
+ * subset, so the server can only over-return; `rankDocuments` makes it exact.
+ */
+export function searchTermFor(question: string): string {
+  const { include, exclude, phrases } = questionTerms(question);
+  if (include.length === 0) return '';
+  const phraseSet = new Set(phrases.flatMap((phrase) => phrase.split(' ')));
+  const disjuncts = [
+    ...phrases.map((phrase) => `"${phrase}"`),
+    ...include.filter((term) => !phraseSet.has(term)),
+  ];
+  const positive = disjuncts.join(' or ');
+  return [positive, ...exclude.map((term) => `-${term}`)].filter(Boolean).join(' ');
+}
+
 /**
  * Is this record inside the asking principal's scope?
  *
@@ -97,6 +223,17 @@ export function enforceScope<T extends { scope: CorpusScope }>(
 /** Half-life of the recency prior, in days. */
 const RECENCY_HALF_LIFE_DAYS = 120;
 
+/** Every piece of a record the reader is allowed to see, as one string. */
+function documentHaystack(document: SourceDocument): string {
+  return [
+    document.title,
+    document.summary ?? '',
+    document.body,
+    ...document.sections.map((section) => section.text),
+    ...document.participants.map((party) => `${party.name} ${party.email ?? ''}`),
+  ].join(' ');
+}
+
 /**
  * Fields are weighted because a title hit means more than a transcript hit.
  *
@@ -116,14 +253,81 @@ function recencyWeight(occurredAt: string | null | undefined, now: number): numb
   return 2 ** (-ageDays / RECENCY_HALF_LIFE_DAYS);
 }
 
-function countHits(tokens: string[], terms: Set<string>): { hits: number; matched: string[] } {
+/**
+ * TWO RETRIEVAL LAYERS THAT DISAGREE ABOUT WHAT A MATCH IS RETRIEVE NOTHING.
+ *
+ * The tenant indexes with `to_tsvector('english', …)`, so `blocked` and `block`
+ * are one lexeme there. This ranker compared raw tokens, so a record the server
+ * had correctly matched scored zero here and was thrown away again on the way to
+ * the prompt — invisible from either side, because each layer was behaving
+ * perfectly on its own. Live: `?search=blocked` returned the July record (its
+ * summary says "could block the funding round") and Ask still answered "nothing
+ * in your corpus matches that", having considered exactly zero documents.
+ *
+ * SO BOTH SIDES ARE STEMMED, with the Porter implementation this package
+ * ALREADY has in `./text` — the one the extraction pipeline dedupes commitments
+ * with. Postgres's `english` configuration is snowball, which is Porter2, and
+ * on the vocabulary that matters the two agree: `blocked`/`block`,
+ * `agreed`/`agree`, `filing`/`file`, `lease`/`leases`, `payments`/`payment`,
+ * `renewal`/`renewals` all collapse; `relapsed`/`relationship` and
+ * `block`/`blockchain` do not. Writing a second, weaker approximation next to a
+ * working stemmer would have been a third spelling of "what is a match", which
+ * is the whole shape of this bug.
+ *
+ * THE AGREEMENT IS MADE OVER TEXT THE READER MAY SEE, never by trusting the
+ * server's verdict. Scoring a record because the SERVER matched it would undo
+ * the field read gate: on a tenant that is leaking, the term that matched could
+ * be one from a withheld attribute. A record whose only hit is inside withheld
+ * text still scores zero here, because that text never reaches this function.
+ */
+
+/**
+ * Stems, memoized for the pass.
+ *
+ * One `stem()` call per distinct WORD rather than per token: a candidate set is
+ * up to 200 records and a record may be a 40-minute transcript, so the token
+ * count runs to millions while the vocabulary stays in the thousands.
+ */
+const stemCache = new Map<string, string>();
+
+function cachedStem(word: string): string {
+  const hit = stemCache.get(word);
+  if (hit !== undefined) return hit;
+  const value = stem(word);
+  // Bounded, because this module is long-lived in a server process and a corpus
+  // is not: an unbounded cache here is a slow leak, not a fast lookup.
+  if (stemCache.size > 20_000) stemCache.clear();
+  stemCache.set(word, value);
+  return value;
+}
+
+/**
+ * The query terms, indexed by stem for one pass per document, and mapped back to
+ * the word the user actually typed so `matchedTerms` can say what matched in
+ * their words rather than in Porter's.
+ */
+interface TermIndex {
+  terms: string[];
+  byStem: Map<string, string>;
+}
+
+function indexTerms(terms: string[]): TermIndex {
+  const byStem = new Map<string, string>();
+  for (const term of terms) {
+    const key = cachedStem(term);
+    if (!byStem.has(key)) byStem.set(key, term);
+  }
+  return { terms, byStem };
+}
+
+function countHits(tokens: string[], index: TermIndex): { hits: number; matched: string[] } {
   let hits = 0;
   const matched = new Set<string>();
   for (const token of tokens) {
-    if (terms.has(token)) {
-      hits += 1;
-      matched.add(token);
-    }
+    const term = index.byStem.get(cachedStem(token));
+    if (term === undefined) continue;
+    hits += 1;
+    matched.add(term);
   }
   return { hits, matched: [...matched] };
 }
@@ -132,7 +336,11 @@ function countHits(tokens: string[], terms: Set<string>): { hits: number; matche
  * A passage around the first matching term, so the citation card shows the user
  * WHY this record came back rather than making them open it to find out.
  */
-export function snippetFor(document: SourceDocument, terms: Set<string>, length = 220): string {
+export function snippetFor(
+  document: SourceDocument,
+  terms: Set<string> | Iterable<string>,
+  length = 220,
+): string {
   // The summary is searched FIRST, so a citation card quotes the curated
   // sentence rather than the moment somebody said the word out loud.
   const haystacks = [
@@ -140,12 +348,17 @@ export function snippetFor(document: SourceDocument, terms: Set<string>, length
     document.body,
     ...document.sections.map((section) => section.text),
   ];
-  for (const text of haystacks) {
+  // A term the ranker matched by STEM ("blocked" against "block") does not
+  // appear verbatim, so the passage is found by walking words and comparing
+  // stems rather than by `indexOf`. Otherwise the citation card falls back to
+  // the document's opening line and the reader is not shown WHY it came back —
+  // which is the one job of a snippet.
+  const wanted = new Set([...terms].map((term) => cachedStem(term.toLowerCase())));
+  for (const text of wanted.size ? haystacks : []) {
     if (!text) continue;
-    const lower = text.toLowerCase();
-    for (const term of terms) {
-      const at = lower.indexOf(term);
-      if (at === -1) continue;
+    for (const match of text.toLowerCase().matchAll(/[a-z0-9][a-z0-9'&.-]*/g)) {
+      if (!wanted.has(cachedStem(match[0]))) continue;
+      const at = match.index ?? 0;
       const start = Math.max(0, at - Math.floor(length / 3));
       const slice = text.slice(start, start + length).trim();
       return `${start > 0 ? '…' : ''}${slice}${start + length < text.length ? '…' : ''}`;
@@ -181,7 +394,8 @@ export function rankDocuments(
   options: RetrieveOptions = {},
 ): RetrievedDocument[] {
   const { limit = 8, now = Date.now(), minScore = 0.0001 } = options;
-  const terms = new Set(tokenize(question));
+  const { include, exclude } = questionTerms(question);
+  const terms = new Set(include);
   if (terms.size === 0) {
     // No usable terms — fall back to pure recency rather than returning
     // nothing. "What happened lately" is a real question.
@@ -196,23 +410,29 @@ export function rankDocuments(
       }));
   }
 
+  const index = indexTerms(include);
+  // `websearch_to_tsquery` has no parentheses, so the server can only approximate
+  // an exclusion inside a disjunction. It is made exact here.
+  const excluded = exclude.length ? indexTerms(exclude) : null;
+
   const scored: RetrievedDocument[] = [];
   for (const document of documents) {
-    const title = countHits(tokenize(document.title), terms);
+    if (excluded && countHits(tokenize(documentHaystack(document)), excluded).hits > 0) continue;
+    const title = countHits(tokenize(document.title), index);
     const participants = countHits(
       tokenize(document.participants.map((party) => `${party.name} ${party.email ?? ''}`).join(' ')),
-      terms,
+      index,
     );
     // Scored SEPARATELY from `body`, and only when they are actually different
     // strings — `body` falls back to the summary on a corpus that has never
     // delivered a full text, and counting the same words twice there would
     // silently re-weight every document on the tenant.
     const summaryText = document.summary && document.summary !== document.body ? document.summary : '';
-    const summary = countHits(tokenize(summaryText), terms);
-    const body = countHits(tokenize(document.body), terms);
+    const summary = countHits(tokenize(summaryText), index);
+    const body = countHits(tokenize(document.body), index);
     const sections = countHits(
       tokenize(document.sections.map((section) => section.text).join(' ')),
-      terms,
+      index,
     );
 
     // Saturating term counts (log1p) so one long transcript repeating a word