@startsimpli/integrations 0.2.17 → 0.2.19

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@startsimpli/integrations",
-  "version": "0.2.17",
+  "version": "0.2.19",
   "description": "Provider-agnostic types, API clients, hooks, and UI primitives for email + calendar integrations across StartSimpli apps.",
   "main": "./src/index.ts",
   "types": "./src/index.ts",

package/src/__tests__/corpus-field-gate.test.ts

@@ -217,6 +217,35 @@ describe('askCorpus — the withheld text never reaches the model', () => {
     expect(result.answer).not.toContain('relapsed');
   });
 
+  it('does not retrieve on a withheld term carried in on a LOOSER query', async () => {
+    // The widening in bd startsim-768w.38.36 is the obvious way to reopen this.
+    // A question is no longer sent as a conjunction — it is sent as a
+    // DISJUNCTION of its content words — so "lease" is now enough to bring this
+    // record back on its own, and the withheld word rides along in the same
+    // query. If the ranker ever scored on what the SERVER matched rather than on
+    // what the reader may see, that record would arrive with the withheld term
+    // in its matched set and the model would be invited to answer about it.
+    //
+    // The fixture server here is the leaking one and searches the withheld
+    // `body` too, so the row genuinely comes back on this query. Everything
+    // after that has to hold on its own.
+    const { llm, seen } = spyLlm();
+    const result = await askCorpus(
+      { question: 'was anything said about the lease or relapsed treatment?', mode: 'ask', scope: SCOPE },
+      { corpus: apiFor([leakingEntity()]), llm },
+    );
+
+    // It DID retrieve — on `lease`, which is in the summary and always was.
+    expect(result.status).toBe('answered');
+    expect(seen[0]).toContain('lease');
+    // ...and the withheld word reached neither the prompt, the answer, nor the
+    // evidence the surface will show for it.
+    expect(seen[0]).not.toContain('relapsed');
+    expect(JSON.stringify(result.citations)).not.toContain('relapsed');
+    expect(JSON.stringify(result.considered)).not.toContain('relapsed');
+    expect(result.considered.flatMap((item) => item.matchedTerms)).not.toContain('relapsed');
+  });
+
   it('answers identically whether the server leaked or behaved', async () => {
     // The falsifiable version of "the client does not depend on the server
     // getting it right": same question, one API stripping the field and one

package/src/__tests__/corpus-question.test.ts

@@ -0,0 +1,318 @@
+/**
+ * A QUESTION IS NOT A BOOLEAN AND OF ITS WORDS (bd startsim-768w.38.36).
+ *
+ * `corpus-ask.test.ts` next door protects the honesty rules — that an untraceable
+ * answer can never be rendered as a traceable one. This file protects the stage
+ * BEFORE any of that is reachable: whether the records that would answer the
+ * question are still in the candidate set by the time the ranker runs.
+ *
+ * WHAT WENT WRONG, MEASURED ON THE LIVE TANDEM TENANT. `loadCandidates` sent the
+ * WHOLE question as `?search=`. The tenant compiles that with
+ * `websearch_to_tsquery('english', …)`, which joins bare words with `&` — so
+ * "What did we agree about the parking allocation?" only retrieved a document
+ * containing `agree` AND `parking` AND `allocation`, and 18 documents and 63,208
+ * characters of summary were discarded before ranking ever happened. A single
+ * rare word still answered, which is what made it look like a corpus problem;
+ * the surface said so out loud, blaming the connector for a query bug.
+ *
+ * SO THE FIXTURE HERE RUNS THE TENANT'S OWN SEARCH (`searchBackend:
+ * 'postgres_fulltext'`). Against the permissive fixture this package shipped
+ * with — whitespace-split substrings, OR'd — every case below passes on the
+ * broken client. A fixture easier to satisfy than production does not fail safe.
+ *
+ * THE TWO FAILURES ARE OPPOSITE, and both are here on purpose:
+ *
+ *   TOO NARROW is the bug: an ordinary question retrieves nothing.
+ *   TOO WIDE is the tempting fix: match everything, put the whole corpus in the
+ *   prompt and let the model sort it out. That trades a visible failure for an
+ *   invisible one — a confident answer assembled from whatever happened to be
+ *   recent. So a rare single term must still retrieve exactly its own record.
+ */
+import { describe, expect, it } from 'vitest';
+
+import { askCorpus } from '../corpus/ask';
+import { createCorpusApi } from '../corpus/client';
+import type { TenantEntity } from '../corpus/entities';
+import { createFixtureCorpusHttp, createFixtureLlm, FIXTURE_ORG } from '../corpus/fixtures';
+import { searchTermFor } from '../corpus/retrieval';
+import type { AskResult, CorpusHttp, CorpusScope } from '../corpus/types';
+
+const SCOPE: CorpusScope = { orgId: FIXTURE_ORG };
+
+/**
+ * A corpus shaped like the real one: meeting notes whose whole substance is a
+ * long `summary`, written the way a note-taker writes rather than the way a
+ * query is typed. No document contains every word of any question asked below,
+ * which is the entire point.
+ */
+function meeting(
+  id: string,
+  title: string,
+  occurredAt: string,
+  summary: string,
+): TenantEntity {
+  return {
+    id,
+    entity_type: 'source_document',
+    external_id: `granola:${id}`,
+    name: title,
+    org_id: FIXTURE_ORG,
+    created_at: `${occurredAt.slice(0, 10)}T12:00:00Z`,
+    data: { source: 'granola', subtype: 'meeting', title, summary, occurred_at: occurredAt },
+  };
+}
+
+const CORPUS: TenantEntity[] = [
+  meeting(
+    'doc_office',
+    'Office move — landlord terms and fit-out',
+    '2026-07-28T09:00:00Z',
+    [
+      'Landlord came back on the renewal. We agreed to a twelve month extension on the',
+      'lease at the current rate, with a three month break clause either side.',
+      'Fit-out costs are split, capped at 20k. Signature target is the 15th.',
+    ].join(' '),
+  ),
+  meeting(
+    'doc_site',
+    'Site walkthrough — Lamar building',
+    '2026-07-21T09:00:00Z',
+    [
+      'Walked the Lamar building with the broker. Six parking bays come with the unit;',
+      'two are currently used by the ground floor tenant and nobody has written down',
+      'who gets what. Broker to send the site plan so we can settle it before signing.',
+    ].join(' '),
+  ),
+  meeting(
+    'doc_july',
+    'July financials, tenders strategy, staffing update',
+    '2026-07-14T09:00:00Z',
+    [
+      'Collections are ahead of plan. On the patent dispute, a filing action tying up',
+      'the IP could block the funding round, so we hold the interest notice until Jeff',
+      'has reviewed it. Staffing: two offers out, one accepted.',
+    ].join(' '),
+  ),
+  meeting(
+    'doc_insurance',
+    'Property finances — loan renewals and insurance',
+    '2026-06-30T09:00:00Z',
+    [
+      'The insurance renewal is with the broker and we are waiting on their revised',
+      'quote before we can sign anything. Loan renewals for two of the properties are',
+      'due in September; the bank wants updated valuations first.',
+    ].join(' '),
+  ),
+  meeting(
+    'doc_rent',
+    'Rent Manager Express setup and Zillow posting',
+    '2026-06-16T09:00:00Z',
+    [
+      'Rent Manager Express is configured for the two smaller properties. Zillow',
+      'postings go up Monday. The QUIBBLETRON migration script is still the only',
+      'blocker on moving the rest of the portfolio across.',
+    ].join(' '),
+  ),
+  meeting(
+    'doc_mcr',
+    'MCR platform demo — streaming data and task automation',
+    '2026-05-19T09:00:00Z',
+    [
+      'Demoed the streaming ingest. Fuga integration is behind a vendor key we do not',
+      'have yet. Task automation from Granola notes is the piece they cared about.',
+    ].join(' '),
+  ),
+];
+
+function corpusFor(entities: TenantEntity[] = CORPUS) {
+  const { http, requests } = createFixtureCorpusHttp({
+    entities,
+    relationships: [],
+    searchBackend: 'postgres_fulltext',
+  });
+  return {
+    requests,
+    api: createCorpusApi({ http: http as CorpusHttp, scope: SCOPE, supportedFilters: [] }),
+  };
+}
+
+function askWith(question: string, entities?: TenantEntity[]): Promise<AskResult> {
+  return askCorpus(
+    { question, mode: 'ask', scope: SCOPE },
+    { corpus: corpusFor(entities).api, llm: createFixtureLlm() },
+  );
+}
+
+const citedIds = (result: AskResult) => result.citations.map((citation) => citation.documentId);
+
+// ---------------------------------------------------------------------------
+
+describe('an ordinary question retrieves', () => {
+  /**
+   * The three the user reproduced on prod, plus the phrasings people actually
+   * type. Every one of these returned `no_evidence` with zero citations against
+   * a corpus that answers them.
+   */
+  const cases: Array<[question: string, expected: string]> = [
+    ['What did we agree about the lease?', 'doc_office'],
+    ['what is blocked?', 'doc_july'],
+    ['What did we agree about the parking allocation?', 'doc_site'],
+    ['who are we waiting on for the insurance renewal?', 'doc_insurance'],
+    ['what did we decide about staffing?', 'doc_july'],
+    ['when are the loan renewals due?', 'doc_insurance'],
+    ['did we sort out the parking bays', 'doc_site'],
+    ['whats the story with the landlord', 'doc_office'],
+  ];
+
+  for (const [question, expected] of cases) {
+    it(`answers "${question}" and cites ${expected}`, async () => {
+      const result = await askWith(question);
+      expect(result.status).toBe('answered');
+      expect(result.citations.length).toBeGreaterThan(0);
+      expect(citedIds(result)).toContain(expected);
+    });
+  }
+
+  it('retrieves a question whose words are stemmed apart from the record’s', async () => {
+    // The tenant's `english` configuration folds `blocked` and `block` to one
+    // lexeme, so the SERVER matches. The client ranker tokenizes exactly, so it
+    // used to throw the row away again on the way to the prompt — two retrieval
+    // layers disagreeing about what a match is, which is invisible from either
+    // side alone.
+    const result = await askWith('what is blocked?');
+    expect(citedIds(result)).toContain('doc_july');
+  });
+
+  it('does not need every word of the question to appear in one record', async () => {
+    // No document contains `allocation` at all. Under a conjunction that is the
+    // end of it; under a question, `parking` is plenty.
+    const result = await askWith('What did we agree about the parking allocation?');
+    expect(result.status).toBe('answered');
+  });
+});
+
+describe('but a question is not an excuse to retrieve everything', () => {
+  it('retrieves ONLY the record holding a rare single term', async () => {
+    const result = await askWith('QUIBBLETRON');
+    expect(result.status).toBe('answered');
+    expect(citedIds(result)).toEqual(['doc_rent']);
+  });
+
+  it('keeps a rare term rare even when it is asked as a sentence', async () => {
+    const result = await askWith('what is the QUIBBLETRON migration waiting on?');
+    expect(citedIds(result)).toContain('doc_rent');
+    // `migration` and `waiting` are real words in this corpus, so a couple of
+    // other records may legitimately come along — but not the whole shelf.
+    expect(result.citations.length).toBeLessThan(CORPUS.length);
+  });
+
+  it('does not put the entire corpus in front of the model for a narrow question', async () => {
+    const seen: number[] = [];
+    await askCorpus(
+      { question: 'what did we agree about the lease?', mode: 'ask', scope: SCOPE },
+      {
+        corpus: corpusFor().api,
+        llm: {
+          describe: () => ({ provider: 'spy', model: 'spy', live: false }),
+          async answer(input) {
+            seen.push(input.documents.length);
+            return { answer: 'ok', citations: [], unanswerable: false };
+          },
+        },
+      },
+    );
+    expect(seen[0]).toBeGreaterThan(0);
+    expect(seen[0]).toBeLessThan(CORPUS.length);
+  });
+
+  it('still refuses when the corpus genuinely has nothing, without calling the model', async () => {
+    const calls: unknown[] = [];
+    const result = await askCorpus(
+      { question: 'what did the arbitration panel rule on the trademark?', mode: 'ask', scope: SCOPE },
+      {
+        corpus: corpusFor().api,
+        llm: {
+          describe: () => ({ provider: 'spy', model: 'spy', live: false }),
+          async answer(input) {
+            calls.push(input);
+            return { answer: 'I could improvise.', citations: [], unanswerable: false };
+          },
+        },
+      },
+    );
+    expect(result.status).toBe('no_evidence');
+    expect(calls).toHaveLength(0);
+  });
+});
+
+describe('the search term the tenant is actually sent', () => {
+  it('is a disjunction of the question’s content words, not the question', async () => {
+    const { api, requests } = corpusFor();
+    await askCorpus(
+      { question: 'What did we agree about the parking allocation?', mode: 'ask', scope: SCOPE },
+      { corpus: api, llm: createFixtureLlm() },
+    );
+    const term = String(
+      requests.find((request) => request.params.type === 'source_document')?.params.search ?? '',
+    );
+    expect(term).not.toContain('?');
+    expect(term.toLowerCase()).toContain(' or ');
+    expect(term.toLowerCase()).toContain('parking');
+    // Interrogatives and function words carry no signal and, ANDed, delete the
+    // question outright.
+    expect(term.toLowerCase().split(/\s+or\s+/)).not.toContain('what');
+  });
+
+  it('leaves a single rare term exactly as it was typed', () => {
+    // The precision case has to be untouched: one word in, one word out, no
+    // disjunction to widen it.
+    expect(searchTermFor('QUIBBLETRON')).toBe('quibbletron');
+  });
+
+  it('sends NO search term at all when the question is only function words', () => {
+    // `websearch_to_tsquery` compiles a stopword-only query to an EMPTY tsquery,
+    // which matches no row. Sending it would be asking the server for nothing.
+    expect(searchTermFor('what is it?')).toBe('');
+  });
+
+  it('keeps a quoted phrase quoted, and an excluded word excluded', () => {
+    expect(searchTermFor('"funding round"')).toBe('"funding round"');
+    expect(searchTermFor('parking -zillow')).toContain('-zillow');
+  });
+
+  it('asks the server to order by relevance, so page one is the relevant page', async () => {
+    const { api, requests } = corpusFor();
+    await askCorpus(
+      { question: 'what did we agree about the lease?', mode: 'ask', scope: SCOPE },
+      { corpus: api, llm: createFixtureLlm() },
+    );
+    const request = requests.find((entry) => entry.params.type === 'source_document');
+    expect(request?.params.order).toBe('relevance');
+  });
+});
+
+describe('a question with no content words falls back to the recent window', () => {
+  it('answers "what happened lately?" from the newest records', async () => {
+    const result = await askWith('what is going on?');
+    expect(result.status).toBe('answered');
+    expect(citedIds(result)).toContain('doc_office'); // the newest record
+  });
+});
+
+describe('the copy tells the truth about whose fault it is', () => {
+  it('does not blame the corpus when the corpus is full and the question missed', async () => {
+    const result = await askWith('what did the arbitration panel rule on the trademark?');
+    expect(result.status).toBe('no_evidence');
+    // The shipped sentence — "Either it has not been pulled in yet, or it
+    // belongs to a different workspace" — sent someone to check the connector
+    // for a query bug. Six records were sitting right there.
+    expect(result.answer).not.toMatch(/has not been pulled in/i);
+    expect(result.answer).toMatch(/6 record/i);
+  });
+
+  it('DOES say the workspace is empty when the workspace is empty', async () => {
+    const result = await askWith('what did we agree about the lease?', []);
+    expect(result.status).toBe('no_evidence');
+    expect(result.answer).toMatch(/nothing has been pulled in|no records/i);
+  });
+});

package/src/__tests__/corpus-scope.test.ts

@@ -80,10 +80,19 @@ describe('enforceScope — nothing is dropped silently', () => {
 });
 
 describe('retrieval refuses the bait', () => {
-  it('ranks the foreign document FIRST when the guard is removed', () => {
-    // Establishes that the guard is doing real work: without it, the leak wins.
+  it('ranks the foreign document at the very top when the guard is removed', () => {
+    // Establishes that the guard is doing real work: without it, the leak is not
+    // some tail result the model would ignore — it is the most attractive answer
+    // in the set and it reaches the prompt.
+    //
+    // ASSERTED AS "in the top two", not "first", since bd startsim-768w.38.36:
+    // the ranker now agrees with the tenant's stemmer, so the caller's OWN Acme
+    // note matches `agree` against its "agreed" and covers all six terms of the
+    // question rather than five. It edges ahead by 3%. The bait is no less
+    // attractive and the guard is no less necessary; pinning an exact ordinal
+    // pinned an artifact of the tokenizer instead of the property.
     const unguarded = rankDocuments(FIXTURE_DOCUMENTS, BAIT, { limit: 5 });
-    expect(unguarded[0]?.document.id).toBe('nw_contract_44');
+    expect(unguarded.slice(0, 2).map((item) => item.document.id)).toContain('nw_contract_44');
   });
 
   it('never returns it once the guard runs', () => {

package/src/corpus/ask.ts

@@ -19,7 +19,7 @@
  * given has invented a source, and the only safe rendering of that is to say so.
  */
 import { moneyEventAsDocument } from './entities';
-import { retrieveForQuestion } from './retrieval';
+import { retrieveForQuestion, searchTermFor } from './retrieval';
 import { documentText } from './text';
 import type {
   AnswerRequest,
@@ -209,6 +209,20 @@ function snippetOf(document: SourceDocument): string {
  * Pull the candidate set. Two reads, in PARALLEL, not one read per document —
  * the server pages and filters, this only decides how wide a net to cast.
  *
+ * THE QUESTION IS NOT THE QUERY (bd startsim-768w.38.36). It used to be: the
+ * whole question went out as `?search=`, the tenant compiled it with
+ * `websearch_to_tsquery`, and the resulting conjunction of every content word
+ * matched no record at all. `searchTermFor` turns it into a disjunction of the
+ * content words instead — see the argument for that shape in `./retrieval`.
+ *
+ * `order: 'relevance'` is what makes the widening safe at scale. Once the query
+ * is a disjunction, more records match than fit on a page, and an unordered page
+ * one is a random sample of them. Asking the server to sort by its own
+ * `ts_rank_cd` means the page this pulls is the page worth ranking.
+ *
+ * A question with no content words sends NO search term (see `searchTermFor`),
+ * which pulls the recent window — the honest reading of "what is going on?".
+ *
  * Money events are folded in as documents so one ranking pass spans meetings and
  * revenue; "what did we agree with this company" is a question about both.
  */
@@ -216,9 +230,10 @@ async function loadCandidates(
   corpus: CorpusApi,
   request: AskRequest,
   candidateLimit: number,
-): Promise<SourceDocument[]> {
+): Promise<{ candidates: SourceDocument[]; searchTerm: string; matchedCount: number }> {
+  const searchTerm = searchTermFor(request.question);
   const documentQuery = {
-    search: request.question,
+    ...(searchTerm ? { search: searchTerm, order: 'relevance' as const } : {}),
     ...request.filters,
     page: 1,
     pageSize: candidateLimit,
@@ -229,7 +244,7 @@ async function loadCandidates(
     corpus.listMoneyEvents
       ? corpus
           .listMoneyEvents({
-            search: request.question,
+            ...(searchTerm ? { search: searchTerm } : {}),
             partyId: request.filters?.partyId,
             page: 1,
             pageSize: Math.ceil(candidateLimit / 4),
@@ -240,7 +255,54 @@ async function loadCandidates(
       : Promise.resolve({ count: 0, next: null, previous: null, results: [] }),
   ]);
 
-  return [...documents.results, ...money.results.map(moneyEventAsDocument)];
+  return {
+    candidates: [...documents.results, ...money.results.map(moneyEventAsDocument)],
+    searchTerm,
+    matchedCount: documents.count ?? documents.results.length,
+  };
+}
+
+/**
+ * How many records this workspace holds AT ALL — asked only when the answer is
+ * going to be "nothing matched", and only to decide WHICH "nothing".
+ *
+ * "Your corpus has not been pulled in yet" and "none of your 18 records mention
+ * that" send a person to two different places, and the surface shipped the first
+ * sentence for both. Somebody went to check a connector that was working.
+ */
+async function corpusSize(corpus: CorpusApi, filters: AskRequest['filters']): Promise<number> {
+  return corpus
+    .listDocuments({ ...filters, search: undefined, page: 1, pageSize: 1 })
+    .then((page) => page.count ?? page.results.length)
+    .catch(() => -1);
+}
+
+/**
+ * What to say when retrieval came back empty, without blaming the wrong thing.
+ *
+ * THE OLD SENTENCE WAS A LIE OF ATTRIBUTION. "Nothing in your corpus matches
+ * that. Either it has not been pulled in yet, or it belongs to a different
+ * workspace" was returned for a query-construction bug, over a workspace holding
+ * eighteen documents that were pulled in perfectly. It reads as a data problem
+ * and sends someone to check a connector that is working.
+ *
+ * THE QUESTION IS NOT REPEATED BACK. An earlier draft named the terms it had
+ * searched for, which reads well and is wrong here: the corpus-field-gate suite
+ * asks a question made entirely of a withheld word and requires the answer not
+ * to contain it. Echoing the query would have made the ONE sentence this branch
+ * exists to produce the one place that word appears on screen — and a reader
+ * cannot tell an echo from a quote.
+ */
+function noEvidenceCopy(total: number): string {
+  if (total === 0) {
+    return 'Nothing has been pulled into this workspace yet, so there are no records to search. Connect a source, or check that the last sync succeeded.';
+  }
+  if (total < 0) {
+    // The count read failed. Say what IS known — the search found nothing — and
+    // claim nothing about the corpus.
+    return 'The search came back empty. Try naming the company, person or document you mean.';
+  }
+  return `Searched all ${total} record${total === 1 ? '' : 's'} in this workspace and none of them matched. Try naming the company, person or document you mean — or ask about something they actually discussed.`;
 }
 
 /**
@@ -264,7 +326,11 @@ export async function askCorpus(
     return emptyResult(request, llm, 'This session has no tenant scope, so there is nothing to search.');
   }
 
-  const candidates = await loadCandidates(corpus, request, candidateLimit);
+  const { candidates, searchTerm, matchedCount } = await loadCandidates(
+    corpus,
+    request,
+    candidateLimit,
+  );
   const { retrieved, droppedOutOfScope } = retrieveForQuestion(
     candidates,
     request.question,
@@ -275,10 +341,12 @@ export async function askCorpus(
   const freshness = await corpus.listSources().catch(() => [] as SourceFreshness[]);
 
   if (retrieved.length === 0) {
+    // An unfiltered pull already tells us the size, so the extra read only
+    // happens on the path that is about to make a claim about the corpus.
+    const total = searchTerm ? await corpusSize(corpus, request.filters) : matchedCount;
     return {
       status: 'no_evidence',
-      answer:
-        'Nothing in your corpus matches that. Either it has not been pulled in yet, or it belongs to a different workspace.',
+      answer: noEvidenceCopy(total),
       citations: [],
       considered: [],
       unresolvedCitations: [],

package/src/corpus/client.ts

@@ -228,6 +228,9 @@ function params(query: DocumentQuery, type: string) {
   const out: Record<string, unknown> = { type, page: query.page ?? 1 };
   if (query.pageSize) out.page_size = query.pageSize;
   if (query.search) out.search = query.search;
+  // Only alongside a term: `order=relevance` with nothing to be relevant TO is a
+  // parameter the server has to reject or ignore, and neither is worth asking.
+  if (query.order && (query.order !== 'relevance' || query.search)) out.order = query.order;
   if (query.source) out.source = query.source;
   if (query.partyId) out.party = query.partyId;
   if (query.occurredAfter) out.occurred_after = query.occurredAfter;

package/src/corpus/fixtures.ts

@@ -43,7 +43,7 @@ import {
   toSourceDocument,
 } from './entities';
 import type { TenantEntity, TenantRelationship } from './entities';
-import { splitSections } from './text';
+import { splitSections, stem } from './text';
 import type {
   CorpusApi,
   CorpusPage,
@@ -571,11 +571,125 @@ export const FIXTURE_SOURCES: SourceFreshness[] = [
   },
 ];
 
+// ---------------------------------------------------------------------------
+//  the tenant's OWN search semantics
+// ---------------------------------------------------------------------------
+
+/**
+ * `websearch_to_tsquery('english', term) @@ to_tsvector('english', text)`, close
+ * enough to fail the same way (bd startsim-768w.38.36).
+ *
+ * WHY A FIXTURE HAS TO MODEL THIS AT ALL. Both fixtures below used to be MORE
+ * PERMISSIVE than the tenant — one OR'd whitespace-split substrings, the other
+ * ignored `?search=` entirely — so a client that sent a whole English question
+ * as the search term passed every test in this package and retrieved NOTHING on
+ * the live tenant, where the same term is a conjunction of stemmed words. A
+ * fixture server that is easier to satisfy than the real one does not fail
+ * safe; it fails silently, and then it ships.
+ *
+ * The two properties it exists to reproduce, because they are the two that
+ * broke:
+ *
+ *   AND, not OR. `websearch_to_tsquery` joins bare words with `&`, so a
+ *   document must contain EVERY content word of the query. Adding a word to a
+ *   question narrows; it never widens.
+ *
+ *   STEMMING AND STOPWORDS. The `english` configuration folds `blocked` and
+ *   `block` to one lexeme and deletes `what`/`is`/`the` outright — so a query
+ *   made only of stopwords compiles to an EMPTY tsquery, which matches no row
+ *   at all rather than every row.
+ *
+ * It also honours the syntax `websearch_to_tsquery` gives a person: `"quoted
+ * phrases"`, a bare `or` for disjunction, and a leading `-` to exclude.
+ *
+ * THE STEMMER IS THE PACKAGE'S OWN (`./text`), the same Porter implementation
+ * the ranker and the extraction dedupe use. Postgres's `english` is snowball,
+ * which is Porter2; using a third approximation here would let the fixture
+ * disagree with production for reasons that have nothing to do with the code
+ * under test — which is exactly the class of bug this file now exists to catch.
+ */
+
+/** The `english` configuration's stoplist, near enough. */
+const TSQUERY_STOPWORDS = new Set(
+  ('a about all am an and any are as at be been but by can did do does for from had has have he her his how i if in is it its me my not of on or our she so that the their them then there these they this to us was we were what when where which who why will with you your')
+    .split(' ')
+    .map((word) => stem(word)),
+);
+
+interface TsQueryItem {
+  /** One or more folded lexemes; more than one means a quoted phrase. */
+  lexemes: string[];
+  negated: boolean;
+}
+
+/** Parse the term the way `websearch_to_tsquery` does: AND groups, `or` between. */
+function parseTsQuery(term: string): TsQueryItem[][] {
+  const groups: TsQueryItem[][] = [[]];
+  const tokens = term.match(/"[^"]*"|\S+/g) ?? [];
+  for (const raw of tokens) {
+    if (/^-?or$/i.test(raw)) {
+      groups.push([]);
+      continue;
+    }
+    const negated = raw.startsWith('-');
+    const value = (negated ? raw.slice(1) : raw).replace(/^"|"$/g, '');
+    const lexemes = (value.match(/[a-z0-9]+/gi) ?? [])
+      .map((word) => stem(word))
+      .filter((lexeme) => lexeme && !TSQUERY_STOPWORDS.has(lexeme));
+    if (lexemes.length === 0) continue; // a stopword compiles to nothing at all
+    groups.at(-1)!.push({ lexemes, negated });
+  }
+  return groups.filter((group) => group.length > 0);
+}
+
+/** Does `text` satisfy the tsquery `term` compiles to? */
+export function matchesTenantSearch(term: string, text: string): boolean {
+  const groups = parseTsQuery(term);
+  // An EMPTY tsquery matches no row in Postgres. It does not match every row,
+  // which is the assumption a client makes right up until it ships.
+  if (groups.length === 0) return false;
+  const lexemes = (text.match(/[a-z0-9]+/gi) ?? []).map((word) => stem(word));
+  const present = new Set(lexemes);
+  const hasPhrase = (phrase: string[]) =>
+    phrase.length === 1
+      ? present.has(phrase[0])
+      : lexemes.some((_, index) => phrase.every((lexeme, offset) => lexemes[index + offset] === lexeme));
+  return groups.some((group) =>
+    group.every((item) => (item.negated ? !hasPhrase(item.lexemes) : hasPhrase(item.lexemes))),
+  );
+}
+
+/**
+ * Which search a fixture "server" performs.
+ *
+ * `postgres_fulltext` is what the live tenant runs and is what a retrieval test
+ * has to be written against. The permissive default is kept only so suites that
+ * predate the tenant learning to search keep testing what they were written to
+ * test; new retrieval work should ask for the real one.
+ */
+export type FixtureSearchBackend = 'permissive' | 'postgres_fulltext';
+
+/**
+ * Every string VALUE in a `data` blob, the way `jsonb_to_tsvector(cfg, data,
+ * '["string"]')` indexes it — values, never keys, so `?search=title` does not
+ * match every record in the tenant.
+ */
+function jsonStrings(value: unknown): string {
+  if (typeof value === 'string') return value;
+  if (Array.isArray(value)) return value.map(jsonStrings).join(' ');
+  if (value && typeof value === 'object') {
+    return Object.values(value as Record<string, unknown>).map(jsonStrings).join(' ');
+  }
+  return '';
+}
+
 // ---------------------------------------------------------------------------
 //  a CorpusApi over the fixtures
 // ---------------------------------------------------------------------------
 
 export interface FixtureCorpusOptions {
+  /** Which `?search=` semantics this "server" applies. See `FixtureSearchBackend`. */
+  searchBackend?: FixtureSearchBackend;
   documents?: SourceDocument[];
   moneyEvents?: MoneyEvent[];
   parties?: Party[];
@@ -632,9 +746,19 @@ export function createFixtureCorpusApi(options: FixtureCorpusOptions = {}): Corp
     defaultPageSize = 5,
     scope: serverScope,
     leakScope = false,
+    searchBackend = 'permissive',
     onQuery,
   } = options;
 
+  const matchesSearch = (term: string, text: string) =>
+    searchBackend === 'postgres_fulltext'
+      ? matchesTenantSearch(term, text)
+      : term
+          .toLowerCase()
+          .split(/\s+/)
+          .filter(Boolean)
+          .some((needle) => text.toLowerCase().includes(needle));
+
   const inScope = <T extends { scope: CorpusScope }>(rows: T[]): T[] => {
     if (leakScope || !serverScope) return rows;
     return rows.filter(
@@ -670,11 +794,7 @@ export function createFixtureCorpusApi(options: FixtureCorpusOptions = {}): Corp
       }
       if (query.search) {
         applied.push('search');
-        const needles = query.search.toLowerCase().split(/\s+/).filter(Boolean);
-        rows = rows.filter((document) => {
-          const text = haystack(document);
-          return needles.some((needle) => text.includes(needle));
-        });
+        rows = rows.filter((document) => matchesSearch(query.search!, haystack(document)));
       }
 
       rows = [...rows].sort((a, b) => (b.occurredAt ?? '').localeCompare(a.occurredAt ?? ''));
@@ -715,11 +835,12 @@ export function createFixtureCorpusApi(options: FixtureCorpusOptions = {}): Corp
       }
       if (query.search) {
         applied.push('search');
-        const needles = query.search.toLowerCase().split(/\s+/).filter(Boolean);
-        rows = rows.filter((event) => {
-          const text = `${event.subtype} ${event.status} ${event.description ?? ''} ${event.party?.name ?? ''}`.toLowerCase();
-          return needles.some((needle) => text.includes(needle));
-        });
+        rows = rows.filter((event) =>
+          matchesSearch(
+            query.search!,
+            `${event.subtype} ${event.status} ${event.description ?? ''} ${event.party?.name ?? ''}`,
+          ),
+        );
       }
       return paginate(rows, query.page ?? 1, query.pageSize ?? 20, applied);
     },
@@ -743,8 +864,12 @@ export function createFixtureCorpusApi(options: FixtureCorpusOptions = {}): Corp
  * block was lost (bd startsim-768w.38.22). It reproduces the tenant's real
  * behaviour, including the parts that are inconvenient:
  *
- *   • `?type=` is the ONLY filter honoured — `search`, `source` and `party` are
- *     accepted and ignored, exactly as `EntityViewSet` does today;
+ *   • `?type=` is honoured, and `?search=` is honoured ONLY when the caller asks
+ *     for `searchBackend: 'postgres_fulltext'` — the tenant has learned to
+ *     search since this was written (bd startsim-768w.38.12) and a retrieval
+ *     test must be written against the real semantics, but the suites that
+ *     predate it are still testing what they were written to test. `source` and
+ *     `party` are still accepted and ignored, exactly as `EntityViewSet` does;
  *   • `page_size` is IGNORED (`PageNumberPagination` with no
  *     `page_size_query_param`), so a caller asking for 200 gets `pageSize`;
  *   • `/api/v1/relationships/` returns every edge in the org, unfiltered,
@@ -760,6 +885,8 @@ export function createFixtureCorpusHttp(options: {
   pageSize?: number;
   /** Endpoints to fail, so the fail-soft paths are exercised. */
   missing?: string[];
+  /** Which `?search=` semantics this "tenant" applies. See `FixtureSearchBackend`. */
+  searchBackend?: FixtureSearchBackend;
   onRequest?: (path: string, params: Record<string, unknown>) => void;
 } = {}) {
   const entities = options.entities ?? [
@@ -803,8 +930,19 @@ export function createFixtureCorpusHttp(options: {
           return found as T;
         }
         const type = params.type ? String(params.type) : '';
-        // `search`/`source`/`party` are deliberately NOT applied.
-        const rows = type ? entities.filter((entity) => entity.entity_type === type) : entities;
+        // `source`/`party` are deliberately NOT applied. `search` is, but only
+        // for a caller that asked for the tenant's real backend.
+        let rows = type ? entities.filter((entity) => entity.entity_type === type) : entities;
+        const term = params.search ? String(params.search) : '';
+        if (term && options.searchBackend === 'postgres_fulltext') {
+          // The vector is built from `name` and every STRING VALUE in `data`,
+          // exactly like the generated column — and from the data the server
+          // WOULD send, so a leaking fixture's search leaks too. The client is
+          // not allowed to be safe only because the server was.
+          rows = rows.filter((entity) =>
+            matchesTenantSearch(term, `${entity.name ?? ''} ${jsonStrings(entity.data)}`),
+          );
+        }
         return slice(rows, params) as T;
       }
       throw Object.assign(new Error(`${path} responded 404`), { status: 404 });

package/src/corpus/index.ts

@@ -73,12 +73,14 @@ export type { CorpusTypeKeys, TenantEntity, TenantRelationship } from './entitie
 export {
   enforceScope,
   isInScope,
+  questionTerms,
   rankDocuments,
   retrieveForQuestion,
+  searchTermFor,
   snippetFor,
   tokenize,
 } from './retrieval';
-export type { RetrieveOptions, ScopeFilterResult } from './retrieval';
+export type { QuestionTerms, RetrieveOptions, ScopeFilterResult } from './retrieval';
 
 export {
   answerSystemPrompt,

package/src/corpus/retrieval.ts

@@ -29,12 +29,21 @@
  * what was dropped is surfaced rather than swallowed, because a non-zero drop is
  * a bug somewhere upstream that someone needs to see.
  */
+import { stem } from './text';
 import type { CorpusScope, RetrievedDocument, SourceDocument } from './types';
 
 /**
  * Terms too common to carry meaning. Deliberately small: an aggressive stoplist
  * silently deletes the user's question. `contract`, `invoice` and every domain
  * word stay in, because they are exactly what distinguishes one record here.
+ *
+ * The second block is the QUESTION vocabulary — the interrogatives, auxiliaries
+ * and discourse verbs that appear because somebody asked rather than because
+ * they meant something ("what did we decide", "can you tell me", "should we").
+ * They matter for the same reason as the rest: a term in this list is a term the
+ * server is not asked to match, and every word sent is a word that has to be
+ * found. Domain nouns are never in here — `decision` stays, `decide` goes,
+ * because one is a thing a record is about and the other is how a person asks.
  */
 const STOPWORDS = new Set([
   'a', 'about', 'all', 'am', 'an', 'and', 'any', 'are', 'as', 'at', 'be', 'been',
@@ -43,6 +52,16 @@ const STOPWORDS = new Set([
   'on', 'or', 'our', 'she', 'so', 'that', 'the', 'their', 'them', 'then', 'there',
   'these', 'they', 'this', 'to', 'us', 'was', 'we', 'were', 'what', 'when',
   'where', 'which', 'who', 'why', 'will', 'with', 'you', 'your',
+  // asked-not-meant
+  'again', 'also', 'anything', 'been', 'being', 'could', 'currently', 'decide',
+  'decided', 'doing', 'done', 'ever', 'everything', 'get', 'give', 'going',
+  'gonna', 'got', 'hey', 'im', 'into', 'just', 'know', 'lately', 'let', 'like',
+  'll', 'look', 'many', 'may', 'might', 'much', 'must', 'need', 'now', 'ok',
+  'okay', 'only', 'out', 'over', 'own', 'please', 'pull', 'really', 'remind',
+  're', 'said', 'same', 'say', 'says', 'see', 'shall', 'should', 'show', 'some',
+  'something', 'still', 'such', 'sure', 'tell', 'thing', 'things', 'think',
+  'told', 'up', 've', 've', 'very', 'want', 'wanna', 'whats', 'whom', 'whose',
+  'would', 'yet',
 ]);
 
 /** Lowercase word tokens, 2+ chars, stopwords removed. */
@@ -52,6 +71,113 @@ export function tokenize(text: string): string[] {
     .filter((token) => token.length >= 2 && !STOPWORDS.has(token));
 }
 
+// ---------------------------------------------------------------------------
+//  a question -> a query
+// ---------------------------------------------------------------------------
+
+/**
+ * THE FIX FOR bd startsim-768w.38.36, and the argument for its shape.
+ *
+ * The tenant compiles `?search=` with `websearch_to_tsquery('english', …)`,
+ * which joins bare words with `&`. Handing it a whole English question therefore
+ * asks for a document containing EVERY content word of that question — and on a
+ * corpus of a few thousand records, essentially no such document exists. The
+ * live measurement: `?search=parking` returned 2 records, `?search=parking
+ * allocation` returned 0, and every natural-language question returned 0 while a
+ * single rare word still answered.
+ *
+ * WHAT WAS CHOSEN: ONE request, a DISJUNCTION of the question's content words,
+ * ordered by the server's own `ts_rank_cd`, narrowed by the ranker below.
+ *
+ * Why not tier it — conjunction first, disjunction only when that returns
+ * nothing? It is one more round trip on exactly the questions that are already
+ * failing, and it puts a discontinuity in the middle of the product: adding one
+ * word to a question would silently move you between two different rankings for
+ * reasons no user can see. The deeper reason is that the two stages have
+ * different jobs. RETRIEVAL'S JOB IS RECALL — get the plausible records into the
+ * process. RANKING'S JOB IS PRECISION — decide which of them the model sees. A
+ * candidate set that is too narrow can never be rescued by a better ranker; one
+ * that is too broad always can, and the ranker for it already exists.
+ *
+ * Why not drop `?search=` and rank a recent window instead? Because that is the
+ * failure mode in the other direction: it answers every question from whatever
+ * happened to be recent, and it stops being possible at all the moment the
+ * corpus outgrows one page. The disjunction keeps the server doing the work it
+ * is good at — a stemmed, indexed, ranked scan of the whole corpus.
+ *
+ * PRECISION IS NOT TRADED AWAY. A one-word question compiles to exactly that one
+ * word, unchanged, so the rare-term case behaves identically to before. The
+ * widening only ever applies where a conjunction was going to return nothing.
+ *
+ * The syntax a person may already be typing survives: a `"quoted phrase"` stays
+ * quoted (the server phrase-matches it) and a `-word` stays excluded.
+ */
+export interface QuestionTerms {
+  /** Content words to match on, lowercased and deduplicated. */
+  include: string[];
+  /** Words the asker explicitly excluded with a leading `-`. */
+  exclude: string[];
+  /** Quoted phrases, kept verbatim so the server can phrase-match them. */
+  phrases: string[];
+}
+
+/**
+ * How many terms are worth sending. A question longer than this is a paragraph,
+ * and every extra disjunct widens the candidate set without adding signal —
+ * the ranker's coverage factor is what separates them once they are here.
+ */
+const MAX_QUERY_TERMS = 16;
+
+export function questionTerms(question: string): QuestionTerms {
+  const include: string[] = [];
+  const exclude: string[] = [];
+  const phrases: string[] = [];
+  const seen = new Set<string>();
+
+  for (const raw of question.match(/-?"[^"]*"|\S+/g) ?? []) {
+    const negated = raw.startsWith('-');
+    const value = negated ? raw.slice(1) : raw;
+    const quoted = value.startsWith('"') && value.endsWith('"') && value.length > 2;
+    const words = tokenize(quoted ? value.slice(1, -1) : value);
+    if (words.length === 0) continue;
+    if (quoted && !negated && words.length > 1) phrases.push(words.join(' '));
+    for (const word of words) {
+      const bucket = negated ? exclude : include;
+      const key = `${negated ? '-' : ''}${word}`;
+      if (seen.has(key)) continue;
+      seen.add(key);
+      bucket.push(word);
+    }
+  }
+  return { include: include.slice(0, MAX_QUERY_TERMS), exclude, phrases };
+}
+
+/**
+ * The `?search=` value for a question — `''` when there is nothing worth asking.
+ *
+ * An EMPTY term is returned rather than a term made of stopwords, because
+ * `websearch_to_tsquery` compiles a stopword-only query to an EMPTY tsquery and
+ * an empty tsquery matches NO row. "What is going on?" would ask the server for
+ * nothing and be told, accurately, that there is nothing. Omitting the parameter
+ * instead pulls the recent window, which is what that question means.
+ *
+ * Exclusions are sent AND re-applied by the ranker. `websearch_to_tsquery` has
+ * no parentheses, so `a or b -c` compiles to `a | b & !c` — the exclusion binds
+ * to the last disjunct only. That is a superset of what was asked for, never a
+ * subset, so the server can only over-return; `rankDocuments` makes it exact.
+ */
+export function searchTermFor(question: string): string {
+  const { include, exclude, phrases } = questionTerms(question);
+  if (include.length === 0) return '';
+  const phraseSet = new Set(phrases.flatMap((phrase) => phrase.split(' ')));
+  const disjuncts = [
+    ...phrases.map((phrase) => `"${phrase}"`),
+    ...include.filter((term) => !phraseSet.has(term)),
+  ];
+  const positive = disjuncts.join(' or ');
+  return [positive, ...exclude.map((term) => `-${term}`)].filter(Boolean).join(' ');
+}
+
 /**
  * Is this record inside the asking principal's scope?
  *
@@ -97,6 +223,17 @@ export function enforceScope<T extends { scope: CorpusScope }>(
 /** Half-life of the recency prior, in days. */
 const RECENCY_HALF_LIFE_DAYS = 120;
 
+/** Every piece of a record the reader is allowed to see, as one string. */
+function documentHaystack(document: SourceDocument): string {
+  return [
+    document.title,
+    document.summary ?? '',
+    document.body,
+    ...document.sections.map((section) => section.text),
+    ...document.participants.map((party) => `${party.name} ${party.email ?? ''}`),
+  ].join(' ');
+}
+
 /**
  * Fields are weighted because a title hit means more than a transcript hit.
  *
@@ -116,14 +253,81 @@ function recencyWeight(occurredAt: string | null | undefined, now: number): numb
   return 2 ** (-ageDays / RECENCY_HALF_LIFE_DAYS);
 }
 
-function countHits(tokens: string[], terms: Set<string>): { hits: number; matched: string[] } {
+/**
+ * TWO RETRIEVAL LAYERS THAT DISAGREE ABOUT WHAT A MATCH IS RETRIEVE NOTHING.
+ *
+ * The tenant indexes with `to_tsvector('english', …)`, so `blocked` and `block`
+ * are one lexeme there. This ranker compared raw tokens, so a record the server
+ * had correctly matched scored zero here and was thrown away again on the way to
+ * the prompt — invisible from either side, because each layer was behaving
+ * perfectly on its own. Live: `?search=blocked` returned the July record (its
+ * summary says "could block the funding round") and Ask still answered "nothing
+ * in your corpus matches that", having considered exactly zero documents.
+ *
+ * SO BOTH SIDES ARE STEMMED, with the Porter implementation this package
+ * ALREADY has in `./text` — the one the extraction pipeline dedupes commitments
+ * with. Postgres's `english` configuration is snowball, which is Porter2, and
+ * on the vocabulary that matters the two agree: `blocked`/`block`,
+ * `agreed`/`agree`, `filing`/`file`, `lease`/`leases`, `payments`/`payment`,
+ * `renewal`/`renewals` all collapse; `relapsed`/`relationship` and
+ * `block`/`blockchain` do not. Writing a second, weaker approximation next to a
+ * working stemmer would have been a third spelling of "what is a match", which
+ * is the whole shape of this bug.
+ *
+ * THE AGREEMENT IS MADE OVER TEXT THE READER MAY SEE, never by trusting the
+ * server's verdict. Scoring a record because the SERVER matched it would undo
+ * the field read gate: on a tenant that is leaking, the term that matched could
+ * be one from a withheld attribute. A record whose only hit is inside withheld
+ * text still scores zero here, because that text never reaches this function.
+ */
+
+/**
+ * Stems, memoized for the pass.
+ *
+ * One `stem()` call per distinct WORD rather than per token: a candidate set is
+ * up to 200 records and a record may be a 40-minute transcript, so the token
+ * count runs to millions while the vocabulary stays in the thousands.
+ */
+const stemCache = new Map<string, string>();
+
+function cachedStem(word: string): string {
+  const hit = stemCache.get(word);
+  if (hit !== undefined) return hit;
+  const value = stem(word);
+  // Bounded, because this module is long-lived in a server process and a corpus
+  // is not: an unbounded cache here is a slow leak, not a fast lookup.
+  if (stemCache.size > 20_000) stemCache.clear();
+  stemCache.set(word, value);
+  return value;
+}
+
+/**
+ * The query terms, indexed by stem for one pass per document, and mapped back to
+ * the word the user actually typed so `matchedTerms` can say what matched in
+ * their words rather than in Porter's.
+ */
+interface TermIndex {
+  terms: string[];
+  byStem: Map<string, string>;
+}
+
+function indexTerms(terms: string[]): TermIndex {
+  const byStem = new Map<string, string>();
+  for (const term of terms) {
+    const key = cachedStem(term);
+    if (!byStem.has(key)) byStem.set(key, term);
+  }
+  return { terms, byStem };
+}
+
+function countHits(tokens: string[], index: TermIndex): { hits: number; matched: string[] } {
   let hits = 0;
   const matched = new Set<string>();
   for (const token of tokens) {
-    if (terms.has(token)) {
-      hits += 1;
-      matched.add(token);
-    }
+    const term = index.byStem.get(cachedStem(token));
+    if (term === undefined) continue;
+    hits += 1;
+    matched.add(term);
   }
   return { hits, matched: [...matched] };
 }
@@ -132,7 +336,11 @@ function countHits(tokens: string[], terms: Set<string>): { hits: number; matche
  * A passage around the first matching term, so the citation card shows the user
  * WHY this record came back rather than making them open it to find out.
  */
-export function snippetFor(document: SourceDocument, terms: Set<string>, length = 220): string {
+export function snippetFor(
+  document: SourceDocument,
+  terms: Set<string> | Iterable<string>,
+  length = 220,
+): string {
   // The summary is searched FIRST, so a citation card quotes the curated
   // sentence rather than the moment somebody said the word out loud.
   const haystacks = [
@@ -140,12 +348,17 @@ export function snippetFor(document: SourceDocument, terms: Set<string>, length
     document.body,
     ...document.sections.map((section) => section.text),
   ];
-  for (const text of haystacks) {
+  // A term the ranker matched by STEM ("blocked" against "block") does not
+  // appear verbatim, so the passage is found by walking words and comparing
+  // stems rather than by `indexOf`. Otherwise the citation card falls back to
+  // the document's opening line and the reader is not shown WHY it came back —
+  // which is the one job of a snippet.
+  const wanted = new Set([...terms].map((term) => cachedStem(term.toLowerCase())));
+  for (const text of wanted.size ? haystacks : []) {
     if (!text) continue;
-    const lower = text.toLowerCase();
-    for (const term of terms) {
-      const at = lower.indexOf(term);
-      if (at === -1) continue;
+    for (const match of text.toLowerCase().matchAll(/[a-z0-9][a-z0-9'&.-]*/g)) {
+      if (!wanted.has(cachedStem(match[0]))) continue;
+      const at = match.index ?? 0;
       const start = Math.max(0, at - Math.floor(length / 3));
       const slice = text.slice(start, start + length).trim();
       return `${start > 0 ? '…' : ''}${slice}${start + length < text.length ? '…' : ''}`;
@@ -181,7 +394,8 @@ export function rankDocuments(
   options: RetrieveOptions = {},
 ): RetrievedDocument[] {
   const { limit = 8, now = Date.now(), minScore = 0.0001 } = options;
-  const terms = new Set(tokenize(question));
+  const { include, exclude } = questionTerms(question);
+  const terms = new Set(include);
   if (terms.size === 0) {
     // No usable terms — fall back to pure recency rather than returning
     // nothing. "What happened lately" is a real question.
@@ -196,23 +410,29 @@ export function rankDocuments(
       }));
   }
 
+  const index = indexTerms(include);
+  // `websearch_to_tsquery` has no parentheses, so the server can only approximate
+  // an exclusion inside a disjunction. It is made exact here.
+  const excluded = exclude.length ? indexTerms(exclude) : null;
+
   const scored: RetrievedDocument[] = [];
   for (const document of documents) {
-    const title = countHits(tokenize(document.title), terms);
+    if (excluded && countHits(tokenize(documentHaystack(document)), excluded).hits > 0) continue;
+    const title = countHits(tokenize(document.title), index);
     const participants = countHits(
       tokenize(document.participants.map((party) => `${party.name} ${party.email ?? ''}`).join(' ')),
-      terms,
+      index,
     );
     // Scored SEPARATELY from `body`, and only when they are actually different
     // strings — `body` falls back to the summary on a corpus that has never
     // delivered a full text, and counting the same words twice there would
     // silently re-weight every document on the tenant.
     const summaryText = document.summary && document.summary !== document.body ? document.summary : '';
-    const summary = countHits(tokenize(summaryText), terms);
-    const body = countHits(tokenize(document.body), terms);
+    const summary = countHits(tokenize(summaryText), index);
+    const body = countHits(tokenize(document.body), index);
     const sections = countHits(
       tokenize(document.sections.map((section) => section.text).join(' ')),
-      terms,
+      index,
     );
 
     // Saturating term counts (log1p) so one long transcript repeating a word

package/src/corpus/types.ts

@@ -210,6 +210,14 @@ export interface CorpusPage<T> {
  */
 export interface DocumentQuery {
   search?: string;
+  /**
+   * Server-side ordering. `relevance` is only meaningful alongside `search` —
+   * it sorts by the tenant's own `ts_rank_cd` over the same term, which is what
+   * makes page one of a broad query the page worth ranking (bd
+   * startsim-768w.38.36). A deployment that does not understand the parameter
+   * reports it under `ignored_filters` and orders as it always did.
+   */
+  order?: 'relevance' | 'recent' | 'oldest';
   source?: SourceKey;
   partyId?: string;
   /** ISO instants, filtering on `occurredAt`. */