@startsimpli/integrations 0.2.18 → 0.2.20

package/src/__tests__/translation-corpus.test.ts

@@ -0,0 +1,428 @@
+/**
+ * THE CLIENT CORPUS — ingestion, term extraction, memory, isolation
+ * (bd startsim-ppfl, RED).
+ *
+ * The bead asks for four things and this file is all four:
+ *
+ *   - client documents ingest, and terminology is extracted with the right
+ *     candidates surfaced FOR HUMAN CONFIRMATION — "an unconfirmed
+ *     auto-extracted term must not silently become authoritative";
+ *   - an approved translation pair becomes a memory entry and is hit on a
+ *     matching later segment;
+ *   - an accepted human edit becomes a new entry;
+ *   - one client's corpus is unreachable from another client's tenant.
+ *
+ * THE ISOLATION CASES ARE THE POINT OF THE LANE. `startsim-hopl` sells "one
+ * client's material never appears against another's", and a glossary term is
+ * the sharpest possible violation of it: short, memorable, and it would surface
+ * INSIDE ANOTHER CLIENT'S TRANSLATED DOCUMENT, in their own language, looking
+ * entirely legitimate.
+ *
+ * Every case here runs against records deliberately mixed across two orgs —
+ * the same posture `corpus-scope.test.ts` already takes, which assumes the query
+ * layer DOES NOT hold the line and proves the output is safe anyway.
+ */
+
+import { describe, expect, it } from 'vitest';
+
+import { FIXTURE_ORG, FIXTURE_OTHER_ORG } from '../corpus/fixtures';
+import {
+  TRANSLATION_TYPE_KEYS,
+  applyTermVerdict,
+  exactMemoryHit,
+  extractTermCandidates,
+  fuzzyMemoryMatches,
+  glossaryForSegment,
+  ingestForTranslation,
+  memoryUnitFromReviewerEdit,
+  normalizeForMemory,
+  readAllGlossaryTerms,
+  scopedMemory,
+  scopedTerms,
+  type GlossaryTermRecord,
+  type MemoryUnitRecord,
+  type TranslationCorpusApi,
+} from '../corpus/translation';
+import type { CorpusPage, CorpusScope, SourceDocument } from '../corpus/types';
+
+const TANDEM: CorpusScope = { orgId: FIXTURE_ORG };
+const NORTHWIND: CorpusScope = { orgId: FIXTURE_OTHER_ORG };
+
+const EN_AR = { sourceLocale: 'en', targetLocale: 'ar' };
+
+function doc(
+  id: string,
+  body: string,
+  scope: CorpusScope = TANDEM,
+  title = 'Client document',
+): SourceDocument {
+  return {
+    id,
+    title,
+    body,
+    sections: [],
+    occurredAt: '2026-03-01T00:00:00.000Z',
+    source: 'upload',
+    sourceLabel: 'Upload',
+    sourceRef: id,
+    participants: [],
+    scope,
+  };
+}
+
+function term(
+  over: Partial<GlossaryTermRecord> & Pick<GlossaryTermRecord, 'sourceTerm'>,
+): GlossaryTermRecord {
+  return {
+    id: `term-${over.sourceTerm}`,
+    sourceLocale: 'en',
+    targetLocale: 'ar',
+    targetTerm: 'اتفاقية التسوية',
+    status: 'approved',
+    occurrences: 3,
+    documentIds: ['doc-1'],
+    scope: TANDEM,
+    ...over,
+  };
+}
+
+function unit(over: Partial<MemoryUnitRecord> & Pick<MemoryUnitRecord, 'sourceText'>): MemoryUnitRecord {
+  return {
+    id: `unit-${over.sourceText.slice(0, 12)}`,
+    sourceLocale: 'en',
+    targetLocale: 'ar',
+    targetText: 'نص تجريبي',
+    sourceHash: over.sourceText.trim().toLowerCase(),
+    origin: 'reviewer_edit',
+    approved: true,
+    scope: TANDEM,
+    ...over,
+  };
+}
+
+// ---------------------------------------------------------------------------
+
+describe('ingestion', () => {
+  it('writes rows of the translation types and NOTHING else', () => {
+    // A connector that quietly invents a sixth entity_type is a schema change
+    // nobody reviewed. Generic envelope, declared types, no surprises.
+    const writes = ingestForTranslation(doc('doc-1', 'The settlement agreement is enclosed.'), {
+      scope: TANDEM,
+    });
+
+    const types = new Set(writes.map((w) => w.entity_type));
+    for (const type of types) {
+      expect(Object.values(TRANSLATION_TYPE_KEYS)).toContain(type);
+    }
+    expect(writes.length).toBeGreaterThan(0);
+  });
+
+  it('carries an external_id on every row, so a re-ingest converges instead of duplicating', () => {
+    const writes = ingestForTranslation(doc('doc-1', 'The settlement agreement is enclosed.'), {
+      scope: TANDEM,
+    });
+
+    for (const write of writes) expect(write.external_id).toBeTruthy();
+    expect(new Set(writes.map((w) => w.external_id)).size).toBe(writes.length);
+  });
+});
+
+describe('term extraction proposes, it does not decide', () => {
+  // Planted terms repeat; the noise does not. That is the whole signal, and it
+  // is why `minOccurrences` is a parameter rather than a constant.
+  const documents = [
+    doc(
+      'doc-1',
+      [
+        'The settlement agreement was signed on Tuesday.',
+        'A settlement agreement binds both parties.',
+        'Northwind Logistics accepted the settlement agreement.',
+        'Force majeure applies to Northwind Logistics only.',
+        'The force majeure clause was invoked.',
+        'Somebody left a blue umbrella in the lobby.',
+      ].join(' '),
+    ),
+    doc('doc-2', 'Northwind Logistics confirmed the force majeure position.'),
+  ];
+
+  const options = { scope: TANDEM, ...EN_AR, minOccurrences: 2 };
+
+  it('yields the planted terms', () => {
+    const candidates = extractTermCandidates(documents, options);
+    const sources = candidates.map((c) => c.sourceTerm.toLowerCase());
+
+    expect(sources).toContain('settlement agreement');
+    expect(sources).toContain('force majeure');
+    expect(sources).toContain('northwind logistics');
+  });
+
+  it('does NOT yield the planted noise', () => {
+    const candidates = extractTermCandidates(documents, options);
+    const sources = candidates.map((c) => c.sourceTerm.toLowerCase());
+
+    // Occurs once, and is not terminology however memorable it is.
+    expect(sources).not.toContain('blue umbrella');
+  });
+
+  it('marks every extracted term a CANDIDATE, never approved, never binding', () => {
+    // The rule the bead puts in capitals. A glossary that fills itself is a
+    // glossary the client never agreed to — and adherence is measured against it.
+    const candidates = extractTermCandidates(documents, options);
+
+    expect(candidates.length).toBeGreaterThan(0);
+    for (const candidate of candidates) {
+      expect(candidate.status).toBe('candidate');
+      expect(candidate.targetTerm).toBe('');
+    }
+  });
+
+  it('never proposes a term a human already rejected', () => {
+    const rejected = term({ sourceTerm: 'force majeure', status: 'rejected' });
+
+    const candidates = extractTermCandidates(documents, { ...options, known: [rejected] });
+
+    expect(candidates.map((c) => c.sourceTerm.toLowerCase())).not.toContain('force majeure');
+  });
+
+  it('promotes a term only through an explicit verdict', () => {
+    const candidate = term({ sourceTerm: 'settlement agreement', status: 'candidate', targetTerm: '' });
+
+    const approved = applyTermVerdict(candidate, {
+      status: 'approved',
+      targetTerm: 'اتفاقية التسوية',
+    });
+
+    expect(approved.status).toBe('approved');
+    expect(approved.targetTerm).toBe('اتفاقية التسوية');
+  });
+});
+
+describe('glossary lookup', () => {
+  const present = term({ sourceTerm: 'settlement agreement' });
+  const absent = term({ sourceTerm: 'force majeure', id: 'term-fm' });
+  const candidate = term({
+    sourceTerm: 'settlement agreement',
+    id: 'term-candidate',
+    status: 'candidate',
+    targetTerm: '',
+  });
+
+  it('returns only the terms whose source form OCCURS in the segment', () => {
+    // The prompt scales with the text, not with the glossary.
+    const got = glossaryForSegment('Please review the settlement agreement.', [present, absent]);
+
+    expect(got.map((t) => t.id)).toEqual([present.id]);
+  });
+
+  it('never returns an unconfirmed candidate, even when it occurs', () => {
+    const got = glossaryForSegment('Please review the settlement agreement.', [candidate]);
+
+    expect(got).toEqual([]);
+  });
+
+  it('matches case-insensitively without matching a different word', () => {
+    expect(
+      glossaryForSegment('The SETTLEMENT AGREEMENT is enclosed.', [present]).map((t) => t.id),
+    ).toEqual([present.id]);
+    expect(glossaryForSegment('The settlement is enclosed.', [present])).toEqual([]);
+  });
+});
+
+describe('translation memory', () => {
+  const stored = unit({
+    sourceText: 'The quarterly report is ready for your review.',
+    targetText: 'نص تجريبي: التقرير ربع السنوي جاهز للمراجعة.',
+  });
+
+  it('hits exactly after normalising whitespace and case', () => {
+    const hit = exactMemoryHit(
+      '  the QUARTERLY   report is ready for your review.  ',
+      [stored],
+      { scope: TANDEM, ...EN_AR },
+    );
+
+    expect(hit?.id).toBe(stored.id);
+    expect(hit?.targetText).toBe(stored.targetText);
+  });
+
+  it('does NOT hit after a material change', () => {
+    // A different number is a different sentence. Normalisation must not eat it.
+    const hit = exactMemoryHit(
+      'The monthly report is ready for your review.',
+      [stored],
+      { scope: TANDEM, ...EN_AR },
+    );
+
+    expect(hit).toBeNull();
+  });
+
+  it('normalises whitespace and case but preserves the material content', () => {
+    expect(normalizeForMemory('  The   Report  ')).toBe(normalizeForMemory('the report'));
+    expect(normalizeForMemory('12 seats')).not.toBe(normalizeForMemory('13 seats'));
+  });
+
+  it('never serves an unapproved unit', () => {
+    const draft = unit({ sourceText: stored.sourceText, id: 'draft', approved: false });
+
+    expect(exactMemoryHit(stored.sourceText, [draft], { scope: TANDEM, ...EN_AR })).toBeNull();
+  });
+
+  it('does not hit across a language pair', () => {
+    const zh = unit({ sourceText: stored.sourceText, id: 'zh', targetLocale: 'zh' });
+
+    expect(
+      exactMemoryHit(stored.sourceText, [zh], { scope: TANDEM, sourceLocale: 'en', targetLocale: 'ar' }),
+    ).toBeNull();
+  });
+
+  it('returns fuzzy neighbours ordered by score and honours the limit', () => {
+    const near = unit({ sourceText: 'The quarterly report is ready for review.', id: 'near' });
+    const mid = unit({ sourceText: 'The quarterly report will be ready soon.', id: 'mid' });
+    const far = unit({ sourceText: 'Parking allocation for the London office.', id: 'far' });
+
+    const matches = fuzzyMemoryMatches(stored.sourceText, [far, mid, near], {
+      scope: TANDEM,
+      ...EN_AR,
+      limit: 2,
+    });
+
+    expect(matches).toHaveLength(2);
+    expect(matches[0]!.unit.id).toBe('near');
+    expect(matches[0]!.score).toBeGreaterThan(matches[1]!.score);
+    expect(matches.map((m) => m.unit.id)).not.toContain('far');
+  });
+});
+
+describe('a reviewer edit becomes memory', () => {
+  it('creates an approved unit that a later identical segment hits exactly', () => {
+    // THE SINGLE TEST THAT PROVES "it learns from your material". Without it the
+    // reviewer corrects the same term forever, which is the upkeep trap that
+    // makes people abandon tools.
+    const source = 'Please review the settlement agreement before Friday.';
+    const edited = 'نص تجريبي مصحَّح: اتفاقية التسوية.';
+
+    const created = memoryUnitFromReviewerEdit(
+      { sourceText: source, editedTargetText: edited, ...EN_AR },
+      TANDEM,
+    );
+
+    expect(created.origin).toBe('reviewer_edit');
+    expect(created.approved).toBe(true);
+
+    const hit = exactMemoryHit(source, [created], { scope: TANDEM, ...EN_AR });
+    // BYTE-IDENTICAL. A reviewer who fixed it once must never see it wrong again.
+    expect(hit?.targetText).toBe(edited);
+  });
+});
+
+describe('isolation between clients', () => {
+  const mine = term({ sourceTerm: 'settlement agreement', id: 'mine', scope: TANDEM });
+  const theirs = term({ sourceTerm: 'settlement agreement', id: 'theirs', scope: NORTHWIND });
+  const myUnit = unit({ sourceText: 'Shared sentence.', id: 'my-unit', scope: TANDEM });
+  const theirUnit = unit({ sourceText: 'Shared sentence.', id: 'their-unit', scope: NORTHWIND });
+
+  it('does not return another tenant’s glossary term, and SAYS it dropped one', () => {
+    const read = scopedTerms([mine, theirs], TANDEM);
+
+    expect(read.kept.map((t) => t.id)).toEqual(['mine']);
+    // Non-zero and surfaced — a silent zero here is how a leak stays invisible.
+    expect(read.droppedOutOfScope).toBe(1);
+  });
+
+  it('does not return another tenant’s memory unit, and SAYS it dropped one', () => {
+    const read = scopedMemory([myUnit, theirUnit], TANDEM);
+
+    expect(read.kept.map((u) => u.id)).toEqual(['my-unit']);
+    expect(read.droppedOutOfScope).toBe(1);
+  });
+
+  it('never serves a foreign memory unit as an exact hit', () => {
+    // The worst case in the lane: a fluent, confident, correctly-formatted
+    // translation sourced from another customer's approved wording.
+    const hit = exactMemoryHit('Shared sentence.', [theirUnit], { scope: TANDEM, ...EN_AR });
+
+    expect(hit).toBeNull();
+  });
+
+  it('never offers a foreign unit as a fuzzy example', () => {
+    const matches = fuzzyMemoryMatches('Shared sentence.', [theirUnit], {
+      scope: TANDEM,
+      ...EN_AR,
+      limit: 5,
+    });
+
+    expect(matches).toEqual([]);
+  });
+
+  it('returns NOTHING for a principal carrying no org claim', () => {
+    // A missing claim must read as "nothing", never as "unscoped, so everything".
+    const read = scopedTerms([mine, theirs], { orgId: '' });
+
+    expect(read.kept).toEqual([]);
+    expect(read.droppedOutOfScope).toBe(2);
+  });
+
+  /**
+   * The RLS half of bd startsim-ppfl item 7 is NOT in this repository.
+   *
+   * It belongs to `tenant-starter`, a separate Django project with its own
+   * runner (`DJANGO_DEBUG=true .venv/bin/python manage.py test`, sqlite, its own
+   * venv — NOT the main-backend Docker stack). There is no `tenant-starter`
+   * checkout in this worktree, so the test cannot be written here, and writing
+   * a TypeScript imitation of it would be worse than leaving it undone: it would
+   * look like coverage of a database guarantee it does not touch.
+   *
+   * `startsim-bfem` already records this as UNVERIFIED, owner Quinn.
+   */
+  it.todo(
+    'reading outside org_scope() returns nothing rather than everything — tenant-starter, not this repo',
+  );
+});
+
+describe('paging', () => {
+  /** Three pages of two. A reader that stops at the first sees a third of the glossary. */
+  function pagedApi(): TranslationCorpusApi {
+    const all = [
+      term({ sourceTerm: 'alpha', id: 't1' }),
+      term({ sourceTerm: 'bravo', id: 't2' }),
+      term({ sourceTerm: 'charlie', id: 't3' }),
+      term({ sourceTerm: 'delta', id: 't4' }),
+      term({ sourceTerm: 'echo', id: 't5' }),
+      term({ sourceTerm: 'foxtrot', id: 't6' }),
+    ];
+    const PAGE = 2;
+    return {
+      async listGlossaryTerms({ page = 1 }): Promise<CorpusPage<GlossaryTermRecord>> {
+        const start = (page - 1) * PAGE;
+        const results = all.slice(start, start + PAGE);
+        return {
+          count: all.length,
+          // The server ignores `pageSize`, exactly as the live tenant does.
+          next: start + PAGE < all.length ? `?page=${page + 1}` : null,
+          previous: page > 1 ? `?page=${page - 1}` : null,
+          results,
+        };
+      },
+      async listMemoryUnits(): Promise<CorpusPage<MemoryUnitRecord>> {
+        return { count: 0, next: null, previous: null, results: [] };
+      },
+    };
+  }
+
+  it('reads the glossary PAST the first page', async () => {
+    const read = await readAllGlossaryTerms(pagedApi(), TANDEM);
+
+    expect(read.kept).toHaveLength(6);
+    expect(read.kept.map((t) => t.id)).toEqual(['t1', 't2', 't3', 't4', 't5', 't6']);
+  });
+
+  it('scope-filters every page, not just the first', async () => {
+    const read = await readAllGlossaryTerms(pagedApi(), NORTHWIND);
+
+    // Every fixture term belongs to tandem, so a northwind principal gets none
+    // — and is TOLD that six rows were dropped rather than shown an empty
+    // glossary that looks like a clean read.
+    expect(read.kept).toEqual([]);
+    expect(read.droppedOutOfScope).toBe(6);
+  });
+});

package/src/corpus/ask.ts

@@ -19,7 +19,7 @@
  * given has invented a source, and the only safe rendering of that is to say so.
  */
 import { moneyEventAsDocument } from './entities';
-import { retrieveForQuestion } from './retrieval';
+import { retrieveForQuestion, searchTermFor } from './retrieval';
 import { documentText } from './text';
 import type {
   AnswerRequest,
@@ -209,6 +209,20 @@ function snippetOf(document: SourceDocument): string {
  * Pull the candidate set. Two reads, in PARALLEL, not one read per document —
  * the server pages and filters, this only decides how wide a net to cast.
  *
+ * THE QUESTION IS NOT THE QUERY (bd startsim-768w.38.36). It used to be: the
+ * whole question went out as `?search=`, the tenant compiled it with
+ * `websearch_to_tsquery`, and the resulting conjunction of every content word
+ * matched no record at all. `searchTermFor` turns it into a disjunction of the
+ * content words instead — see the argument for that shape in `./retrieval`.
+ *
+ * `order: 'relevance'` is what makes the widening safe at scale. Once the query
+ * is a disjunction, more records match than fit on a page, and an unordered page
+ * one is a random sample of them. Asking the server to sort by its own
+ * `ts_rank_cd` means the page this pulls is the page worth ranking.
+ *
+ * A question with no content words sends NO search term (see `searchTermFor`),
+ * which pulls the recent window — the honest reading of "what is going on?".
+ *
  * Money events are folded in as documents so one ranking pass spans meetings and
  * revenue; "what did we agree with this company" is a question about both.
  */
@@ -216,9 +230,10 @@ async function loadCandidates(
   corpus: CorpusApi,
   request: AskRequest,
   candidateLimit: number,
-): Promise<SourceDocument[]> {
+): Promise<{ candidates: SourceDocument[]; searchTerm: string; matchedCount: number }> {
+  const searchTerm = searchTermFor(request.question);
   const documentQuery = {
-    search: request.question,
+    ...(searchTerm ? { search: searchTerm, order: 'relevance' as const } : {}),
     ...request.filters,
     page: 1,
     pageSize: candidateLimit,
@@ -229,7 +244,7 @@ async function loadCandidates(
     corpus.listMoneyEvents
       ? corpus
           .listMoneyEvents({
-            search: request.question,
+            ...(searchTerm ? { search: searchTerm } : {}),
             partyId: request.filters?.partyId,
             page: 1,
             pageSize: Math.ceil(candidateLimit / 4),
@@ -240,7 +255,54 @@ async function loadCandidates(
       : Promise.resolve({ count: 0, next: null, previous: null, results: [] }),
   ]);
 
-  return [...documents.results, ...money.results.map(moneyEventAsDocument)];
+  return {
+    candidates: [...documents.results, ...money.results.map(moneyEventAsDocument)],
+    searchTerm,
+    matchedCount: documents.count ?? documents.results.length,
+  };
+}
+
+/**
+ * How many records this workspace holds AT ALL — asked only when the answer is
+ * going to be "nothing matched", and only to decide WHICH "nothing".
+ *
+ * "Your corpus has not been pulled in yet" and "none of your 18 records mention
+ * that" send a person to two different places, and the surface shipped the first
+ * sentence for both. Somebody went to check a connector that was working.
+ */
+async function corpusSize(corpus: CorpusApi, filters: AskRequest['filters']): Promise<number> {
+  return corpus
+    .listDocuments({ ...filters, search: undefined, page: 1, pageSize: 1 })
+    .then((page) => page.count ?? page.results.length)
+    .catch(() => -1);
+}
+
+/**
+ * What to say when retrieval came back empty, without blaming the wrong thing.
+ *
+ * THE OLD SENTENCE WAS A LIE OF ATTRIBUTION. "Nothing in your corpus matches
+ * that. Either it has not been pulled in yet, or it belongs to a different
+ * workspace" was returned for a query-construction bug, over a workspace holding
+ * eighteen documents that were pulled in perfectly. It reads as a data problem
+ * and sends someone to check a connector that is working.
+ *
+ * THE QUESTION IS NOT REPEATED BACK. An earlier draft named the terms it had
+ * searched for, which reads well and is wrong here: the corpus-field-gate suite
+ * asks a question made entirely of a withheld word and requires the answer not
+ * to contain it. Echoing the query would have made the ONE sentence this branch
+ * exists to produce the one place that word appears on screen — and a reader
+ * cannot tell an echo from a quote.
+ */
+function noEvidenceCopy(total: number): string {
+  if (total === 0) {
+    return 'Nothing has been pulled into this workspace yet, so there are no records to search. Connect a source, or check that the last sync succeeded.';
+  }
+  if (total < 0) {
+    // The count read failed. Say what IS known — the search found nothing — and
+    // claim nothing about the corpus.
+    return 'The search came back empty. Try naming the company, person or document you mean.';
+  }
+  return `Searched all ${total} record${total === 1 ? '' : 's'} in this workspace and none of them matched. Try naming the company, person or document you mean — or ask about something they actually discussed.`;
 }
 
 /**
@@ -264,7 +326,11 @@ export async function askCorpus(
     return emptyResult(request, llm, 'This session has no tenant scope, so there is nothing to search.');
   }
 
-  const candidates = await loadCandidates(corpus, request, candidateLimit);
+  const { candidates, searchTerm, matchedCount } = await loadCandidates(
+    corpus,
+    request,
+    candidateLimit,
+  );
   const { retrieved, droppedOutOfScope } = retrieveForQuestion(
     candidates,
     request.question,
@@ -275,10 +341,12 @@ export async function askCorpus(
   const freshness = await corpus.listSources().catch(() => [] as SourceFreshness[]);
 
   if (retrieved.length === 0) {
+    // An unfiltered pull already tells us the size, so the extra read only
+    // happens on the path that is about to make a claim about the corpus.
+    const total = searchTerm ? await corpusSize(corpus, request.filters) : matchedCount;
     return {
       status: 'no_evidence',
-      answer:
-        'Nothing in your corpus matches that. Either it has not been pulled in yet, or it belongs to a different workspace.',
+      answer: noEvidenceCopy(total),
       citations: [],
       considered: [],
       unresolvedCitations: [],

package/src/corpus/client.ts

@@ -228,6 +228,9 @@ function params(query: DocumentQuery, type: string) {
   const out: Record<string, unknown> = { type, page: query.page ?? 1 };
   if (query.pageSize) out.page_size = query.pageSize;
   if (query.search) out.search = query.search;
+  // Only alongside a term: `order=relevance` with nothing to be relevant TO is a
+  // parameter the server has to reject or ignore, and neither is worth asking.
+  if (query.order && (query.order !== 'relevance' || query.search)) out.order = query.order;
   if (query.source) out.source = query.source;
   if (query.partyId) out.party = query.partyId;
   if (query.occurredAfter) out.occurred_after = query.occurredAfter;