@startsimpli/integrations 0.1.6 → 0.2.2

package/src/__tests__/corpus-extract.test.ts

@@ -0,0 +1,537 @@
+/**
+ * EXTRACTION — conversations in, trackable commitments out (bd startsim-768w.38.25).
+ *
+ * The bead's thesis is that the hard part is extraction and identity, and that
+ * tracking is trivial once the material is structured. This file tests the hard
+ * part. It is organised by the four ways it can be wrong, in the order they
+ * would destroy the product:
+ *
+ *   1. IDEMPOTENCE. Run it twice, get the same list. A pipeline that grows the
+ *      list every time it runs is worse than no pipeline.
+ *   2. DEDUPE. One commitment discussed in three meetings is ONE item with three
+ *      mentions. Get this wrong and the list is instantly useless.
+ *   3. RESOLUTION. A later meeting closing an earlier item is the difference
+ *      between a to-do list and a system of record — and a WRONG close is the
+ *      one direction a to-do list must never lie in.
+ *   4. ISOLATION. Same adversarial posture as `corpus-scope.test.ts`: every case
+ *      runs against an API deliberately configured to leak.
+ *
+ * The markdown in the fixtures is the REAL shape — bolded imperatives with an
+ * owner in parentheses under a `## Next Steps` heading — because that is what
+ * all sixteen documents on the live tandem tenant actually contain.
+ */
+import { describe, expect, it, vi } from 'vitest';
+
+import {
+  applyResolutions,
+  commitmentIdentity,
+  mergeDrafts,
+  reconcile,
+} from '../corpus/commitments';
+import { extractCommitments, rulesExtract } from '../corpus/extract';
+import {
+  createFixtureCorpusApi,
+  createFixtureExtractor,
+  FIXTURE_ORG,
+  FIXTURE_OTHER_ORG,
+  FIXTURE_TRACKED_DOCUMENTS,
+} from '../corpus/fixtures';
+import { jaccard, outcomeSections, stem, tokenize } from '../corpus/text';
+import type { Commitment, CommitmentDraft, SourceDocument } from '../corpus/types';
+
+const TANDEM = { orgId: FIXTURE_ORG };
+
+// ---------------------------------------------------------------------------
+//  the normaliser identity is built on
+// ---------------------------------------------------------------------------
+
+describe('normalisation', () => {
+  it('collapses the inflections two people use for one commitment', () => {
+    const same = (a: string, b: string) => expect(stem(a)).toBe(stem(b));
+    same('agree', 'agreed');
+    same('agree', 'agreement');
+    same('agree', 'agreements');
+    same('bill', 'billed');
+    same('bill', 'billing');
+    same('file', 'filed');
+    same('file', 'filing');
+    same('renew', 'renewal');
+    same('sign', 'signed');
+    same('schedule', 'scheduling');
+  });
+
+  it('does NOT collapse words that merely share a prefix', () => {
+    // Over-eager stemming merges two unrelated commitments into one row, which
+    // silently deletes work. That is worse than listing something twice.
+    const differs = (a: string, b: string) => expect(stem(a)).not.toBe(stem(b));
+    differs('bill', 'billion');
+    differs('sign', 'signal');
+    differs('file', 'filet');
+    differs('rent', 'rental');
+    differs('patent', 'patient');
+  });
+
+  it('leaves matter numbers alone, because they are the strongest identity there is', () => {
+    expect(stem('2040-01')).toBe('2040-01');
+    expect(tokenize('Matter 2040-01 drafting')).toContain('2040-01');
+  });
+});
+
+// ---------------------------------------------------------------------------
+//  the vendor already did half the work
+// ---------------------------------------------------------------------------
+
+describe('the deterministic pre-pass over the summariser’s own structure', () => {
+  const NEXT_STEPS = [
+    '## Rexford Paperwork',
+    '- Easement documents sent over for signature',
+    '',
+    '## Next Steps',
+    '',
+    '- **Complete the equity item today** (Jonathan)',
+    '',
+    '  Needs to go out urgently; other party will forward it once done.',
+    '- **Schedule call with Peggy for next week**',
+    '',
+    '  Also coordinate calls with Julia and Lindsay.',
+    '- **Set up call with Kevin, Jonathan, Brian (CTO)** (Jonathan, Kevin)',
+    '',
+    '  Align on iManage replacement requirements.',
+  ].join('\n');
+
+  it('finds the outcome block whatever the summariser called it', () => {
+    expect(outcomeSections('## Next Steps\n- a')).toHaveLength(1);
+    expect(outcomeSections('## Action Items\n- a')).toHaveLength(1);
+    expect(outcomeSections('### Follow-ups\n- a')).toHaveLength(1);
+    expect(outcomeSections('## Tender Pipeline\n- a')).toHaveLength(0);
+  });
+
+  it('reads the bullets as commitments with NO model involved', () => {
+    const drafts = rulesExtract(doc('d1', 'Rexford', NEXT_STEPS));
+    expect(drafts.map((draft) => draft.title)).toEqual([
+      'Complete the equity item today',
+      'Schedule call with Peggy for next week',
+      'Set up call with Kevin, Jonathan, Brian (CTO)',
+    ]);
+    expect(drafts.every((draft) => draft.extractor === 'rules')).toBe(true);
+    // The detail paragraph under the bullet belongs to the bullet.
+    expect(drafts[0].detail).toContain('Needs to go out urgently');
+  });
+
+  it('reads the owner out of the parentheses the summariser wrote', () => {
+    const drafts = rulesExtract(doc('d1', 'Rexford', NEXT_STEPS));
+    expect(drafts[0].ownerName).toBe('Jonathan');
+    expect(drafts[1].ownerName).toBeUndefined(); // no owner stated, and none invented
+    // A trailing parenthetical that is part of the sentence, not an owner, must
+    // not become one — "(CTO)" is a job title inside the title itself.
+    expect(drafts[2].ownerName).toBe('Jonathan, Kevin');
+    expect(drafts[2].title).toContain('(CTO)');
+  });
+
+  it('quotes the source verbatim, so every row can be checked against the record', () => {
+    const drafts = rulesExtract(doc('d1', 'Rexford', NEXT_STEPS));
+    for (const draft of drafts) {
+      expect(NEXT_STEPS).toContain(draft.quote);
+    }
+  });
+
+  it('produces a byte-stable origin key — no model, so no drift between runs', () => {
+    const first = rulesExtract(doc('d1', 'Rexford', NEXT_STEPS));
+    const second = rulesExtract(doc('d1', 'Rexford', NEXT_STEPS));
+    expect(first.map((draft) => draft.originKey)).toEqual(second.map((draft) => draft.originKey));
+  });
+
+  it('finds nothing, rather than guessing, in a document with no outcome block', () => {
+    expect(rulesExtract(doc('d2', 'Rambling', '## Notes\n- we talked about things'))).toEqual([]);
+  });
+});
+
+// ---------------------------------------------------------------------------
+//  1. IDENTITY AND IDEMPOTENCE
+// ---------------------------------------------------------------------------
+
+describe('identity: what makes two statements the same commitment', () => {
+  it('keys on the SUBJECT plus the action, not on the model’s wording', () => {
+    const a = commitmentIdentity({ title: 'Set a drafting deadline for matter 2040-01' });
+    const b = commitmentIdentity({ title: 'Deadline needs to be set on 2040-01', subject: 'matter 2040-01' });
+    expect(a.subjectKey).toBe('2040-01');
+    expect(b.subjectKey).toBe('2040-01');
+    // Not byte-equal keys — the wording differs — but near enough to merge, which
+    // is what `reconcile` uses. The exact key is the fast path, not the contract.
+    expect(jaccard(a.actionTokens, b.actionTokens)).toBeGreaterThanOrEqual(0.5);
+  });
+
+  it('prefers a matter number over prose when the text contains one', () => {
+    expect(commitmentIdentity({ title: 'Chase Dr. Zhang for comments on 2148' }).subjectKey).toBe('2148');
+  });
+
+  it('falls back to the stated subject when there is no identifier', () => {
+    const identity = commitmentIdentity({ title: 'Send the equity sheet', subject: 'Harry Holcomb' });
+    expect(identity.subjectKey).toBe('harri.holcomb');
+  });
+
+  it('gives two genuinely different commitments different keys', () => {
+    const a = commitmentIdentity({ title: 'Send the equity sheet to Harry', subject: 'Pigoso' });
+    const b = commitmentIdentity({ title: 'Restore remote desktop access', subject: 'QuickBooks' });
+    expect(a.dedupeKey).not.toBe(b.dedupeKey);
+  });
+});
+
+describe('idempotence: running it twice changes nothing', () => {
+  it('re-extracting the same document does not duplicate or inflate', () => {
+    const drafts = rulesExtract(FIXTURE_TRACKED_DOCUMENTS[0]);
+    const first = reconcile([], mergeDrafts(drafts), { scope: TANDEM, documents: FIXTURE_TRACKED_DOCUMENTS });
+    const second = reconcile(first.commitments, mergeDrafts(drafts), {
+      scope: TANDEM,
+      documents: FIXTURE_TRACKED_DOCUMENTS,
+    });
+
+    expect(second.commitments).toHaveLength(first.commitments.length);
+    expect(second.created).toBe(0);
+    for (const commitment of second.commitments) {
+      // The crux: a mention is keyed by (document, originKey), so a second pass
+      // over the same document REPLACES its mention rather than appending one.
+      expect(commitment.mentions).toHaveLength(1);
+    }
+    expect(second.commitments.map((item) => item.externalId).sort()).toEqual(
+      first.commitments.map((item) => item.externalId).sort(),
+    );
+  });
+
+  it('keeps a human’s edit when the extractor runs again over the same source', () => {
+    const drafts = rulesExtract(FIXTURE_TRACKED_DOCUMENTS[0]);
+    const first = reconcile([], mergeDrafts(drafts), { scope: TANDEM, documents: FIXTURE_TRACKED_DOCUMENTS });
+    // Somebody ticked it off on the screen.
+    const edited = first.commitments.map((item, index) =>
+      index === 0 ? { ...item, status: 'done' as const, statusReason: 'Closed by Jonathan' } : item,
+    );
+    const second = reconcile(edited, mergeDrafts(drafts), { scope: TANDEM, documents: FIXTURE_TRACKED_DOCUMENTS });
+
+    const same = second.commitments.find((item) => item.externalId === edited[0].externalId);
+    expect(same?.status).toBe('done');
+    expect(same?.statusReason).toBe('Closed by Jonathan');
+  });
+});
+
+// ---------------------------------------------------------------------------
+//  2. DEDUPE ACROSS MEETINGS
+// ---------------------------------------------------------------------------
+
+describe('dedupe: one commitment, three meetings, one row', () => {
+  const mentionsOf = (items: Commitment[], subject: string) =>
+    items.find((item) => item.subjectKey === subject)?.mentions ?? [];
+
+  it('collapses the same commitment stated three different ways', () => {
+    const drafts: CommitmentDraft[] = [
+      draft('m1', 'Set a drafting deadline for matter 2040-01', '2026-08-01'),
+      draft('m2', 'Deadline still needs setting on 2040-01', '2026-08-04'),
+      draft('m3', 'Agree the drafting deadline for 2040-01', '2026-08-07'),
+    ];
+    const merged = mergeDrafts(drafts);
+    const { commitments } = reconcile([], merged, {
+      scope: TANDEM,
+      documents: [documentFor('m1', '2026-08-01'), documentFor('m2', '2026-08-04'), documentFor('m3', '2026-08-07')],
+    });
+
+    expect(commitments).toHaveLength(1);
+    expect(mentionsOf(commitments, '2040-01')).toHaveLength(3);
+    // The FIRST time it was raised is the useful date on the screen — "this has
+    // been outstanding since the 1st" is the sentence the product exists to say.
+    expect(commitments[0].firstSeenAt).toBe('2026-08-01T00:00:00.000Z');
+    expect(commitments[0].lastSeenAt).toBe('2026-08-07T00:00:00.000Z');
+  });
+
+  it('does NOT merge two different commitments about the same subject', () => {
+    const merged = mergeDrafts([
+      draft('m1', 'Set a drafting deadline for 2040-01', '2026-08-01'),
+      draft('m2', 'Send the filing receipt for 2040-01 to the client', '2026-08-02'),
+    ]);
+    const { commitments } = reconcile([], merged, {
+      scope: TANDEM,
+      documents: [documentFor('m1', '2026-08-01'), documentFor('m2', '2026-08-02')],
+    });
+    expect(commitments).toHaveLength(2);
+  });
+
+  it('does NOT merge the same words about different subjects', () => {
+    // The failure that makes the list actively dangerous: "chase for comments"
+    // on two different matters is two different obligations.
+    const merged = mergeDrafts([
+      draft('m1', 'Chase for comments on 2148', '2026-08-01'),
+      draft('m2', 'Chase for comments on 2198', '2026-08-01'),
+    ]);
+    const { commitments } = reconcile([], merged, {
+      scope: TANDEM,
+      documents: [documentFor('m1', '2026-08-01'), documentFor('m2', '2026-08-01')],
+    });
+    expect(commitments).toHaveLength(2);
+  });
+
+  it('keeps the EARLIEST row’s identity when a later mention merges into it', () => {
+    // The id is what a citation, a link and a human's edit hang off. A merge
+    // that re-keys the row breaks all three.
+    const first = reconcile([], mergeDrafts([draft('m1', 'Set a deadline for 2040-01', '2026-08-01')]), {
+      scope: TANDEM,
+      documents: [documentFor('m1', '2026-08-01')],
+    });
+    const originalId = first.commitments[0].externalId;
+
+    const second = reconcile(
+      first.commitments,
+      mergeDrafts([draft('m2', 'Deadline needs to be set for matter 2040-01', '2026-08-05')]),
+      { scope: TANDEM, documents: [documentFor('m2', '2026-08-05')] },
+    );
+
+    expect(second.commitments).toHaveLength(1);
+    expect(second.commitments[0].externalId).toBe(originalId);
+    expect(second.merged).toBe(1);
+    expect(second.created).toBe(0);
+  });
+});
+
+// ---------------------------------------------------------------------------
+//  3. RESOLUTION DETECTION
+// ---------------------------------------------------------------------------
+
+describe('resolution: a later meeting closes an earlier item', () => {
+  function openItem(): Commitment[] {
+    return reconcile([], mergeDrafts([draft('m1', 'Chase Dr. Zhang for comments on 2148', '2026-08-01')]), {
+      scope: TANDEM,
+      documents: [documentFor('m1', '2026-08-01')],
+    }).commitments;
+  }
+
+  it('closes the item and records WHAT closed it', () => {
+    const { commitments, resolved } = applyResolutions(
+      openItem(),
+      [
+        {
+          documentId: 'm2',
+          subject: '2148',
+          statement: 'Comments came back from Dr. Zhang on 2148',
+          quote: '2148: comments received from Dr. Zhang, ready to proceed',
+          outcome: 'done',
+        },
+      ],
+      { documents: [documentFor('m2', '2026-08-06')] },
+    );
+
+    expect(resolved).toBe(1);
+    expect(commitments[0].status).toBe('done');
+    expect(commitments[0].resolvedBy).toBe('m2');
+    expect(commitments[0].resolvedAt).toBe('2026-08-06T00:00:00.000Z');
+    // The quote is the whole point: "why does it say this is done" has to be
+    // answerable without opening the meeting.
+    expect(commitments[0].statusReason).toContain('Dr. Zhang');
+  });
+
+  it('refuses to close an item raised AFTER the meeting that supposedly closed it', () => {
+    const later = reconcile([], mergeDrafts([draft('m9', 'Chase Dr. Zhang for comments on 2148', '2026-09-01')]), {
+      scope: TANDEM,
+      documents: [documentFor('m9', '2026-09-01')],
+    }).commitments;
+
+    const { commitments, resolved } = applyResolutions(
+      later,
+      [{ documentId: 'm2', subject: '2148', statement: 'Comments came back from Dr. Zhang', quote: 'q', outcome: 'done' }],
+      { documents: [documentFor('m2', '2026-08-06')] },
+    );
+
+    expect(resolved).toBe(0);
+    expect(commitments[0].status).toBe('open');
+  });
+
+  it('does NOT close a different item that happens to share the subject', () => {
+    const items = reconcile(
+      [],
+      mergeDrafts([draft('m1', 'Send the filing receipt for 2148 to the client', '2026-08-01')]),
+      { scope: TANDEM, documents: [documentFor('m1', '2026-08-01')] },
+    ).commitments;
+
+    const { commitments, needsReview, resolved } = applyResolutions(
+      items,
+      [{ documentId: 'm2', subject: '2148', statement: 'Comments came back from Dr. Zhang', quote: 'q', outcome: 'done' }],
+      { documents: [documentFor('m2', '2026-08-06')] },
+    );
+
+    expect(resolved).toBe(0);
+    expect(commitments[0].status).toBe('open');
+    // Weak matches are SURFACED, not applied and not thrown away. A to-do list
+    // that quietly empties itself is the one failure mode with no symptom.
+    expect(needsReview.length).toBeGreaterThan(0);
+  });
+
+  it('never closes an item a human already moved', () => {
+    const items = openItem().map((item) => ({ ...item, status: 'dropped' as const, statusReason: 'Not doing it' }));
+    const { commitments, resolved } = applyResolutions(
+      items,
+      [{ documentId: 'm2', subject: '2148', statement: 'Chase Dr. Zhang for comments on 2148', quote: 'q', outcome: 'done' }],
+      { documents: [documentFor('m2', '2026-08-06')] },
+    );
+    expect(resolved).toBe(0);
+    expect(commitments[0].status).toBe('dropped');
+  });
+});
+
+// ---------------------------------------------------------------------------
+//  the pipeline, end to end
+// ---------------------------------------------------------------------------
+
+describe('extractCommitments', () => {
+  it('sends every document in ONE call, with ids the model must echo', async () => {
+    const extractor = createFixtureExtractor();
+    const spy = vi.spyOn(extractor, 'extract');
+    const corpus = createFixtureCorpusApi({ scope: TANDEM, documents: FIXTURE_TRACKED_DOCUMENTS });
+
+    await extractCommitments({ scope: TANDEM }, { corpus, extractor });
+
+    expect(spy).toHaveBeenCalledTimes(1);
+    expect(spy.mock.calls[0][0].documents.length).toBe(FIXTURE_TRACKED_DOCUMENTS.length);
+  });
+
+  it('tells the model what the rules already found, so it adds rather than restates', async () => {
+    const extractor = createFixtureExtractor();
+    const spy = vi.spyOn(extractor, 'extract');
+    const corpus = createFixtureCorpusApi({ scope: TANDEM, documents: FIXTURE_TRACKED_DOCUMENTS });
+
+    await extractCommitments({ scope: TANDEM }, { corpus, extractor });
+
+    const sent = spy.mock.calls[0][0].documents;
+    expect(sent.some((document) => document.alreadyFound.length > 0)).toBe(true);
+  });
+
+  it('reports rules and model contributions separately, so the model’s value is measurable', async () => {
+    const corpus = createFixtureCorpusApi({ scope: TANDEM, documents: FIXTURE_TRACKED_DOCUMENTS });
+    const result = await extractCommitments(
+      { scope: TANDEM },
+      { corpus, extractor: createFixtureExtractor() },
+    );
+    expect(result.fromRules).toBeGreaterThan(0);
+    expect(result.documentsProcessed).toBe(FIXTURE_TRACKED_DOCUMENTS.length);
+    expect(result.created).toBeGreaterThan(0);
+  });
+
+  it('still produces the vendor’s own items when the model is unreachable', async () => {
+    // The deterministic half must not depend on the probabilistic half. A
+    // provider outage should cost the extras, not the whole list.
+    const corpus = createFixtureCorpusApi({ scope: TANDEM, documents: FIXTURE_TRACKED_DOCUMENTS });
+    const extractor = createFixtureExtractor();
+    const result = await extractCommitments(
+      { scope: TANDEM },
+      {
+        corpus,
+        extractor: {
+          describe: extractor.describe,
+          extract: async () => { throw new Error('provider is down'); },
+        },
+      },
+    );
+    expect(result.fromRules).toBeGreaterThan(0);
+    expect(result.fromModel).toBe(0);
+    expect(result.created).toBeGreaterThan(0);
+  });
+
+  it('refuses outright when the session carries no workspace claim', async () => {
+    const corpus = createFixtureCorpusApi({ leakScope: true, documents: FIXTURE_TRACKED_DOCUMENTS });
+    const listDocuments = vi.spyOn(corpus, 'listDocuments');
+    const result = await extractCommitments(
+      { scope: { orgId: '' } },
+      { corpus, extractor: createFixtureExtractor() },
+    );
+    expect(result.created).toBe(0);
+    expect(listDocuments).not.toHaveBeenCalled();
+  });
+});
+
+// ---------------------------------------------------------------------------
+//  4. ISOLATION — against an API configured to leak
+// ---------------------------------------------------------------------------
+
+describe('extraction holds the workspace boundary', () => {
+  const foreign = (document: SourceDocument): SourceDocument => ({
+    ...document,
+    id: `nw_${document.id}`,
+    scope: { orgId: FIXTURE_OTHER_ORG },
+  });
+
+  it('never sends another workspace’s document to the extractor', async () => {
+    const documents = [...FIXTURE_TRACKED_DOCUMENTS, ...FIXTURE_TRACKED_DOCUMENTS.map(foreign)];
+    const corpus = createFixtureCorpusApi({ scope: TANDEM, leakScope: true, documents });
+    const extractor = createFixtureExtractor();
+    const spy = vi.spyOn(extractor, 'extract');
+
+    const result = await extractCommitments({ scope: TANDEM }, { corpus, extractor });
+
+    const sentIds = spy.mock.calls[0][0].documents.map((document) => document.id);
+    expect(sentIds.every((id) => !id.startsWith('nw_'))).toBe(true);
+    expect(result.droppedOutOfScope).toBe(FIXTURE_TRACKED_DOCUMENTS.length);
+  });
+
+  it('never writes a commitment carrying another workspace’s scope', async () => {
+    const documents = [...FIXTURE_TRACKED_DOCUMENTS, ...FIXTURE_TRACKED_DOCUMENTS.map(foreign)];
+    const corpus = createFixtureCorpusApi({ scope: TANDEM, leakScope: true, documents });
+    const saved: Commitment[] = [];
+    (corpus as { saveCommitments?: unknown }).saveCommitments = async (items: Commitment[]) => {
+      saved.push(...items);
+    };
+
+    await extractCommitments({ scope: TANDEM }, { corpus, extractor: createFixtureExtractor() });
+
+    expect(saved.length).toBeGreaterThan(0);
+    expect(saved.every((item) => item.scope.orgId === FIXTURE_ORG)).toBe(true);
+    expect(saved.every((item) => item.mentions.every((mention) => !mention.documentId.startsWith('nw_')))).toBe(true);
+  });
+
+  it('a resolution from another workspace cannot close this workspace’s item', () => {
+    const items = reconcile([], mergeDrafts([draft('m1', 'Chase Dr. Zhang for comments on 2148', '2026-08-01')]), {
+      scope: TANDEM,
+      documents: [documentFor('m1', '2026-08-01')],
+    }).commitments;
+
+    // The resolving document is not in the in-scope set handed to `applyResolutions`,
+    // which is the only place a document id is trusted.
+    const { commitments, resolved } = applyResolutions(
+      items,
+      [{ documentId: 'nw_m2', subject: '2148', statement: 'Comments came back from Dr. Zhang on 2148', quote: 'q', outcome: 'done' }],
+      { documents: [documentFor('m1', '2026-08-01')] },
+    );
+
+    expect(resolved).toBe(0);
+    expect(commitments[0].status).toBe('open');
+  });
+});
+
+// ---------------------------------------------------------------------------
+
+function doc(id: string, title: string, body: string, occurredAt = '2026-08-01T00:00:00Z'): SourceDocument {
+  return {
+    id,
+    title,
+    body,
+    sections: [],
+    occurredAt,
+    source: 'granola',
+    sourceLabel: 'Granola',
+    sourceRef: `granola:${id}`,
+    participants: [],
+    scope: { orgId: FIXTURE_ORG },
+  };
+}
+
+function documentFor(id: string, date: string): SourceDocument {
+  return doc(id, `Meeting ${id}`, '', `${date}T00:00:00Z`);
+}
+
+function draft(documentId: string, title: string, date: string): CommitmentDraft {
+  return {
+    documentId,
+    originKey: `${documentId}:${title.slice(0, 24)}`,
+    subtype: 'action',
+    title,
+    detail: '',
+    status: 'open',
+    quote: title,
+    extractor: 'rules',
+    ...(date ? {} : {}),
+  };
+}