@startsimpli/integrations 0.2.3 → 0.2.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@startsimpli/integrations",
-  "version": "0.2.3",
+  "version": "0.2.7",
   "description": "Provider-agnostic types, API clients, hooks, and UI primitives for email + calendar integrations across StartSimpli apps.",
   "main": "./src/index.ts",
   "types": "./src/index.ts",

package/src/__tests__/corpus-extract.test.ts

@@ -198,6 +198,60 @@ describe('idempotence: running it twice changes nothing', () => {
     );
   });
 
+  it('survives the MODEL rewording its own output between runs', () => {
+    // THE PRODUCTION BUG, pinned. On the live tandem tenant a second extraction
+    // over the same four documents created SIXTEEN duplicate rows: the model
+    // reworded its own `subject` ("ketamine regimen" -> "ketamine treatment")
+    // while emitting a byte-identical title, and a disagreeing subject was
+    // vetoing the match. Prose subjects are now weak evidence, not a veto.
+    const first = reconcile([], mergeDrafts([
+      { ...draft('m1', 'Find a sustainable ketamine regimen', '2026-08-01'), subject: 'ketamine regimen', extractor: 'model' as const },
+    ]), { scope: TANDEM, documents: [documentFor('m1', '2026-08-01')] });
+
+    const second = reconcile(first.commitments, mergeDrafts([
+      { ...draft('m1', 'Find a sustainable ketamine regimen', '2026-08-01'), subject: "Quinn's ketamine treatment", extractor: 'model' as const },
+    ]), { scope: TANDEM, documents: [documentFor('m1', '2026-08-01')] });
+
+    expect(second.created).toBe(0);
+    expect(second.commitments).toHaveLength(1);
+  });
+
+  it('still refuses to merge two MATTER NUMBERS that disagree, however alike the words', () => {
+    // The other side of the same rule. An identifier is authoritative: "chase
+    // for comments on 2148" and "...on 2198" have identical action stems and are
+    // two different obligations. Merging them deletes work.
+    const { commitments } = reconcile([], mergeDrafts([
+      draft('m1', 'Chase for comments on 2148', '2026-08-01'),
+      draft('m1', 'Chase for comments on 2198', '2026-08-01'),
+    ]), { scope: TANDEM, documents: [documentFor('m1', '2026-08-01')] });
+    expect(commitments).toHaveLength(2);
+  });
+
+  it('tells the model what is ALREADY TRACKED for a document, not just what the rules found', async () => {
+    // The structural half of idempotency: the fuzzy matcher is a safety net, and
+    // not asking the model to invent the same item twice is the actual fix.
+    const corpus = createFixtureCorpusApi({ scope: TANDEM, documents: FIXTURE_TRACKED_DOCUMENTS });
+    const tracked: Commitment[] = [];
+    (corpus as { listCommitments?: unknown }).listCommitments = async () => ({
+      count: tracked.length, next: null, previous: null, results: tracked,
+    });
+    (corpus as { saveCommitments?: unknown }).saveCommitments = async (items: Commitment[]) => {
+      tracked.splice(0, tracked.length, ...items);
+    };
+
+    const extractor = createFixtureExtractor();
+    const spy = vi.spyOn(extractor, 'extract');
+
+    await extractCommitments({ scope: TANDEM }, { corpus, extractor });
+    const firstAsk = spy.mock.calls[0][0].documents.flatMap((d) => d.alreadyFound).length;
+
+    await extractCommitments({ scope: TANDEM }, { corpus, extractor });
+    const secondAsk = spy.mock.calls[1][0].documents.flatMap((d) => d.alreadyFound).length;
+
+    expect(tracked.length).toBeGreaterThan(0);
+    expect(secondAsk).toBeGreaterThan(firstAsk);
+  });
+
   it('keeps a human’s edit when the extractor runs again over the same source', () => {
     const drafts = rulesExtract(FIXTURE_TRACKED_DOCUMENTS[0]);
     const first = reconcile([], mergeDrafts(drafts), { scope: TANDEM, documents: FIXTURE_TRACKED_DOCUMENTS });
@@ -453,6 +507,53 @@ describe('extractCommitments', () => {
     expect(second.nextOffset).toBeNull();
   });
 
+  it('writes only what it CHANGED, not the whole corpus', async () => {
+    // Not a micro-optimisation. The tenant has no bulk upsert, so every row is a
+    // request; rewriting untouched rows makes a one-document slice cost one
+    // request per row in the workspace. On the live tenant that pushed a
+    // single-document run past the 60-second gateway at ninety-eight rows.
+    const corpus = createFixtureCorpusApi({ scope: TANDEM, documents: FIXTURE_TRACKED_DOCUMENTS });
+    const tracked: Commitment[] = [];
+    const writes: number[] = [];
+    (corpus as { listCommitments?: unknown }).listCommitments = async () => ({
+      count: tracked.length, next: null, previous: null, results: tracked,
+    });
+    (corpus as { saveCommitments?: unknown }).saveCommitments = async (items: Commitment[]) => {
+      writes.push(items.length);
+      const byId = new Map(tracked.map((item) => [item.externalId, item]));
+      for (const item of items) byId.set(item.externalId, item);
+      tracked.splice(0, tracked.length, ...byId.values());
+    };
+    const extractor = createFixtureExtractor();
+
+    await extractCommitments(
+      { scope: TANDEM, documentIds: [FIXTURE_TRACKED_DOCUMENTS[0].id] },
+      { corpus, extractor },
+    );
+    const afterFirst = tracked.length;
+    await extractCommitments(
+      { scope: TANDEM, documentIds: [FIXTURE_TRACKED_DOCUMENTS[1].id] },
+      { corpus, extractor },
+    );
+
+    // The second slice never rewrites the first slice's untouched rows.
+    expect(writes[1]).toBeLessThan(tracked.length);
+    expect(afterFirst).toBeGreaterThan(0);
+  });
+
+  it('marks NOTHING dirty when a pass changes nothing', () => {
+    // The guarantee stated where it is unambiguous — on the pure reconciler,
+    // with no extractor in the way. Identical input, second time: zero writes.
+    const drafts = mergeDrafts(FIXTURE_TRACKED_DOCUMENTS.flatMap(rulesExtract));
+    const context = { scope: TANDEM, documents: FIXTURE_TRACKED_DOCUMENTS };
+    const first = reconcile([], drafts, context);
+    expect(first.touched.length).toBe(first.commitments.length);
+
+    const second = reconcile(first.commitments, drafts, context);
+    expect(second.created).toBe(0);
+    expect(second.touched).toEqual([]);
+  });
+
   it('refuses outright when the session carries no workspace claim', async () => {
     const corpus = createFixtureCorpusApi({ leakScope: true, documents: FIXTURE_TRACKED_DOCUMENTS });
     const listDocuments = vi.spyOn(corpus, 'listDocuments');

package/src/corpus/commitments.ts

@@ -61,6 +61,23 @@ export const EXTRACTOR_VERSION = 'rules+model@1';
  */
 const MERGE_THRESHOLD = 0.5;
 
+/**
+ * The bar when the two statements do NOT agree on a subject — either because
+ * one of them has none, or because they disagree.
+ *
+ * A DISAGREEING SUBJECT MUST NOT VETO A MATCH, and that was a real production
+ * bug: on the live tandem tenant a second extraction run over the same four
+ * documents created sixteen duplicate rows, because the model had rephrased its
+ * own `subject` between runs ("ketamine regimen" vs "ketamine treatment") and
+ * the veto threw away pairs whose TITLES were byte-identical. Measured across
+ * the resulting 98 rows: five same-document duplicate pairs sat at 0.5 or above
+ * on title overlap, and ZERO cross-document pairs reached even 0.5 — so an
+ * unanchored bar of 0.7 catches the rewording with no measured false merge.
+ *
+ * An agreeing subject still buys the lower bar, because it is real evidence.
+ */
+const UNANCHORED_MERGE_THRESHOLD = 0.7;
+
 /**
  * The bar a resolution has to clear before it CLOSES something, which is higher
  * than the bar for merging because the failure is worse. A wrong merge shows one
@@ -100,6 +117,19 @@ const MATTER_BARE = /\b(?!(?:19|20)\d{2}\b)\d{4}\b/;
 
 export interface CommitmentIdentity {
   subjectKey: string;
+  /**
+   * True when the subject is an IDENTIFIER (a matter number) rather than prose.
+   *
+   * The distinction is load-bearing. Two identifiers that disagree are two
+   * different things and no amount of shared wording changes that — "chase for
+   * comments on 2148" and "chase for comments on 2198" are two obligations whose
+   * action stems are identical. Two PROSE subjects that disagree mean much less:
+   * measured on the live tenant, the model described one commitment as being
+   * about "ketamine regimen" on one run and "ketamine treatment" on the next,
+   * while emitting a byte-identical title. So an identifier vetoes; prose does
+   * not, it only raises the bar.
+   */
+  subjectIsIdentifier: boolean;
   actionKey: string;
   actionTokens: string[];
   dedupeKey: string;
@@ -128,20 +158,43 @@ export function commitmentIdentity(input: {
   const subjectTokens = matter ? [matter] : unique(tokenize(subject));
   const subjectKey = matter ? matter : subjectTokens.slice().sort().join('.');
 
-  const drop = new Set(subjectTokens);
+  // ONLY AN IDENTIFIER IS REMOVED FROM THE ACTION. Removing prose subject words
+  // too would couple the two halves of the key: when the model reworded a
+  // subject from "ketamine regimen" to "Quinn's ketamine treatment", a
+  // byte-identical title produced two different action sets and the row
+  // duplicated. The matter number is noise in the action ("chase 2148" is a
+  // chase); the prose words are part of what the title says.
+  const drop = new Set(matter ? [matter] : []);
   const actionTokens = unique(
     tokenize(title).filter((token) => !drop.has(token) && !ACTION_NOISE.has(token)),
   );
   const actionKey = actionTokens.slice().sort().join('.');
 
-  return { subjectKey, actionKey, actionTokens, dedupeKey: hash(`${subjectKey}|${actionKey}`) };
+  return {
+    subjectKey,
+    subjectIsIdentifier: Boolean(matter),
+    actionKey,
+    actionTokens,
+    dedupeKey: hash(`${subjectKey}|${actionKey}`),
+  };
 }
 
 /** Do two identities describe one commitment? Tier 1, then Tier 2. */
 export function isSameCommitment(a: CommitmentIdentity, b: CommitmentIdentity): boolean {
   if (a.dedupeKey === b.dedupeKey) return true;
-  if (!a.subjectKey || a.subjectKey !== b.subjectKey) return false;
-  return jaccard(a.actionTokens, b.actionTokens) >= MERGE_THRESHOLD;
+  if (subjectsConflict(a, b)) return false;
+  const anchored = Boolean(a.subjectKey) && a.subjectKey === b.subjectKey;
+  const bar = anchored ? MERGE_THRESHOLD : UNANCHORED_MERGE_THRESHOLD;
+  return jaccard(a.actionTokens, b.actionTokens) >= bar;
+}
+
+/** Two IDENTIFIERS that disagree. The one veto worth having. */
+function subjectsConflict(a: CommitmentIdentity, b: CommitmentIdentity): boolean {
+  return (
+    a.subjectIsIdentifier &&
+    b.subjectIsIdentifier &&
+    a.subjectKey !== b.subjectKey
+  );
 }
 
 // ---------------------------------------------------------------------------
@@ -193,6 +246,15 @@ export interface ReconcileResult {
   commitments: Commitment[];
   created: number;
   merged: number;
+  /**
+   * The `externalId`s this pass actually changed.
+   *
+   * A run must write what it CHANGED, not what it holds. Rewriting the whole
+   * corpus on every pass is one request per row per slice — O(corpus) writes for
+   * O(slice) work — and on the live tenant that alone pushed a single-document
+   * run past the 60-second gateway once there were ninety-eight rows.
+   */
+  touched: string[];
 }
 
 export function reconcile(
@@ -202,6 +264,7 @@ export function reconcile(
 ): ReconcileResult {
   const byId = new Map(context.documents.map((document) => [document.id, document]));
   const out = existing.map(cloneCommitment);
+  const touched = new Set<string>();
   let created = 0;
   let merged = 0;
 
@@ -216,6 +279,7 @@ export function reconcile(
 
     if (match) {
       merged += 1;
+      const before = signature(match);
       applyMentions(match, mentions);
       // The extractor NEVER overwrites a status, an owner or a due date that is
       // already recorded. A person ticking something off on the screen must
@@ -223,6 +287,11 @@ export function reconcile(
       if (!match.ownerName && lead.ownerName) match.ownerName = lead.ownerName;
       if (!match.dueAt && lead.dueAt) match.dueAt = lead.dueAt;
       if (!match.detail && lead.detail) match.detail = lead.detail;
+      // Only a row that ACTUALLY changed is written. Marking every match dirty
+      // would make a repeat run rewrite the corpus for no reason, which is the
+      // difference between "running it twice is a no-op" as a claim and as a
+      // fact you can watch on the network tab.
+      if (signature(match) !== before) touched.add(match.externalId);
       continue;
     }
 
@@ -255,9 +324,10 @@ export function reconcile(
     };
     applyMentions(fresh, mentions);
     out.push(fresh);
+    touched.add(fresh.externalId);
   }
 
-  return { commitments: out, created, merged };
+  return { commitments: out, created, merged, touched: [...touched] };
 }
 
 /**
@@ -282,6 +352,22 @@ function applyMentions(commitment: Commitment, mentions: CommitmentMention[]) {
 
 const mentionKey = (mention: CommitmentMention) => `${mention.documentId}::${mention.originKey}`;
 
+/** Everything a re-run could legitimately change, as one comparable string. */
+function signature(commitment: Commitment): string {
+  return [
+    commitment.title,
+    commitment.detail,
+    commitment.status,
+    commitment.statusReason ?? '',
+    commitment.ownerName ?? '',
+    commitment.dueAt ?? '',
+    commitment.resolvedBy ?? '',
+    commitment.firstSeenAt ?? '',
+    commitment.lastSeenAt ?? '',
+    commitment.mentions.map((mention) => `${mentionKey(mention)}|${mention.quote}`).join('~'),
+  ].join('\u0000');
+}
+
 function toMention(draft: CommitmentDraft, document: SourceDocument): CommitmentMention {
   return {
     documentId: draft.documentId,
@@ -295,6 +381,7 @@ function toMention(draft: CommitmentDraft, document: SourceDocument): Commitment
 function identityOf(commitment: Commitment): CommitmentIdentity {
   return {
     subjectKey: commitment.subjectKey,
+    subjectIsIdentifier: MATTER_HYPHENATED.test(commitment.subjectKey) || MATTER_BARE.test(commitment.subjectKey),
     actionKey: commitment.actionKey,
     actionTokens: commitment.actionKey ? commitment.actionKey.split('.') : [],
     dedupeKey: commitment.dedupeKey,
@@ -307,6 +394,8 @@ function identityOf(commitment: Commitment): CommitmentIdentity {
 
 export interface ResolutionResult {
   commitments: Commitment[];
+  /** The `externalId`s this pass closed. Same reasoning as `ReconcileResult`. */
+  touched: string[];
   resolved: number;
   needsReview: Array<{ commitmentId: string; documentId: string; quote: string }>;
 }
@@ -335,6 +424,7 @@ export function applyResolutions(
   const byId = new Map(context.documents.map((document) => [document.id, document]));
   const out = commitments.map(cloneCommitment);
   const needsReview: ResolutionResult['needsReview'] = [];
+  const touched = new Set<string>();
   let resolved = 0;
 
   for (const claim of claims) {
@@ -347,13 +437,15 @@ export function applyResolutions(
     for (const commitment of out) {
       if (commitment.status !== 'open' && commitment.status !== 'blocked') continue;
       if (at && commitment.firstSeenAt && commitment.firstSeenAt > at) continue;
+      const other = identityOf(commitment);
+      // Two IDENTIFIERS that disagree are different things, and no amount of
+      // shared wording makes them the same one. Disagreeing prose is only weak
+      // evidence and raises the bar instead of vetoing — see `subjectsConflict`.
+      if (subjectsConflict(identity, other)) continue;
       const anchored = Boolean(identity.subjectKey) && identity.subjectKey === commitment.subjectKey;
-      // Two subjects that both exist and disagree are different things, and no
-      // amount of shared wording makes them the same one.
-      if (identity.subjectKey && commitment.subjectKey && !anchored) continue;
       const candidate = {
         commitment,
-        score: jaccard(identity.actionTokens, identityOf(commitment).actionTokens),
+        score: jaccard(identity.actionTokens, other.actionTokens),
         bar: anchored ? RESOLVE_THRESHOLD : UNANCHORED_RESOLVE_THRESHOLD,
       };
       if (!best || candidate.score > best.score) best = candidate;
@@ -365,6 +457,7 @@ export function applyResolutions(
       best.commitment.statusReason = claim.quote || claim.statement;
       best.commitment.resolvedBy = claim.documentId;
       best.commitment.resolvedAt = at;
+      touched.add(best.commitment.externalId);
       resolved += 1;
     } else {
       // ONE review row per claim — its single best candidate. Listing every open
@@ -377,7 +470,7 @@ export function applyResolutions(
     }
   }
 
-  return { commitments: out, resolved, needsReview };
+  return { commitments: out, touched: [...touched], resolved, needsReview };
 }
 
 // ---------------------------------------------------------------------------

package/src/corpus/extract.ts

@@ -289,11 +289,35 @@ export async function extractCommitments(
     };
   }
 
+  const allowedIds = new Set(documents.map((document) => document.id));
+
   // 3. rules first, so the model can be told what not to repeat.
   const ruleDrafts = documents.flatMap(rulesExtract);
   const foundByDocument = new Map<string, string[]>();
-  for (const draft of ruleDrafts) {
-    foundByDocument.set(draft.documentId, [...(foundByDocument.get(draft.documentId) ?? []), draft.title]);
+  const note = (documentId: string, title: string) => {
+    foundByDocument.set(documentId, [...(foundByDocument.get(documentId) ?? []), title]);
+  };
+  for (const draft of ruleDrafts) note(draft.documentId, draft.title);
+
+  // WHAT IS ALREADY TRACKED IS ALSO "ALREADY FOUND". Read BEFORE the model call,
+  // not after, so a second run over the same document tells the model what it
+  // produced last time.
+  //
+  // This is the STRUCTURAL half of idempotency and it exists because the fuzzy
+  // half was not enough: on the live tandem tenant a re-run over four documents
+  // created sixteen duplicates purely because the model reworded its own output.
+  // Matching after the fact is a safety net; not asking the model to invent the
+  // same item twice is the fix.
+  const existing = corpus.listCommitments
+    ? await corpus
+        .listCommitments({ pageSize: 500 })
+        .then((page) => page.results)
+        .catch(() => [] as Commitment[])
+    : [];
+  for (const commitment of existing) {
+    for (const mention of commitment.mentions) {
+      if (allowedIds.has(mention.documentId)) note(mention.documentId, commitment.title);
+    }
   }
 
   // 4. ONE call, every document, ids echoed.
@@ -315,27 +339,26 @@ export async function extractCommitments(
     draft = { items: [], resolutions: [] };
   }
 
-  const allowed = new Set(documents.map((document) => document.id));
-  const modelDrafts = normaliseItems(draft.items ?? [], allowed);
-  const claims = normaliseResolutions(draft.resolutions ?? [], allowed);
+  const modelDrafts = normaliseItems(draft.items ?? [], allowedIds);
+  const claims = normaliseResolutions(draft.resolutions ?? [], allowedIds);
 
   // 5. identity, dedupe within the run, then against what is stored.
-  const existing = corpus.listCommitments
-    ? await corpus
-        .listCommitments({ pageSize: 500 })
-        .then((page) => page.results)
-        .catch(() => [] as Commitment[])
-    : [];
-
   const groups = mergeDrafts([...ruleDrafts, ...modelDrafts]);
   const reconciled = reconcile(existing, groups, { scope: request.scope, documents });
 
   // 6. resolutions, against the documents the scope guard approved and nothing else.
   const resolutions = applyResolutions(reconciled.commitments, claims, { documents });
 
-  // 7. write. ONE call carrying every row, keyed on `external_id`.
-  if (corpus.saveCommitments) {
-    await corpus.saveCommitments(resolutions.commitments);
+  // 7. write WHAT CHANGED, keyed on `external_id`.
+  //
+  // Not the whole corpus. The tenant has no bulk upsert (a list body is
+  // rejected — see bd startsim-768w.38.12), so every row is a request, and
+  // rewriting untouched rows makes a one-document slice cost one request per row
+  // in the workspace. On the live tenant that took a single-document run past
+  // the 60-second gateway once there were ninety-eight of them.
+  const changed = new Set([...reconciled.touched, ...resolutions.touched]);
+  if (corpus.saveCommitments && changed.size) {
+    await corpus.saveCommitments(resolutions.commitments.filter((item) => changed.has(item.externalId)));
   }
 
   return {