@startsimpli/integrations 0.2.2 → 0.2.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@startsimpli/integrations",
-  "version": "0.2.2",
+  "version": "0.2.5",
   "description": "Provider-agnostic types, API clients, hooks, and UI primitives for email + calendar integrations across StartSimpli apps.",
   "main": "./src/index.ts",
   "types": "./src/index.ts",

package/src/__tests__/corpus-extract.test.ts

@@ -198,6 +198,60 @@ describe('idempotence: running it twice changes nothing', () => {
     );
   });
 
+  it('survives the MODEL rewording its own output between runs', () => {
+    // THE PRODUCTION BUG, pinned. On the live tandem tenant a second extraction
+    // over the same four documents created SIXTEEN duplicate rows: the model
+    // reworded its own `subject` ("ketamine regimen" -> "ketamine treatment")
+    // while emitting a byte-identical title, and a disagreeing subject was
+    // vetoing the match. Prose subjects are now weak evidence, not a veto.
+    const first = reconcile([], mergeDrafts([
+      { ...draft('m1', 'Find a sustainable ketamine regimen', '2026-08-01'), subject: 'ketamine regimen', extractor: 'model' as const },
+    ]), { scope: TANDEM, documents: [documentFor('m1', '2026-08-01')] });
+
+    const second = reconcile(first.commitments, mergeDrafts([
+      { ...draft('m1', 'Find a sustainable ketamine regimen', '2026-08-01'), subject: "Quinn's ketamine treatment", extractor: 'model' as const },
+    ]), { scope: TANDEM, documents: [documentFor('m1', '2026-08-01')] });
+
+    expect(second.created).toBe(0);
+    expect(second.commitments).toHaveLength(1);
+  });
+
+  it('still refuses to merge two MATTER NUMBERS that disagree, however alike the words', () => {
+    // The other side of the same rule. An identifier is authoritative: "chase
+    // for comments on 2148" and "...on 2198" have identical action stems and are
+    // two different obligations. Merging them deletes work.
+    const { commitments } = reconcile([], mergeDrafts([
+      draft('m1', 'Chase for comments on 2148', '2026-08-01'),
+      draft('m1', 'Chase for comments on 2198', '2026-08-01'),
+    ]), { scope: TANDEM, documents: [documentFor('m1', '2026-08-01')] });
+    expect(commitments).toHaveLength(2);
+  });
+
+  it('tells the model what is ALREADY TRACKED for a document, not just what the rules found', async () => {
+    // The structural half of idempotency: the fuzzy matcher is a safety net, and
+    // not asking the model to invent the same item twice is the actual fix.
+    const corpus = createFixtureCorpusApi({ scope: TANDEM, documents: FIXTURE_TRACKED_DOCUMENTS });
+    const tracked: Commitment[] = [];
+    (corpus as { listCommitments?: unknown }).listCommitments = async () => ({
+      count: tracked.length, next: null, previous: null, results: tracked,
+    });
+    (corpus as { saveCommitments?: unknown }).saveCommitments = async (items: Commitment[]) => {
+      tracked.splice(0, tracked.length, ...items);
+    };
+
+    const extractor = createFixtureExtractor();
+    const spy = vi.spyOn(extractor, 'extract');
+
+    await extractCommitments({ scope: TANDEM }, { corpus, extractor });
+    const firstAsk = spy.mock.calls[0][0].documents.flatMap((d) => d.alreadyFound).length;
+
+    await extractCommitments({ scope: TANDEM }, { corpus, extractor });
+    const secondAsk = spy.mock.calls[1][0].documents.flatMap((d) => d.alreadyFound).length;
+
+    expect(tracked.length).toBeGreaterThan(0);
+    expect(secondAsk).toBeGreaterThan(firstAsk);
+  });
+
   it('keeps a human’s edit when the extractor runs again over the same source', () => {
     const drafts = rulesExtract(FIXTURE_TRACKED_DOCUMENTS[0]);
     const first = reconcile([], mergeDrafts(drafts), { scope: TANDEM, documents: FIXTURE_TRACKED_DOCUMENTS });
@@ -431,6 +485,28 @@ describe('extractCommitments', () => {
     expect(result.created).toBeGreaterThan(0);
   });
 
+  it('slices a sweep so one call fits the gateway, and says where to resume', async () => {
+    // A deployed tenant closes a request at 60s and one call over the whole
+    // corpus does not finish inside it (measured on tandem — 504). The sweep is
+    // therefore BATCHED INTO CALLS THAT FIT, never one call per document.
+    const corpus = createFixtureCorpusApi({ scope: TANDEM, documents: FIXTURE_TRACKED_DOCUMENTS });
+    const extractor = createFixtureExtractor();
+    const spy = vi.spyOn(extractor, 'extract');
+
+    const first = await extractCommitments({ scope: TANDEM, limit: 2 }, { corpus, extractor });
+    expect(first.documentsProcessed).toBe(2);
+    expect(first.totalDocuments).toBe(FIXTURE_TRACKED_DOCUMENTS.length);
+    expect(first.nextOffset).toBe(2);
+    expect(spy.mock.calls[0][0].documents).toHaveLength(2); // still N per call
+
+    const second = await extractCommitments(
+      { scope: TANDEM, offset: first.nextOffset!, limit: 2 },
+      { corpus, extractor },
+    );
+    expect(second.documentsProcessed).toBe(1);
+    expect(second.nextOffset).toBeNull();
+  });
+
   it('refuses outright when the session carries no workspace claim', async () => {
     const corpus = createFixtureCorpusApi({ leakScope: true, documents: FIXTURE_TRACKED_DOCUMENTS });
     const listDocuments = vi.spyOn(corpus, 'listDocuments');

package/src/__tests__/corpus-tracker.test.tsx

@@ -196,6 +196,8 @@ describe('TrackerPage — what is outstanding', () => {
 describe('TrackerPage — running the extraction', () => {
   const RESULT = {
     documentsProcessed: 16,
+    totalDocuments: 16,
+    nextOffset: null,
     fromRules: 24,
     fromModel: 11,
     created: 30,

package/src/corpus/commitments.ts

@@ -61,6 +61,23 @@ export const EXTRACTOR_VERSION = 'rules+model@1';
  */
 const MERGE_THRESHOLD = 0.5;
 
+/**
+ * The bar when the two statements do NOT agree on a subject — either because
+ * one of them has none, or because they disagree.
+ *
+ * A DISAGREEING SUBJECT MUST NOT VETO A MATCH, and that was a real production
+ * bug: on the live tandem tenant a second extraction run over the same four
+ * documents created sixteen duplicate rows, because the model had rephrased its
+ * own `subject` between runs ("ketamine regimen" vs "ketamine treatment") and
+ * the veto threw away pairs whose TITLES were byte-identical. Measured across
+ * the resulting 98 rows: five same-document duplicate pairs sat at 0.5 or above
+ * on title overlap, and ZERO cross-document pairs reached even 0.5 — so an
+ * unanchored bar of 0.7 catches the rewording with no measured false merge.
+ *
+ * An agreeing subject still buys the lower bar, because it is real evidence.
+ */
+const UNANCHORED_MERGE_THRESHOLD = 0.7;
+
 /**
  * The bar a resolution has to clear before it CLOSES something, which is higher
  * than the bar for merging because the failure is worse. A wrong merge shows one
@@ -100,6 +117,19 @@ const MATTER_BARE = /\b(?!(?:19|20)\d{2}\b)\d{4}\b/;
 
 export interface CommitmentIdentity {
   subjectKey: string;
+  /**
+   * True when the subject is an IDENTIFIER (a matter number) rather than prose.
+   *
+   * The distinction is load-bearing. Two identifiers that disagree are two
+   * different things and no amount of shared wording changes that — "chase for
+   * comments on 2148" and "chase for comments on 2198" are two obligations whose
+   * action stems are identical. Two PROSE subjects that disagree mean much less:
+   * measured on the live tenant, the model described one commitment as being
+   * about "ketamine regimen" on one run and "ketamine treatment" on the next,
+   * while emitting a byte-identical title. So an identifier vetoes; prose does
+   * not, it only raises the bar.
+   */
+  subjectIsIdentifier: boolean;
   actionKey: string;
   actionTokens: string[];
   dedupeKey: string;
@@ -128,20 +158,43 @@ export function commitmentIdentity(input: {
   const subjectTokens = matter ? [matter] : unique(tokenize(subject));
   const subjectKey = matter ? matter : subjectTokens.slice().sort().join('.');
 
-  const drop = new Set(subjectTokens);
+  // ONLY AN IDENTIFIER IS REMOVED FROM THE ACTION. Removing prose subject words
+  // too would couple the two halves of the key: when the model reworded a
+  // subject from "ketamine regimen" to "Quinn's ketamine treatment", a
+  // byte-identical title produced two different action sets and the row
+  // duplicated. The matter number is noise in the action ("chase 2148" is a
+  // chase); the prose words are part of what the title says.
+  const drop = new Set(matter ? [matter] : []);
   const actionTokens = unique(
     tokenize(title).filter((token) => !drop.has(token) && !ACTION_NOISE.has(token)),
   );
   const actionKey = actionTokens.slice().sort().join('.');
 
-  return { subjectKey, actionKey, actionTokens, dedupeKey: hash(`${subjectKey}|${actionKey}`) };
+  return {
+    subjectKey,
+    subjectIsIdentifier: Boolean(matter),
+    actionKey,
+    actionTokens,
+    dedupeKey: hash(`${subjectKey}|${actionKey}`),
+  };
 }
 
 /** Do two identities describe one commitment? Tier 1, then Tier 2. */
 export function isSameCommitment(a: CommitmentIdentity, b: CommitmentIdentity): boolean {
   if (a.dedupeKey === b.dedupeKey) return true;
-  if (!a.subjectKey || a.subjectKey !== b.subjectKey) return false;
-  return jaccard(a.actionTokens, b.actionTokens) >= MERGE_THRESHOLD;
+  if (subjectsConflict(a, b)) return false;
+  const anchored = Boolean(a.subjectKey) && a.subjectKey === b.subjectKey;
+  const bar = anchored ? MERGE_THRESHOLD : UNANCHORED_MERGE_THRESHOLD;
+  return jaccard(a.actionTokens, b.actionTokens) >= bar;
+}
+
+/** Two IDENTIFIERS that disagree. The one veto worth having. */
+function subjectsConflict(a: CommitmentIdentity, b: CommitmentIdentity): boolean {
+  return (
+    a.subjectIsIdentifier &&
+    b.subjectIsIdentifier &&
+    a.subjectKey !== b.subjectKey
+  );
 }
 
 // ---------------------------------------------------------------------------
@@ -295,6 +348,7 @@ function toMention(draft: CommitmentDraft, document: SourceDocument): Commitment
 function identityOf(commitment: Commitment): CommitmentIdentity {
   return {
     subjectKey: commitment.subjectKey,
+    subjectIsIdentifier: MATTER_HYPHENATED.test(commitment.subjectKey) || MATTER_BARE.test(commitment.subjectKey),
     actionKey: commitment.actionKey,
     actionTokens: commitment.actionKey ? commitment.actionKey.split('.') : [],
     dedupeKey: commitment.dedupeKey,
@@ -347,13 +401,15 @@ export function applyResolutions(
     for (const commitment of out) {
       if (commitment.status !== 'open' && commitment.status !== 'blocked') continue;
       if (at && commitment.firstSeenAt && commitment.firstSeenAt > at) continue;
+      const other = identityOf(commitment);
+      // Two IDENTIFIERS that disagree are different things, and no amount of
+      // shared wording makes them the same one. Disagreeing prose is only weak
+      // evidence and raises the bar instead of vetoing — see `subjectsConflict`.
+      if (subjectsConflict(identity, other)) continue;
       const anchored = Boolean(identity.subjectKey) && identity.subjectKey === commitment.subjectKey;
-      // Two subjects that both exist and disagree are different things, and no
-      // amount of shared wording makes them the same one.
-      if (identity.subjectKey && commitment.subjectKey && !anchored) continue;
       const candidate = {
         commitment,
-        score: jaccard(identity.actionTokens, identityOf(commitment).actionTokens),
+        score: jaccard(identity.actionTokens, other.actionTokens),
         bar: anchored ? RESOLVE_THRESHOLD : UNANCHORED_RESOLVE_THRESHOLD,
       };
       if (!best || candidate.score > best.score) best = candidate;

package/src/corpus/extract.ts

@@ -235,6 +235,17 @@ export interface ExtractRequest {
   scope: CorpusScope;
   /** Narrow the run to specific records. Omit to sweep the corpus. */
   documentIds?: string[];
+  /**
+   * Where this call starts in the in-scope corpus, and how many documents it
+   * may take. Omit both to do the whole thing in one model call.
+   *
+   * This exists for a deployment fact, not a design preference: the tandem ALB
+   * closes a request at 60 seconds, and one batched call over all sixteen real
+   * documents does not finish inside it (measured — 504). Slicing keeps every
+   * call batched over N documents while letting the caller walk the corpus.
+   */
+  offset?: number;
+  limit?: number;
 }
 
 export interface ExtractDeps {
@@ -258,19 +269,55 @@ export async function extractCommitments(
 
   const { documents: candidates, matched } = await readDocuments(corpus, documentLimit, maxPages);
   const { kept, dropped } = enforceScope(candidates, request.scope);
-  const documents = request.documentIds?.length
+  const inScope = request.documentIds?.length
     ? kept.filter((document) => request.documentIds!.includes(document.id))
     : kept;
 
+  // The slice this call handles. `listDocuments` orders stably (the tenant's
+  // (org_id, entity_type, -created_at) index), so an offset means the same thing
+  // between calls of one sweep.
+  const offset = Math.max(0, request.offset ?? 0);
+  const documents = request.limit ? inScope.slice(offset, offset + request.limit) : inScope.slice(offset);
+  const consumed = offset + documents.length;
+  const nextOffset = consumed < inScope.length ? consumed : null;
+
   if (documents.length === 0) {
-    return { ...empty(extractor, 'nothing in scope'), droppedOutOfScope: dropped.length };
+    return {
+      ...empty(extractor, 'nothing in scope'),
+      totalDocuments: inScope.length,
+      droppedOutOfScope: dropped.length,
+    };
   }
 
+  const allowedIds = new Set(documents.map((document) => document.id));
+
   // 3. rules first, so the model can be told what not to repeat.
   const ruleDrafts = documents.flatMap(rulesExtract);
   const foundByDocument = new Map<string, string[]>();
-  for (const draft of ruleDrafts) {
-    foundByDocument.set(draft.documentId, [...(foundByDocument.get(draft.documentId) ?? []), draft.title]);
+  const note = (documentId: string, title: string) => {
+    foundByDocument.set(documentId, [...(foundByDocument.get(documentId) ?? []), title]);
+  };
+  for (const draft of ruleDrafts) note(draft.documentId, draft.title);
+
+  // WHAT IS ALREADY TRACKED IS ALSO "ALREADY FOUND". Read BEFORE the model call,
+  // not after, so a second run over the same document tells the model what it
+  // produced last time.
+  //
+  // This is the STRUCTURAL half of idempotency and it exists because the fuzzy
+  // half was not enough: on the live tandem tenant a re-run over four documents
+  // created sixteen duplicates purely because the model reworded its own output.
+  // Matching after the fact is a safety net; not asking the model to invent the
+  // same item twice is the fix.
+  const existing = corpus.listCommitments
+    ? await corpus
+        .listCommitments({ pageSize: 500 })
+        .then((page) => page.results)
+        .catch(() => [] as Commitment[])
+    : [];
+  for (const commitment of existing) {
+    for (const mention of commitment.mentions) {
+      if (allowedIds.has(mention.documentId)) note(mention.documentId, commitment.title);
+    }
   }
 
   // 4. ONE call, every document, ids echoed.
@@ -292,18 +339,10 @@ export async function extractCommitments(
     draft = { items: [], resolutions: [] };
   }
 
-  const allowed = new Set(documents.map((document) => document.id));
-  const modelDrafts = normaliseItems(draft.items ?? [], allowed);
-  const claims = normaliseResolutions(draft.resolutions ?? [], allowed);
+  const modelDrafts = normaliseItems(draft.items ?? [], allowedIds);
+  const claims = normaliseResolutions(draft.resolutions ?? [], allowedIds);
 
   // 5. identity, dedupe within the run, then against what is stored.
-  const existing = corpus.listCommitments
-    ? await corpus
-        .listCommitments({ pageSize: 500 })
-        .then((page) => page.results)
-        .catch(() => [] as Commitment[])
-    : [];
-
   const groups = mergeDrafts([...ruleDrafts, ...modelDrafts]);
   const reconciled = reconcile(existing, groups, { scope: request.scope, documents });
 
@@ -317,6 +356,8 @@ export async function extractCommitments(
 
   return {
     documentsProcessed: documents.length,
+    totalDocuments: inScope.length,
+    nextOffset,
     fromRules: ruleDrafts.length,
     fromModel: modelDrafts.length,
     created: reconciled.created,
@@ -325,8 +366,8 @@ export async function extractCommitments(
     needsReview: resolutions.needsReview,
     droppedOutOfScope: dropped.length,
     provider: extractor.describe(),
-    ...(matched > candidates.length ? { } : {}),
   };
+  void matched;
 }
 
 /**
@@ -424,6 +465,8 @@ function empty(extractor: CorpusExtractorLlm, _why: string): ExtractionResult {
   void _why;
   return {
     documentsProcessed: 0,
+    totalDocuments: 0,
+    nextOffset: null,
     fromRules: 0,
     fromModel: 0,
     created: 0,

package/src/corpus/types.ts

@@ -565,12 +565,30 @@ export interface CommitmentTrackerApi extends CommitmentApi {
    * Run extraction over documents that have not been processed. Optional: a
    * read-only deployment omits it and the control does not render.
    */
-  extract?(input: { scope: CorpusScope; documentIds?: string[] }): Promise<ExtractionResult>;
+  extract?(input: {
+    scope: CorpusScope;
+    documentIds?: string[];
+    /** Resume point for a batched sweep. See `ExtractionResult.nextOffset`. */
+    offset?: number;
+    limit?: number;
+  }): Promise<ExtractionResult>;
 }
 
 export interface ExtractionResult {
-  /** Documents read and sent to the extractor. */
+  /** Documents read and sent to the extractor IN THIS CALL. */
   documentsProcessed: number;
+  /** Documents in scope altogether, so a partial sweep can say what it is of. */
+  totalDocuments: number;
+  /**
+   * Where a resumed sweep should start, or `null` when the corpus is done.
+   *
+   * A run is RESUMABLE because it has to be: a deployed tenant sits behind a
+   * gateway with a request timeout (60s on the tandem ALB, measured — a single
+   * call over sixteen documents 504s), and one model call per document is
+   * forbidden. So the sweep is batched into calls that fit, and the caller walks
+   * the offsets. Still N documents per model call; just not always all of them.
+   */
+  nextOffset: number | null;
   /** Items the vendor's own structure produced, with no model involved. */
   fromRules: number;
   /** Items only the model found. */