@startsimpli/integrations 0.1.6 → 0.2.2

package/src/corpus/commitments.ts

@@ -0,0 +1,416 @@
+/**
+ * IDENTITY, DEDUPE AND RESOLUTION — the crux of bd startsim-768w.38.25.
+ *
+ * Extraction is the interesting problem; identity is the one that decides
+ * whether the product is usable. A pipeline that produces a perfect list on
+ * Monday and a doubled list on Tuesday is not a system of record, and a list
+ * that shows one commitment three times because three meetings mentioned it is
+ * the thing a user abandons on first contact.
+ *
+ * THE IDENTITY PROBLEM, STATED HONESTLY. A commitment's identity cannot be a
+ * string a model emitted, because the model will emit a different string next
+ * time. It cannot be the document it came from either, because the whole point
+ * is that one commitment spans several meetings. So identity is RESOLVED, not
+ * computed, and it is resolved in two tiers:
+ *
+ *   TIER 1 — the exact key. `dedupeKey = hash(subjectKey | actionKey)`, where
+ *   both halves are normalised stems and the subject prefers a matter number
+ *   over prose. This is the fast path and it is what the tenant's `external_id`
+ *   upsert keys on, so two concurrent runs converge on one row instead of
+ *   racing.
+ *
+ *   TIER 2 — the near match. Same non-empty `subjectKey` plus enough overlap in
+ *   the action stems. This is what actually carries the cross-meeting dedupe,
+ *   because "set a drafting deadline for 2040-01" and "deadline still needs
+ *   setting on 2040-01" are the same obligation and will never hash the same.
+ *
+ * WHY NOT ASK A MODEL TO DEDUPE. Because the user asked for deterministic, and
+ * they were right to: a matcher whose answer changes between runs turns the
+ * item id — the thing a link, a citation and a human's tick-box all hang off —
+ * into a coin flip. Every rule below is arithmetic over stems, so a merge can be
+ * explained ("both are about 2040-01 and share set/deadline") and argued with.
+ *
+ * MENTIONS ARE THE UNIT OF IDEMPOTENCY, and this is the single most important
+ * line in the file. A mention is keyed by (documentId, originKey). Re-running
+ * extraction over a document REPLACES that document's mentions; it can never
+ * append a second copy. That is what makes a second pass over the same corpus a
+ * no-op rather than a doubling, and it is why `mention_count` is trustworthy
+ * enough to sort by.
+ */
+import { jaccard, tokenize } from './text';
+import type {
+  Commitment,
+  CommitmentDraft,
+  CommitmentMention,
+  CorpusScope,
+  ResolutionClaim,
+  SourceDocument,
+} from './types';
+
+/** Bumped when a change to this file or the prompt would re-derive items differently. */
+export const EXTRACTOR_VERSION = 'rules+model@1';
+
+/**
+ * How much of the action has to overlap before two statements are one
+ * commitment, GIVEN that they already agree on the subject.
+ *
+ * 0.5 is where "set a drafting deadline" meets "deadline still needs setting"
+ * and where "set a drafting deadline" does NOT meet "send the filing receipt".
+ * Both of those are real pairs from the live corpus, and the gap between them is
+ * wide, which is the only reason a single constant is defensible here.
+ */
+const MERGE_THRESHOLD = 0.5;
+
+/**
+ * The bar a resolution has to clear before it CLOSES something, which is higher
+ * than the bar for merging because the failure is worse. A wrong merge shows one
+ * item where there were two; a wrong close shows nothing where there was work,
+ * and an empty list has no symptom.
+ */
+const RESOLVE_THRESHOLD = 0.45;
+
+/** With no subject to anchor on, the wording has to carry the whole claim. */
+const UNANCHORED_RESOLVE_THRESHOLD = 0.7;
+
+/**
+ * Words that are about the FORM of a commitment rather than its content.
+ * Removing them stops "set a deadline on the matter" and "set a deadline on the
+ * item" from looking like different obligations.
+ */
+const ACTION_NOISE = new Set(['matter', 'item', 'thing', 'still', 'also', 'just', 'again', 'per', 'via']);
+
+// ---------------------------------------------------------------------------
+//  identity
+// ---------------------------------------------------------------------------
+
+/**
+ * A matter/case number, which is by far the strongest identity signal this
+ * corpus has: the live tenant's records are full of `2040-01`, `2148`, `2198`,
+ * `2205`, `2217`, and two statements carrying the same one are almost always
+ * about the same thing.
+ *
+ * The hyphenated form is tried FIRST, because the bare-four-digit rule has to
+ * exclude 19xx/20xx to avoid reading a year as a matter number — and `2040-01`
+ * would otherwise be thrown away by that exclusion. A tenant whose matters are
+ * numbered in the 2000s loses this signal and falls back to prose; that is a
+ * stated limit, not a silent one.
+ */
+const MATTER_HYPHENATED = /\b\d{4}-\d{2}\b/;
+const MATTER_BARE = /\b(?!(?:19|20)\d{2}\b)\d{4}\b/;
+
+export interface CommitmentIdentity {
+  subjectKey: string;
+  actionKey: string;
+  actionTokens: string[];
+  dedupeKey: string;
+}
+
+/**
+ * What a commitment is ABOUT and what is being DONE, normalised.
+ *
+ * The subject is taken from an identifier when the text contains one, then from
+ * the extractor's stated subject, and only then from nothing — an empty subject
+ * is honest and simply means Tier 2 has no anchor and will demand a much closer
+ * wording match.
+ */
+export function commitmentIdentity(input: {
+  title: string;
+  subject?: string;
+  detail?: string;
+}): CommitmentIdentity {
+  const title = input.title ?? '';
+  const subject = input.subject ?? '';
+  const haystack = `${subject} ${title} ${input.detail ?? ''}`;
+
+  const matter =
+    MATTER_HYPHENATED.exec(haystack)?.[0] ?? MATTER_BARE.exec(haystack)?.[0] ?? '';
+
+  const subjectTokens = matter ? [matter] : unique(tokenize(subject));
+  const subjectKey = matter ? matter : subjectTokens.slice().sort().join('.');
+
+  const drop = new Set(subjectTokens);
+  const actionTokens = unique(
+    tokenize(title).filter((token) => !drop.has(token) && !ACTION_NOISE.has(token)),
+  );
+  const actionKey = actionTokens.slice().sort().join('.');
+
+  return { subjectKey, actionKey, actionTokens, dedupeKey: hash(`${subjectKey}|${actionKey}`) };
+}
+
+/** Do two identities describe one commitment? Tier 1, then Tier 2. */
+export function isSameCommitment(a: CommitmentIdentity, b: CommitmentIdentity): boolean {
+  if (a.dedupeKey === b.dedupeKey) return true;
+  if (!a.subjectKey || a.subjectKey !== b.subjectKey) return false;
+  return jaccard(a.actionTokens, b.actionTokens) >= MERGE_THRESHOLD;
+}
+
+// ---------------------------------------------------------------------------
+//  merging drafts within one run
+// ---------------------------------------------------------------------------
+
+export interface MergedDraft {
+  identity: CommitmentIdentity;
+  /** Every statement that turned out to be this one commitment. */
+  drafts: CommitmentDraft[];
+}
+
+/**
+ * Collapse a run's drafts into one entry per commitment.
+ *
+ * Grouped against the FIRST draft in each group rather than against the group's
+ * accumulated union: a union grows every time it absorbs something and ends up
+ * matching everything, which is how a dedupe quietly turns into a single row
+ * called "do the thing".
+ */
+export function mergeDrafts(drafts: CommitmentDraft[]): MergedDraft[] {
+  const groups: MergedDraft[] = [];
+  for (const draft of drafts) {
+    const identity = commitmentIdentity(draft);
+    const existing = groups.find((group) => isSameCommitment(group.identity, identity));
+    if (existing) existing.drafts.push(draft);
+    else groups.push({ identity, drafts: [draft] });
+  }
+  return groups;
+}
+
+// ---------------------------------------------------------------------------
+//  reconciling a run against what is already stored
+// ---------------------------------------------------------------------------
+
+export interface ReconcileContext {
+  scope: CorpusScope;
+  /**
+   * The IN-SCOPE documents this run is allowed to cite. A draft naming anything
+   * else contributes nothing: a document id is the one field an extractor could
+   * hallucinate into a cross-workspace reference, and it is checked against the
+   * set the scope guard already approved rather than trusted.
+   */
+  documents: SourceDocument[];
+  now?: number;
+}
+
+export interface ReconcileResult {
+  commitments: Commitment[];
+  created: number;
+  merged: number;
+}
+
+export function reconcile(
+  existing: Commitment[],
+  groups: MergedDraft[],
+  context: ReconcileContext,
+): ReconcileResult {
+  const byId = new Map(context.documents.map((document) => [document.id, document]));
+  const out = existing.map(cloneCommitment);
+  let created = 0;
+  let merged = 0;
+
+  for (const group of groups) {
+    const mentions = group.drafts
+      .filter((draft) => byId.has(draft.documentId))
+      .map((draft) => toMention(draft, byId.get(draft.documentId)!));
+    if (mentions.length === 0) continue;
+
+    const lead = group.drafts.find((draft) => byId.has(draft.documentId))!;
+    const match = out.find((commitment) => isSameCommitment(identityOf(commitment), group.identity));
+
+    if (match) {
+      merged += 1;
+      applyMentions(match, mentions);
+      // The extractor NEVER overwrites a status, an owner or a due date that is
+      // already recorded. A person ticking something off on the screen must
+      // survive the next run, or the screen is a suggestion box.
+      if (!match.ownerName && lead.ownerName) match.ownerName = lead.ownerName;
+      if (!match.dueAt && lead.dueAt) match.dueAt = lead.dueAt;
+      if (!match.detail && lead.detail) match.detail = lead.detail;
+      continue;
+    }
+
+    created += 1;
+    const document = byId.get(lead.documentId)!;
+    const fresh: Commitment = {
+      id: `commit:${group.identity.dedupeKey}`,
+      externalId: `commit:${group.identity.dedupeKey}`,
+      dedupeKey: group.identity.dedupeKey,
+      subtype: lead.subtype || 'action',
+      title: lead.title,
+      detail: lead.detail ?? '',
+      status: lead.status || 'open',
+      ...(lead.ownerName ? { ownerName: lead.ownerName } : {}),
+      ...(lead.subject ? { subject: lead.subject } : {}),
+      subjectKey: group.identity.subjectKey,
+      actionKey: group.identity.actionKey,
+      ...(lead.dueAt ? { dueAt: lead.dueAt } : {}),
+      mentions: [],
+      firstSeenAt: null,
+      lastSeenAt: null,
+      extractorVersion: EXTRACTOR_VERSION,
+      source: document.source,
+      sourceLabel: document.sourceLabel,
+      // The commitment inherits the CALLER's scope, not the document's. They are
+      // the same by construction here — the documents were already filtered —
+      // but writing the caller's scope means a row can never be minted carrying
+      // a workspace the asking principal does not hold.
+      scope: context.scope,
+    };
+    applyMentions(fresh, mentions);
+    out.push(fresh);
+  }
+
+  return { commitments: out, created, merged };
+}
+
+/**
+ * Attach this run's mentions, REPLACING any the same document contributed before.
+ * See the note at the top of the file: this is the whole of idempotency.
+ */
+function applyMentions(commitment: Commitment, mentions: CommitmentMention[]) {
+  const incoming = new Set(mentions.map(mentionKey));
+  const kept = commitment.mentions.filter((mention) => !incoming.has(mentionKey(mention)));
+  // Also drop any earlier mention from the same DOCUMENT whose origin key has
+  // changed — a re-worded model output must not leave its previous self behind.
+  const documents = new Set(mentions.map((mention) => mention.documentId));
+  commitment.mentions = [
+    ...kept.filter((mention) => !documents.has(mention.documentId)),
+    ...mentions,
+  ].sort((a, b) => (a.occurredAt ?? '').localeCompare(b.occurredAt ?? ''));
+
+  const dates = commitment.mentions.map((mention) => mention.occurredAt).filter(Boolean) as string[];
+  commitment.firstSeenAt = dates[0] ?? null;
+  commitment.lastSeenAt = dates[dates.length - 1] ?? null;
+}
+
+const mentionKey = (mention: CommitmentMention) => `${mention.documentId}::${mention.originKey}`;
+
+function toMention(draft: CommitmentDraft, document: SourceDocument): CommitmentMention {
+  return {
+    documentId: draft.documentId,
+    originKey: draft.originKey,
+    occurredAt: iso(document.occurredAt),
+    quote: draft.quote ?? '',
+    extractor: draft.extractor,
+  };
+}
+
+function identityOf(commitment: Commitment): CommitmentIdentity {
+  return {
+    subjectKey: commitment.subjectKey,
+    actionKey: commitment.actionKey,
+    actionTokens: commitment.actionKey ? commitment.actionKey.split('.') : [],
+    dedupeKey: commitment.dedupeKey,
+  };
+}
+
+// ---------------------------------------------------------------------------
+//  resolution
+// ---------------------------------------------------------------------------
+
+export interface ResolutionResult {
+  commitments: Commitment[];
+  resolved: number;
+  needsReview: Array<{ commitmentId: string; documentId: string; quote: string }>;
+}
+
+/**
+ * Close items a LATER meeting says are done.
+ *
+ * This is the difference between a to-do list and a system of record, and it is
+ * also the most dangerous thing in the pipeline, so every guard below exists
+ * because of a specific way it could lie:
+ *
+ *   • the resolving document must be IN SCOPE — a document id is the one field
+ *     an extractor could invent into another workspace's record;
+ *   • the item must have been raised at or before that meeting, or a note from
+ *     August is closing work agreed in September;
+ *   • only `open` and `blocked` items move, so a human's own `done`/`dropped`
+ *     is never rewritten by a re-run;
+ *   • a match too weak to act on is REPORTED rather than applied or discarded,
+ *     because the failure mode of a self-emptying to-do list has no symptom.
+ */
+export function applyResolutions(
+  commitments: Commitment[],
+  claims: ResolutionClaim[],
+  context: { documents: SourceDocument[] },
+): ResolutionResult {
+  const byId = new Map(context.documents.map((document) => [document.id, document]));
+  const out = commitments.map(cloneCommitment);
+  const needsReview: ResolutionResult['needsReview'] = [];
+  let resolved = 0;
+
+  for (const claim of claims) {
+    const document = byId.get(claim.documentId);
+    if (!document) continue; // out of scope, or invented
+    const at = iso(document.occurredAt);
+    const identity = commitmentIdentity({ title: claim.statement, subject: claim.subject });
+
+    let best: { commitment: Commitment; score: number; bar: number } | null = null;
+    for (const commitment of out) {
+      if (commitment.status !== 'open' && commitment.status !== 'blocked') continue;
+      if (at && commitment.firstSeenAt && commitment.firstSeenAt > at) continue;
+      const anchored = Boolean(identity.subjectKey) && identity.subjectKey === commitment.subjectKey;
+      // Two subjects that both exist and disagree are different things, and no
+      // amount of shared wording makes them the same one.
+      if (identity.subjectKey && commitment.subjectKey && !anchored) continue;
+      const candidate = {
+        commitment,
+        score: jaccard(identity.actionTokens, identityOf(commitment).actionTokens),
+        bar: anchored ? RESOLVE_THRESHOLD : UNANCHORED_RESOLVE_THRESHOLD,
+      };
+      if (!best || candidate.score > best.score) best = candidate;
+    }
+
+    if (!best) continue;
+    if (best.score >= best.bar) {
+      best.commitment.status = claim.outcome === 'dropped' ? 'dropped' : 'done';
+      best.commitment.statusReason = claim.quote || claim.statement;
+      best.commitment.resolvedBy = claim.documentId;
+      best.commitment.resolvedAt = at;
+      resolved += 1;
+    } else {
+      // ONE review row per claim — its single best candidate. Listing every open
+      // item on the matter would bury the one that matters.
+      needsReview.push({
+        commitmentId: best.commitment.externalId,
+        documentId: claim.documentId,
+        quote: claim.quote || claim.statement,
+      });
+    }
+  }
+
+  return { commitments: out, resolved, needsReview };
+}
+
+// ---------------------------------------------------------------------------
+
+function cloneCommitment(commitment: Commitment): Commitment {
+  return { ...commitment, mentions: commitment.mentions.map((mention) => ({ ...mention })) };
+}
+
+function unique(values: string[]): string[] {
+  return [...new Set(values.filter(Boolean))];
+}
+
+function iso(value: string | null | undefined): string | null {
+  if (!value) return null;
+  const parsed = Date.parse(value);
+  return Number.isNaN(parsed) ? null : new Date(parsed).toISOString();
+}
+
+/**
+ * A stable, dependency-free content hash (FNV-1a, doubled to 64 bits).
+ *
+ * It is an IDENTITY, not a security primitive: the only property required is
+ * that the same normalised text produces the same key in every runtime, which a
+ * crypto import would also give at the cost of a Node/browser split in a module
+ * that runs in both.
+ */
+export function hash(input: string): string {
+  let a = 0x811c9dc5;
+  let b = 0x01000193;
+  for (let index = 0; index < input.length; index += 1) {
+    const code = input.charCodeAt(index);
+    a = Math.imul(a ^ code, 0x01000193) >>> 0;
+    b = Math.imul(b ^ (code + index), 0x85ebca6b) >>> 0;
+  }
+  return `${a.toString(16).padStart(8, '0')}${b.toString(16).padStart(8, '0')}`;
+}

package/src/corpus/entities.ts

@@ -23,6 +23,8 @@
  * row with a missing subtitle.
  */
 import type {
+  Commitment,
+  CommitmentMention,
   CorpusScope,
   DocumentSection,
   MoneyEvent,
@@ -94,12 +96,15 @@ export interface CorpusTypeKeys {
   document: string;
   party: string;
   moneyEvent: string;
+  /** The trackable-commitment type (bd startsim-768w.38.25). */
+  commitment: string;
 }
 
 export const DEFAULT_TYPE_KEYS: CorpusTypeKeys = {
   document: 'source_document',
   party: 'party',
   moneyEvent: 'money_event',
+  commitment: 'commitment',
 };
 
 // ---------------------------------------------------------------------------
@@ -493,3 +498,132 @@ export function moneyEventAsDocument(event: MoneyEvent): SourceDocument {
     scope: event.scope,
   };
 }
+
+// ---------------------------------------------------------------------------
+//  Commitment (bd startsim-768w.38.25)
+// ---------------------------------------------------------------------------
+//
+// STORED IN THE SAME GENERIC ENVELOPE as everything else: `entity_type:
+// "commitment"`, discriminated by `data.subtype`. No new table, no new model, no
+// new app — a tenant that wants a sixth kind of trackable thing writes a
+// different subtype string (feedback_brutally_generic).
+//
+// `external_id` IS THE IDEMPOTENCY. It carries the dedupe key, and the tenant's
+// `/api/v1/entities/upsert/` is keyed on (entity_type, external_id), so two runs
+// racing each other converge on one row instead of writing two. Verified on the
+// live tandem tenant: a second upsert of the same external_id returns 200 and
+// updates in place.
+//
+// MENTIONS LIVE IN `data`, NOT AS EDGES, and that is a stated compromise rather
+// than an oversight. The canonical `derived_from` edge is declared
+// source_document -> source_document, a relationship row carries only
+// (rel_type, source, target), and a mention has to carry its own verbatim quote
+// or the row on screen cannot be checked against the record. Putting the quote
+// in `data` and the graph nowhere is the honest half; writing the edge as well
+// needs a schema change on the tenant side and is filed rather than half-done.
+
+/** How many mentions to keep on a row. A commitment discussed thirty times is
+ *  interesting; the thirty-first quote is not, and the row has to stay small. */
+const MAX_MENTIONS = 20;
+
+export function toCommitment(
+  entity: TenantEntity,
+  fallbackScope: CorpusScope,
+  sourceLabels: Record<string, string> = {},
+): Commitment {
+  const data = bag(entity);
+  const source = asText(pick(data, 'source', 'source_key', 'sourceKey')) || 'unknown';
+  const externalId = asText(entity.external_id ?? entity.externalId);
+  const mentions = asList(pick(data, 'mentions'))
+    .map(toMention)
+    .filter((mention): mention is CommitmentMention => mention !== null);
+  const status = asText(pick(data, 'status')) as Commitment['status'];
+
+  return {
+    id: String(entity.id),
+    externalId: externalId || `commit:${entity.id}`,
+    dedupeKey: asText(pick(data, 'dedupe_key', 'dedupeKey')) || externalId.replace(/^commit:/, ''),
+    subtype: asText(pick(data, 'subtype')) || 'action',
+    title: asText(pick(data, 'title')) || entity.name || 'Untitled item',
+    detail: asText(pick(data, 'detail', 'description')),
+    status: ['open', 'done', 'blocked', 'dropped'].includes(status) ? status : 'open',
+    ...(pick(data, 'status_reason', 'statusReason')
+      ? { statusReason: asText(pick(data, 'status_reason', 'statusReason')) } : {}),
+    ...(pick(data, 'resolved_by', 'resolvedBy')
+      ? { resolvedBy: asText(pick(data, 'resolved_by', 'resolvedBy')) } : {}),
+    resolvedAt: asIso(pick(data, 'resolved_at', 'resolvedAt')),
+    ...(pick(data, 'owner_name', 'ownerName') ? { ownerName: asText(pick(data, 'owner_name', 'ownerName')) } : {}),
+    ...(pick(data, 'subject') ? { subject: asText(pick(data, 'subject')) } : {}),
+    subjectKey: asText(pick(data, 'subject_key', 'subjectKey')),
+    actionKey: asText(pick(data, 'action_key', 'actionKey')),
+    dueAt: asIso(pick(data, 'due_at', 'dueAt')),
+    mentions,
+    firstSeenAt: asIso(pick(data, 'first_seen_at', 'firstSeenAt')),
+    lastSeenAt: asIso(pick(data, 'last_seen_at', 'lastSeenAt')),
+    extractorVersion: asText(pick(data, 'extractor_version', 'extractorVersion')),
+    source,
+    sourceLabel:
+      asText(pick(data, 'source_label', 'sourceLabel')) || sourceLabels[source] || titleCase(source),
+    scope: scopeOf(entity, fallbackScope),
+  };
+}
+
+function toMention(value: unknown): CommitmentMention | null {
+  if (!value || typeof value !== 'object') return null;
+  const data = value as Record<string, unknown>;
+  const documentId = asText(pick(data, 'document_id', 'documentId'));
+  if (!documentId) return null;
+  const extractor = asText(pick(data, 'extractor'));
+  return {
+    documentId,
+    originKey: asText(pick(data, 'origin_key', 'originKey')),
+    occurredAt: asIso(pick(data, 'occurred_at', 'occurredAt')),
+    quote: asText(pick(data, 'quote')),
+    extractor: extractor === 'model' ? 'model' : 'rules',
+  };
+}
+
+/** The write side: a commitment as the tenant's upsert payload. */
+export function commitmentToEntity(commitment: Commitment): {
+  entity_type: string;
+  external_id: string;
+  name: string;
+  data: Record<string, unknown>;
+} {
+  return {
+    entity_type: DEFAULT_TYPE_KEYS.commitment,
+    external_id: commitment.externalId,
+    name: commitment.title.slice(0, 200),
+    data: {
+      subtype: commitment.subtype,
+      title: commitment.title,
+      detail: commitment.detail,
+      status: commitment.status,
+      ...(commitment.statusReason ? { status_reason: commitment.statusReason } : {}),
+      ...(commitment.resolvedBy ? { resolved_by: commitment.resolvedBy } : {}),
+      ...(commitment.resolvedAt ? { resolved_at: commitment.resolvedAt } : {}),
+      ...(commitment.ownerName ? { owner_name: commitment.ownerName } : {}),
+      ...(commitment.subject ? { subject: commitment.subject } : {}),
+      subject_key: commitment.subjectKey,
+      action_key: commitment.actionKey,
+      dedupe_key: commitment.dedupeKey,
+      ...(commitment.dueAt ? { due_at: commitment.dueAt } : {}),
+      first_seen_at: commitment.firstSeenAt,
+      last_seen_at: commitment.lastSeenAt,
+      mention_count: commitment.mentions.length,
+      mentions: commitment.mentions.slice(-MAX_MENTIONS).map((mention) => ({
+        document_id: mention.documentId,
+        origin_key: mention.originKey,
+        occurred_at: mention.occurredAt,
+        quote: mention.quote.slice(0, 400),
+        extractor: mention.extractor,
+      })),
+      extractor_version: commitment.extractorVersion,
+      source: commitment.source,
+      // Marks the row as pipeline-written, the same way the corpus delivery
+      // marks its own — so a human-created item is distinguishable from a
+      // derived one, and a re-derivation knows what it is allowed to rewrite.
+      _origin: 'tenant-extraction',
+    },
+  };
+}