@startsimpli/integrations 0.1.5 → 0.2.1

package/src/corpus/extract.ts

@@ -0,0 +1,438 @@
+/**
+ * EXTRACTION — conversations in, trackable commitments out (bd startsim-768w.38.25).
+ *
+ * The pipeline, server-side, in one function:
+ *
+ *   1. read documents from the tenant data API, already org-scoped by the
+ *      server and narrowed the same way the Library narrows;
+ *   2. run the SCOPE GUARD over them — a third check, before anything is read
+ *      into a prompt (see ./retrieval for why this is not a convenience);
+ *   3. RULES PASS: parse the commitments the summariser ALREADY wrote down.
+ *      No model, deterministic, byte-stable between runs;
+ *   4. ONE model call over every document, told what the rules already found so
+ *      it ADDS rather than restates (feedback_batch_llm_calls — never a loop);
+ *   5. resolve IDENTITY, dedupe within the run and against what is stored;
+ *   6. apply RESOLUTIONS from later meetings, conservatively;
+ *   7. write the result back with `external_id` doing the idempotency.
+ *
+ * STEP 3 IS THE ONE PEOPLE SKIP. The source material here is Granola's own AI
+ * summaries, and on all sixteen real documents on the live tandem tenant the
+ * last heading is `## Next Steps`, under which sit bolded imperatives with the
+ * owner in parentheses. That is an action list the vendor already produced and
+ * already paid for. Sending the document to a model and asking it to find the
+ * action items redoes that work, less reliably, at a per-token price, and then
+ * has to be deduped against the version the vendor wrote. The rules pass takes
+ * it verbatim; the model's job is what the summariser did NOT promote — items
+ * stated in the body, blockers, decisions, and the subject each one is about.
+ *
+ * A PROVIDER OUTAGE COSTS THE EXTRAS, NOT THE LIST. Step 4 is wrapped: if the
+ * model is unreachable the run still produces every commitment the vendor's own
+ * structure carried, and says so in `fromModel: 0`.
+ */
+import { EXTRACTOR_VERSION, applyResolutions, mergeDrafts, reconcile } from './commitments';
+import { enforceScope } from './retrieval';
+import { outcomeSections } from './text';
+import type {
+  Commitment,
+  CommitmentApi,
+  CommitmentDraft,
+  CommitmentStatus,
+  CorpusApi,
+  CorpusExtractorLlm,
+  CorpusScope,
+  ExtractionDraft,
+  ExtractionResult,
+  ResolutionClaim,
+  SourceDocument,
+} from './types';
+
+export interface ExtractCommitmentsOptions {
+  /** How many documents to read before extracting. */
+  documentLimit?: number;
+  /** Pages of the data API this run may read. Bounds one run's cost. */
+  maxPages?: number;
+  /** Characters of each document's text sent to the model. */
+  maxCharsPerDocument?: number;
+}
+
+const DEFAULTS: Required<ExtractCommitmentsOptions> = {
+  documentLimit: 200,
+  maxPages: 10,
+  maxCharsPerDocument: 12_000,
+};
+
+// ---------------------------------------------------------------------------
+//  3. the rules pass — the vendor already did this
+// ---------------------------------------------------------------------------
+
+/** `- **Do the thing** (Owner)` — a bullet in an outcome section. */
+const BULLET = /^\s{0,3}[-*]\s+(\S.*)$/;
+
+/**
+ * THE BOLD IS REQUIRED, and that is the single most important line in the rules
+ * pass.
+ *
+ * Measured on the sixteen real records: accepting bare bullets took precision
+ * from 94% to 75%, because headings like "Action Items Review" and "Open Issues
+ * and Follow-ups" match the outcome regex while the bullets under them are
+ * DISCUSSION, not commitments — "Matter is on the responsible attorney's radar"
+ * is a status note and putting it on a to-do list is a lie about what was
+ * agreed. The summariser marks a real commitment by bolding it. Trusting that
+ * marker is what makes this pass worth having; guessing past it is what makes a
+ * to-do list nobody believes.
+ */
+const LEADING_BOLD = /^\*\*(.+?)\*\*/;
+
+/**
+ * A trailing parenthetical that is plausibly a person or a short list of them.
+ *
+ * Deliberately narrow, and it rejects ALL-CAPS: `(Jonathan, Kevin)` and `(Kate)`
+ * are owners, `(CTO)` is a job title inside the sentence and `(4 5 AC
+ * eight-plex)` is part of the sentence. Reading either of the last two as an
+ * owner puts junk in the column the whole screen is sorted by. Anything that
+ * fails this stays in the title and the model pass gets a chance at the owner.
+ */
+const TRAILING_PAREN = /^(.*?)\s*\(([^()]{2,60})\)\s*$/;
+const OWNER_SHAPE = /^[A-Z][A-Za-z.'’-]*(?:\s+[A-Za-z.'’-]+)*(?:\s*,\s*[A-Z][A-Za-z.'’-]*(?:\s+[A-Za-z.'’-]+)*)*$/;
+const ACRONYM = /^[A-Z][A-Z0-9.\s,&-]*$/;
+
+/**
+ * The commitments the source document already states as commitments.
+ *
+ * NO MODEL. Given the same text this returns the same drafts with the same
+ * `originKey`s forever, which is what makes the majority of the corpus perfectly
+ * idempotent rather than merely usually idempotent.
+ */
+export function rulesExtract(document: SourceDocument): CommitmentDraft[] {
+  const text = [document.body, ...document.sections.map((section) => section.text)]
+    .filter(Boolean)
+    .join('\n\n');
+
+  const drafts: CommitmentDraft[] = [];
+  for (const section of outcomeSections(text)) {
+    let current: { line: string; detail: string[] } | null = null;
+    const flush = () => {
+      if (!current) return;
+      const draft = toDraft(document, current.line, current.detail.join('\n'));
+      if (draft) drafts.push(draft);
+      current = null;
+    };
+    for (const line of section.body.split('\n')) {
+      const bullet = BULLET.exec(line);
+      if (bullet) {
+        flush();
+        current = { line: line.trim(), detail: [] };
+      } else if (current && line.trim()) {
+        current.detail.push(line.trim());
+      }
+    }
+    flush();
+  }
+  return drafts;
+}
+
+function toDraft(document: SourceDocument, line: string, detail: string): CommitmentDraft | null {
+  const content = line.replace(BULLET, '$1').trim();
+  const bold = LEADING_BOLD.exec(content);
+  if (!bold || bold[1].trim().split(/\s+/).length < 3) return null;
+
+  // The owner is read from the LAST trailing parenthetical, whether the
+  // summariser wrote it inside the bold — `**Email Michelle … (Kate)**` — or
+  // after it — `**Complete the equity item** (Jonathan)`. Both spellings are
+  // present in the real corpus and there is no principled reason to honour one;
+  // measured across the sixteen records, reading only the outside form cost six
+  // of eighteen owners on a single document.
+  const flat = content.replace(/\*\*/g, '').trim();
+  const trailing = TRAILING_PAREN.exec(flat);
+  const candidate = trailing?.[2]?.trim() ?? '';
+  const isOwner =
+    Boolean(candidate) && OWNER_SHAPE.test(candidate) && !ACRONYM.test(candidate) && !/\d/.test(candidate);
+
+  const ownerName = isOwner ? candidate : undefined;
+  let title = isOwner ? trailing![1] : flat;
+
+  title = title.replace(/\s+/g, ' ').trim();
+  // Two words is a fragment, not a commitment. The model pass sees the whole
+  // document anyway, so nothing is lost by being strict here.
+  if (title.split(/\s+/).length < 3) return null;
+
+  return {
+    documentId: document.id,
+    // Byte-stable: derived from the vendor's own words, never from a model.
+    originKey: `rules:${slug(title)}`,
+    subtype: 'action',
+    title,
+    detail: detail.trim(),
+    status: 'open',
+    ...(ownerName ? { ownerName } : {}),
+    // The bullet exactly as written, so every row on the screen can be checked
+    // against the record without opening it.
+    quote: line.trim(),
+    extractor: 'rules',
+  };
+}
+
+const slug = (text: string) =>
+  text.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '').slice(0, 60);
+
+// ---------------------------------------------------------------------------
+//  the prompt
+// ---------------------------------------------------------------------------
+
+/**
+ * Every rule in here is also enforced in code, because a prompt is a request and
+ * a check is a guarantee. What the prompt uniquely buys is the two things code
+ * cannot derive: what a statement is ABOUT, and whether a later meeting settled
+ * an earlier one.
+ */
+export function extractSystemPrompt(): string {
+  return [
+    'You read meeting records for one company and turn them into trackable items.',
+    'You are given documents that ALREADY have some items extracted from them, listed under ALREADY_FOUND. Do NOT repeat those. Find what they missed.',
+    'An item is something a person now owes, has decided, needs, is at risk from, or has asked. It must be stated in the document — never inferred, never invented.',
+    'For every item, copy a short verbatim "quote" from the document it came from, and echo that document\'s exact id.',
+    'Set "subject" to what the item is ABOUT — a matter or case number, a property, a client, a system, a person. Copy an identifier verbatim when the text has one.',
+    'Set "owner" only to a name the document actually names. Leave it out otherwise.',
+    'Set "dueAt" only when the document states a date. Never estimate one.',
+    'Also report RESOLUTIONS: statements that something previously outstanding is now done or abandoned. Give the subject, what happened, and a verbatim quote.',
+    'Respond with JSON only: {"items":[{"documentId":string,"subtype":"action"|"decision"|"need"|"risk"|"question","title":string,"detail":string,"status":"open"|"done"|"blocked","owner":string,"subject":string,"dueAt":string|null,"quote":string}],"resolutions":[{"documentId":string,"subject":string,"statement":string,"quote":string,"outcome":"done"|"dropped"}]}',
+    '"title" must be one short imperative line.',
+  ].join('\n');
+}
+
+/** ONE payload carrying every document, each tagged with the id to echo. */
+export function renderExtractPrompt(input: {
+  documents: Array<{
+    id: string; title: string; source: string; occurredAt: string | null;
+    participants: string[]; text: string; alreadyFound: string[];
+  }>;
+}): string {
+  const blocks = input.documents.map((document) =>
+    [
+      `<document id="${document.id}">`,
+      `title: ${document.title}`,
+      `source: ${document.source}`,
+      `occurred_at: ${document.occurredAt ?? 'unknown'}`,
+      document.participants.length ? `participants: ${document.participants.join(', ')}` : '',
+      document.alreadyFound.length
+        ? `ALREADY_FOUND (do not repeat these):\n${document.alreadyFound.map((title) => `- ${title}`).join('\n')}`
+        : 'ALREADY_FOUND: none',
+      '',
+      document.text,
+      '</document>',
+    ]
+      .filter((line) => line !== '')
+      .join('\n'),
+  );
+  return [`DOCUMENTS (${input.documents.length}):`, blocks.join('\n\n')].join('\n');
+}
+
+// ---------------------------------------------------------------------------
+//  the pipeline
+// ---------------------------------------------------------------------------
+
+export interface ExtractRequest {
+  scope: CorpusScope;
+  /** Narrow the run to specific records. Omit to sweep the corpus. */
+  documentIds?: string[];
+}
+
+export interface ExtractDeps {
+  corpus: CorpusApi & Partial<CommitmentApi>;
+  extractor: CorpusExtractorLlm;
+}
+
+export async function extractCommitments(
+  request: ExtractRequest,
+  deps: ExtractDeps,
+  options: ExtractCommitmentsOptions = {},
+): Promise<ExtractionResult> {
+  const { documentLimit, maxPages, maxCharsPerDocument } = { ...DEFAULTS, ...options };
+  const { corpus, extractor } = deps;
+
+  // An empty scope grants nothing (`isInScope`). Refusing here rather than
+  // downstream means an unscoped read is never MADE, not made and then filtered.
+  if (!request.scope?.orgId) {
+    return empty(extractor, 'no scope');
+  }
+
+  const { documents: candidates, matched } = await readDocuments(corpus, documentLimit, maxPages);
+  const { kept, dropped } = enforceScope(candidates, request.scope);
+  const documents = request.documentIds?.length
+    ? kept.filter((document) => request.documentIds!.includes(document.id))
+    : kept;
+
+  if (documents.length === 0) {
+    return { ...empty(extractor, 'nothing in scope'), droppedOutOfScope: dropped.length };
+  }
+
+  // 3. rules first, so the model can be told what not to repeat.
+  const ruleDrafts = documents.flatMap(rulesExtract);
+  const foundByDocument = new Map<string, string[]>();
+  for (const draft of ruleDrafts) {
+    foundByDocument.set(draft.documentId, [...(foundByDocument.get(draft.documentId) ?? []), draft.title]);
+  }
+
+  // 4. ONE call, every document, ids echoed.
+  let draft: ExtractionDraft = { items: [], resolutions: [] };
+  try {
+    draft = await extractor.extract({
+      documents: documents.map((document) => ({
+        id: document.id,
+        title: document.title,
+        source: document.sourceLabel || document.source,
+        occurredAt: document.occurredAt,
+        participants: document.participants.map((party) => party.name),
+        text: documentText(document, maxCharsPerDocument),
+        alreadyFound: foundByDocument.get(document.id) ?? [],
+      })),
+    });
+  } catch {
+    // The deterministic half does not depend on the probabilistic half.
+    draft = { items: [], resolutions: [] };
+  }
+
+  const allowed = new Set(documents.map((document) => document.id));
+  const modelDrafts = normaliseItems(draft.items ?? [], allowed);
+  const claims = normaliseResolutions(draft.resolutions ?? [], allowed);
+
+  // 5. identity, dedupe within the run, then against what is stored.
+  const existing = corpus.listCommitments
+    ? await corpus
+        .listCommitments({ pageSize: 500 })
+        .then((page) => page.results)
+        .catch(() => [] as Commitment[])
+    : [];
+
+  const groups = mergeDrafts([...ruleDrafts, ...modelDrafts]);
+  const reconciled = reconcile(existing, groups, { scope: request.scope, documents });
+
+  // 6. resolutions, against the documents the scope guard approved and nothing else.
+  const resolutions = applyResolutions(reconciled.commitments, claims, { documents });
+
+  // 7. write. ONE call carrying every row, keyed on `external_id`.
+  if (corpus.saveCommitments) {
+    await corpus.saveCommitments(resolutions.commitments);
+  }
+
+  return {
+    documentsProcessed: documents.length,
+    fromRules: ruleDrafts.length,
+    fromModel: modelDrafts.length,
+    created: reconciled.created,
+    merged: reconciled.merged,
+    resolved: resolutions.resolved,
+    needsReview: resolutions.needsReview,
+    droppedOutOfScope: dropped.length,
+    provider: extractor.describe(),
+    ...(matched > candidates.length ? { } : {}),
+  };
+}
+
+/**
+ * Read the corpus, PAST THE FIRST PAGE.
+ *
+ * The tenant's `EntityViewSet` pages with DRF's default and ignores `page_size`
+ * (verified on the wire against the live tandem tenant: asking for 3 returns
+ * all sixteen), so a caller that reads one page and calls it the corpus silently
+ * stops seeing older records at whatever `PAGE_SIZE` happens to be. Extraction
+ * over "the most recent page" would quietly stop extracting.
+ */
+async function readDocuments(
+  corpus: CorpusApi,
+  limit: number,
+  maxPages: number,
+): Promise<{ documents: SourceDocument[]; matched: number }> {
+  const documents: SourceDocument[] = [];
+  let matched = 0;
+  for (let page = 1; page <= maxPages && documents.length < limit; page += 1) {
+    const response = await corpus.listDocuments({ page, pageSize: limit });
+    matched = response.count ?? response.results.length;
+    documents.push(...response.results);
+    if (!response.next || response.results.length === 0) break;
+  }
+  return { documents: documents.slice(0, limit), matched };
+}
+
+function documentText(document: SourceDocument, maxChars: number): string {
+  const parts = [document.body, ...document.sections.map((section) => `## ${section.label}\n${section.text}`)];
+  const joined = parts.filter(Boolean).join('\n\n');
+  return joined.length > maxChars ? `${joined.slice(0, maxChars)}\n…[truncated]` : joined;
+}
+
+const SUBTYPES = new Set(['action', 'decision', 'need', 'risk', 'question']);
+const STATUSES = new Set<CommitmentStatus>(['open', 'done', 'blocked', 'dropped']);
+
+/**
+ * Shape-check what the model returned.
+ *
+ * An item naming a document the run never supplied is DROPPED, not repaired:
+ * the document id is the one field that could carry a reference into another
+ * workspace, and a plausible-looking id is exactly what a model produces when it
+ * is guessing.
+ */
+function normaliseItems(
+  items: ExtractionDraft['items'],
+  allowed: Set<string>,
+): CommitmentDraft[] {
+  const out: CommitmentDraft[] = [];
+  for (const item of items) {
+    const documentId = String(item?.documentId ?? '').trim();
+    const title = String(item?.title ?? '').replace(/\s+/g, ' ').trim();
+    if (!allowed.has(documentId) || title.split(/\s+/).length < 3) continue;
+    const subtype = String(item.subtype ?? 'action').toLowerCase();
+    const status = String(item.status ?? 'open').toLowerCase() as CommitmentStatus;
+    const owner = String(item.owner ?? '').trim();
+    out.push({
+      documentId,
+      originKey: `model:${slug(title)}`,
+      subtype: SUBTYPES.has(subtype) ? subtype : 'action',
+      title,
+      detail: String(item.detail ?? '').trim(),
+      status: STATUSES.has(status) ? status : 'open',
+      ...(owner && owner.length <= 60 ? { ownerName: owner } : {}),
+      ...(item.subject ? { subject: String(item.subject).trim() } : {}),
+      ...(item.dueAt ? { dueAt: String(item.dueAt) } : {}),
+      quote: String(item.quote ?? '').trim(),
+      extractor: 'model',
+    });
+  }
+  return out;
+}
+
+function normaliseResolutions(
+  resolutions: NonNullable<ExtractionDraft['resolutions']>,
+  allowed: Set<string>,
+): ResolutionClaim[] {
+  const out: ResolutionClaim[] = [];
+  for (const claim of resolutions) {
+    const documentId = String(claim?.documentId ?? '').trim();
+    const statement = String(claim?.statement ?? '').trim();
+    if (!allowed.has(documentId) || !statement) continue;
+    out.push({
+      documentId,
+      ...(claim.subject ? { subject: String(claim.subject).trim() } : {}),
+      statement,
+      quote: String(claim.quote ?? '').trim(),
+      outcome: claim.outcome === 'dropped' ? 'dropped' : 'done',
+    });
+  }
+  return out;
+}
+
+function empty(extractor: CorpusExtractorLlm, _why: string): ExtractionResult {
+  void _why;
+  return {
+    documentsProcessed: 0,
+    fromRules: 0,
+    fromModel: 0,
+    created: 0,
+    merged: 0,
+    resolved: 0,
+    needsReview: [],
+    droppedOutOfScope: 0,
+    provider: extractor.describe(),
+  };
+}
+
+export { EXTRACTOR_VERSION };