@startsimpli/integrations 0.2.14 → 0.2.17

package/src/corpus/runs.ts

@@ -0,0 +1,563 @@
+/**
+ * EXTRACTION AS A JOB (bd startsim-768w.38.28).
+ *
+ * THE MEASUREMENT THAT FORCED THIS. On the live tandem tenant `POST
+ * /corpus/extract` returned 504 at every slice size, `limit=1` included. One
+ * document takes about twenty seconds against claude-sonnet-5, so seventeen
+ * documents is roughly six minutes — and there are two independent sixty-second
+ * walls in front of it: the tenant gateway's, and the provider client's own.
+ * There is no slice size that fits a browser request. A synchronous button
+ * cannot do this job, and no amount of tuning makes it able to.
+ *
+ * ────────────────────────────────────────────────────────────────────────────
+ *  WHY THE QUEUE IS HERE AND NOT IN THE CENTRAL BACKEND
+ * ────────────────────────────────────────────────────────────────────────────
+ *
+ * The obvious answer is "run it in `start-simpli-api`, where Celery, a broker
+ * and the connector framework already live", by analogy with the corpus bridge:
+ * projection happens at DELIVERY time in central, which is what lets a mapping
+ * fix re-derive without re-fetching from the vendor. That analogy is right about
+ * the SHAPE — this work is asynchronous, resumable and schedulable — and wrong
+ * about the LOCATION, for three reasons that are facts rather than preferences.
+ *
+ *   1. DELIVERY PROJECTION AND EXTRACTION ARE NOT THE SAME KIND OF STEP. Central
+ *      projection is a pure, cheap, deterministic function of a vendor payload
+ *      central already holds. Extraction is a twenty-second-per-document
+ *      frontier-model call whose INPUT is the tenant's corpus — including
+ *      documents that never came through a central connector, such as the drafts
+ *      Ask writes back — and whose OUTPUT is tenant records.
+ *   2. IT NEEDS A WRITE CREDENTIAL INSIDE THE TENANT. The run reads the tenant's
+ *      existing commitments to build ALREADY_FOUND (the structural half of
+ *      idempotency), reads the tenant schema registry, and upserts entities.
+ *      Running it centrally means giving central a credential that can write
+ *      into every foundry's workspace. That is a security expansion across the
+ *      whole platform to solve a scheduling problem.
+ *   3. THE EXTRACTOR IS THIS PACKAGE. Rules pass, prompts, grounding,
+ *      verification, identity and reconciliation are TypeScript here, with the
+ *      measured numbers in their comments and a test suite around them. Central
+ *      is Python. "Run it centrally" is either a rewrite of all of that in
+ *      another language or a new Node service — both far larger than the problem.
+ *
+ * The rejected alternatives, and why:
+ *
+ *   • A WORKER AND BROKER IN `tenant-starter`. Generic, but it puts an AI
+ *     dependency in the boot path of every foundry, including the ones with no
+ *     AI surface at all — the same objection that kept pgvector out of it.
+ *   • CLIENT-DRIVEN CHUNKING. Cheapest, and it makes the browser the scheduler.
+ *     A tab that closes stops the sweep; a tab that reopens cannot see it. The
+ *     browser is a bad system of record.
+ *
+ * ────────────────────────────────────────────────────────────────────────────
+ *  WHAT THIS IS INSTEAD
+ * ────────────────────────────────────────────────────────────────────────────
+ *
+ * A DURABLE, LEASED, RESUMABLE RUN RECORD kept in the tenant's own generic
+ * entity store, with an idempotent STEP as the unit of work. Nothing is added to
+ * `tenant-starter`: a run is one more `Entity` row, which is exactly what that
+ * store is for.
+ *
+ * The important consequence is that the DRIVER is swappable and the durability
+ * is not. Today the tenant app server drives it in-process, past the gateway
+ * that killed the old design. Tomorrow a Celery beat in central can drive the
+ * same step over HTTP for unattended sweeps
+ * (feedback_scheduled_work_celery), and nothing here changes. That is the part
+ * of the central argument worth keeping, kept without the rewrite.
+ *
+ * The four properties it owes, and the mechanism for each:
+ *
+ *   RESUMPTION      `cursor` is durable. A sweep interrupted at nine of
+ *                   seventeen restarts at ten. Partial progress is progress
+ *                   because the commitments from those nine are already written.
+ *   IDEMPOTENCE     Every step is a no-op when repeated: identity resolves on
+ *                   `(subject, action)` stems carried as `external_id`, mentions
+ *                   key on `(documentId, originKey)`, and the tenant upsert is
+ *                   keyed on `(entity_type, external_id)`. A retry converges.
+ *   LEGIBILITY      A model that did not answer is `lastModelFailure` and
+ *                   `totals.modelFailures`, never `fromModel: 0` pretending to
+ *                   be a finding.
+ *   ONE DRIVER      A lease. Not an optimisation: two drivers over one cursor
+ *                   means two model calls over one document, and two model calls
+ *                   over one document produce two differently-worded items that
+ *                   identity resolution cannot see are the same. That is exactly
+ *                   how a live re-run once turned 82 commitments into 98.
+ */
+import { extractCommitments } from './extract';
+import type { ExtractCommitmentsOptions, ExtractDeps } from './extract';
+import type {
+  CorpusScope,
+  ExtractionModelFailure,
+  ExtractionResult,
+  ExtractionRun,
+  ExtractionRunStore,
+  ExtractionRunSummary,
+  ExtractionRunTotals,
+} from './types';
+
+export interface ExtractionRunDeps extends ExtractDeps {
+  runs: ExtractionRunStore;
+}
+
+export interface StartExtractionRunRequest {
+  scope: CorpusScope;
+  /** Ignore a resumable sweep and begin again from the first document. */
+  restart?: boolean;
+  /**
+   * Who is claiming the lease. Defaults to a value unique to this process, which
+   * is what makes "somebody else took it" detectable at all.
+   */
+  driverId?: string;
+}
+
+export interface ExtractionRunOptions extends ExtractCommitmentsOptions {
+  /**
+   * Documents one step may ask for. The TEXT budget cuts it again — see
+   * `ExtractCommitmentsOptions.maxCharsPerRun` — so this is a ceiling, not a
+   * promise, and `nextOffset` is what is authoritative about what was done.
+   */
+  documentsPerSlice?: number;
+  /** How long a claim is good for. Renewed after every step. */
+  leaseMs?: number;
+  /** Wall-clock ceiling on one driver's stint. It stops; the run stays resumable. */
+  maxRunMs?: number;
+  /** Attempts on a step that THREW before the run is called failed. */
+  maxSliceAttempts?: number;
+  retryDelayMs?: number;
+  /** Injectable, so the lease and the timestamps are testable. */
+  now?: () => number;
+  driverId?: string;
+  /** Sweeps kept inline on the record. */
+  historyLimit?: number;
+  /** Ids kept on the record, so it never grows without bound. */
+  idLimit?: number;
+}
+
+const RUN_DEFAULTS = {
+  documentsPerSlice: 2,
+  // Comfortably longer than one step. A step is one model call of ~20s plus a
+  // verification call, so a minute is not enough and an hour would make a dead
+  // driver block the run until somebody noticed.
+  leaseMs: 5 * 60_000,
+  maxRunMs: 30 * 60_000,
+  maxSliceAttempts: 3,
+  retryDelayMs: 2_000,
+  historyLimit: 5,
+  idLimit: 1_000,
+};
+
+export const EMPTY_TOTALS: ExtractionRunTotals = {
+  documentsProcessed: 0,
+  fromRules: 0,
+  fromModel: 0,
+  ungrounded: 0,
+  unconfirmed: 0,
+  created: 0,
+  merged: 0,
+  resolved: 0,
+  droppedOutOfScope: 0,
+  modelCalls: 0,
+  modelFailures: 0,
+};
+
+/** States a driver stops at. */
+const TERMINAL = new Set(['succeeded', 'failed', 'cancelled']);
+
+/**
+ * States a NEW sweep is needed for.
+ *
+ * `failed` is deliberately NOT one of them, and that is the difference between
+ * "resumable" as a word and as a property. A sweep that stopped at meeting nine
+ * because the workspace 502'd, or because the session that started it expired
+ * — thirty minutes is the token's life and a full sweep is about ten — has nine
+ * meetings' worth of commitments already written and a cursor pointing at the
+ * tenth. Starting that over is the exact waste this design exists to avoid.
+ * Only a sweep that finished, or one somebody cancelled, means "begin again".
+ */
+const COMPLETE = new Set(['succeeded', 'cancelled']);
+
+/**
+ * A failure of AUTHORITY, not of the work.
+ *
+ * Retried it stays a 401, and three attempts two seconds apart just spends six
+ * seconds proving that. It is also not a reason to call the sweep dead: the
+ * cursor is good, the rows are written, and the very next request carries a
+ * fresh bearer. So it stops the stint and leaves the run adoptable.
+ */
+function isAuthFailure(err: unknown): boolean {
+  const status = (err as { status?: number })?.status;
+  return status === 401 || status === 403;
+}
+
+// ---------------------------------------------------------------------------
+//  reading a run
+// ---------------------------------------------------------------------------
+
+export function isLeaseLive(run: ExtractionRun, now: number): boolean {
+  if (!run.leaseUntil) return false;
+  const until = Date.parse(run.leaseUntil);
+  return !Number.isNaN(until) && until > now;
+}
+
+/**
+ * A run that says it is running but whose driver has gone away.
+ *
+ * The honest name for what happens when a container is replaced mid-sweep. It is
+ * reported rather than hidden, because "this stopped and can be resumed" is
+ * actionable and a spinner that never ends is not.
+ */
+export function isStalled(run: ExtractionRun, now: number): boolean {
+  // A run that FAILED is stopped, not stalled — it reported why, and the screen
+  // says that rather than offering a mystery. `succeeded` and `cancelled` are
+  // simply over.
+  if (TERMINAL.has(run.status)) return false;
+  return !isLeaseLive(run, now);
+}
+
+export function runProgress(run: ExtractionRun): { done: number; total: number; percent: number } {
+  const total = Math.max(run.totalDocuments, run.cursor);
+  const done = Math.min(run.cursor, total || run.cursor);
+  return {
+    done,
+    total,
+    percent: total > 0 ? Math.round((done / total) * 100) : 0,
+  };
+}
+
+/**
+ * Did this sweep read everything the corpus holds?
+ *
+ * The cursor is an offset into a list ordered newest-first, so a document that
+ * arrives mid-sweep shifts it and one document can be stepped over. Recording
+ * ids makes that DETECTABLE — a complete-looking sweep that read fewer documents
+ * than the corpus holds says so, and the next sweep, which starts from the top
+ * and is idempotent, picks the straggler up.
+ */
+export function runCoverage(run: ExtractionRun): { read: number; inCorpus: number; missed: number } {
+  const read = run.processedIds.length;
+  const inCorpus = run.totalDocuments;
+  return { read, inCorpus, missed: Math.max(0, inCorpus - read) };
+}
+
+export function newExtractionRun(input: {
+  scope?: CorpusScope;
+  totalDocuments?: number;
+  now?: number;
+  runId?: string;
+}): ExtractionRun {
+  const at = new Date(input.now ?? Date.now()).toISOString();
+  return {
+    runId: input.runId ?? `sweep-${at.replace(/[^0-9]/g, '').slice(0, 14)}-${randomSuffix()}`,
+    scope: input.scope ?? { orgId: '' },
+    status: 'queued',
+    cursor: 0,
+    totalDocuments: input.totalDocuments ?? 0,
+    totals: { ...EMPTY_TOTALS },
+    processedIds: [],
+    modelIncompleteIds: [],
+    needsReview: [],
+    provider: { provider: 'none', model: '', live: false },
+    startedAt: at,
+    updatedAt: at,
+    finishedAt: null,
+    leaseOwner: null,
+    leaseUntil: null,
+    error: null,
+    lastModelFailure: null,
+    history: [],
+  };
+}
+
+// ---------------------------------------------------------------------------
+//  starting one
+// ---------------------------------------------------------------------------
+
+/**
+ * Start a sweep, or take over one that stopped, and RETURN WITHOUT DOING THE
+ * WORK.
+ *
+ * That last part is the whole contract. This is what a browser request calls, so
+ * it must not contain a model call — the moment it does, it is the design that
+ * 504s.
+ *
+ * The three cases it decides between:
+ *
+ *   • a sweep is running and its lease is LIVE → hand back that sweep. Pressing
+ *     the button twice must not produce two drivers.
+ *   • a sweep is unfinished and its lease is DEAD → adopt it, from its cursor.
+ *     This is the container-was-replaced case and it must not start over.
+ *   • anything else → a new sweep, from the first document, with the previous
+ *     one filed in `history`.
+ */
+export async function startExtractionRun(
+  request: StartExtractionRunRequest,
+  deps: ExtractionRunDeps,
+  options: ExtractionRunOptions = {},
+): Promise<ExtractionRun> {
+  // The same refusal `extractCommitments` makes, made before anything is written
+  // rather than after. An empty scope grants nothing.
+  if (!request.scope?.orgId) {
+    throw new Error('This session has no workspace, so there is nothing to extract.');
+  }
+
+  const config = { ...RUN_DEFAULTS, ...options };
+  const now = options.now ?? Date.now;
+  const at = now();
+  const driver = request.driverId ?? options.driverId ?? defaultDriverId();
+
+  const existing = await deps.runs.readRun().catch(() => null);
+
+  if (existing && !COMPLETE.has(existing.status) && !request.restart) {
+    if (isLeaseLive(existing, at) && existing.leaseOwner !== driver) {
+      // Somebody is already driving. Reporting their run is the honest answer to
+      // "start a run" and it is also the one that cannot duplicate.
+      return { ...existing, adopted: false };
+    }
+    const adopted: ExtractionRun = {
+      ...existing,
+      // The adopter proved this from its own token a moment ago; the record's
+      // copy is only ever a cache of it.
+      scope: request.scope,
+      status: 'running',
+      leaseOwner: driver,
+      leaseUntil: new Date(at + config.leaseMs).toISOString(),
+      updatedAt: new Date(at).toISOString(),
+      error: null,
+      adopted: !isLeaseLive(existing, at),
+    };
+    return { ...(await deps.runs.writeRun(adopted)), adopted: adopted.adopted };
+  }
+
+  // ONE CHEAP READ, so the very first progress line says "0 of 17" instead of
+  // "0 of 0". A count is a page of one row; it is not a model call and it does
+  // not put this function anywhere near the gateway's sixty seconds.
+  const totalDocuments = await deps.corpus
+    .listDocuments({ page: 1, pageSize: 1 })
+    .then((page) => page.count)
+    .catch(() => existing?.totalDocuments ?? 0);
+
+  const fresh: ExtractionRun = {
+    ...newExtractionRun({ scope: request.scope, totalDocuments, now: at }),
+    status: 'queued',
+    leaseOwner: driver,
+    leaseUntil: new Date(at + config.leaseMs).toISOString(),
+    history: existing
+      ? [summarise(existing), ...(existing.history ?? [])].slice(0, config.historyLimit)
+      : [],
+  };
+  return { ...(await deps.runs.writeRun(fresh)), adopted: false };
+}
+
+function summarise(run: ExtractionRun): ExtractionRunSummary {
+  return {
+    runId: run.runId,
+    status: run.status,
+    startedAt: run.startedAt,
+    finishedAt: run.finishedAt,
+    totals: run.totals,
+    error: run.error,
+  };
+}
+
+// ---------------------------------------------------------------------------
+//  one step
+// ---------------------------------------------------------------------------
+
+/**
+ * ONE SLICE: read from the cursor, extract, write what changed, move the cursor,
+ * persist.
+ *
+ * IT ALWAYS ADVANCES, even when the model half failed. That is deliberate. The
+ * rules pass DID land for those documents, re-running the same slice against the
+ * same prompt of the same size will hit the same deadline, and a run that
+ * retries a timeout forever never reaches document ten. The failure is recorded
+ * against the document ids instead, and the fix — a bigger deadline, or less
+ * text per call — is an operator's, not a loop's. A whole re-run is idempotent,
+ * so those documents get another chance for free.
+ *
+ * A slice that THROWS is a different thing entirely and does not advance: see
+ * `driveExtractionRun`.
+ */
+export async function advanceExtractionRun(
+  run: ExtractionRun,
+  deps: ExtractionRunDeps,
+  options: ExtractionRunOptions = {},
+): Promise<ExtractionRun> {
+  const config = { ...RUN_DEFAULTS, ...options };
+  const now = options.now ?? Date.now;
+  if (!run.scope?.orgId) {
+    throw new Error('This run has no workspace, so there is nothing to extract.');
+  }
+
+  const result = await extractCommitments(
+    { scope: run.scope, offset: run.cursor, limit: config.documentsPerSlice },
+    deps,
+    options,
+  );
+
+  const at = now();
+  const advanced = result.nextOffset ?? run.cursor + result.documentsProcessed;
+  const complete = result.nextOffset === null;
+
+  const next: ExtractionRun = {
+    ...run,
+    status: complete ? 'succeeded' : 'running',
+    cursor: advanced,
+    totalDocuments: result.totalDocuments || run.totalDocuments,
+    totals: addSlice(run.totals, result),
+    processedIds: capped([...run.processedIds, ...result.documentIds], config.idLimit),
+    modelIncompleteIds: result.modelFailure
+      ? capped([...run.modelIncompleteIds, ...result.documentIds], config.idLimit)
+      : run.modelIncompleteIds,
+    needsReview: capped([...run.needsReview, ...result.needsReview], config.idLimit),
+    // WHAT ANSWERED beats what was configured, and a slice where nothing
+    // answered must not overwrite the name of the model that did.
+    provider: result.provider.live ? result.provider : run.provider,
+    updatedAt: new Date(at).toISOString(),
+    finishedAt: complete ? new Date(at).toISOString() : null,
+    // A finished run holds no lease. Leaving one behind makes the next start
+    // look like a race with a driver that is not there.
+    leaseUntil: complete ? null : new Date(at + config.leaseMs).toISOString(),
+    lastModelFailure: result.modelFailure ?? run.lastModelFailure,
+    error: null,
+  };
+
+  return deps.runs.writeRun(next);
+}
+
+function addSlice(totals: ExtractionRunTotals, result: ExtractionResult): ExtractionRunTotals {
+  return {
+    documentsProcessed: totals.documentsProcessed + result.documentsProcessed,
+    fromRules: totals.fromRules + result.fromRules,
+    fromModel: totals.fromModel + result.fromModel,
+    ungrounded: totals.ungrounded + result.ungrounded,
+    unconfirmed: totals.unconfirmed + result.unconfirmed,
+    created: totals.created + result.created,
+    merged: totals.merged + result.merged,
+    resolved: totals.resolved + result.resolved,
+    droppedOutOfScope: Math.max(totals.droppedOutOfScope, result.droppedOutOfScope),
+    // Counted per SLICE, not per document: one call covers the slice, so one
+    // failure costs the slice. `modelIncompleteIds` carries the documents.
+    modelCalls: totals.modelCalls + (result.documentsProcessed > 0 && !result.modelFailure ? 1 : 0),
+    modelFailures: totals.modelFailures + (result.modelFailure ? 1 : 0),
+  };
+}
+
+// ---------------------------------------------------------------------------
+//  driving one to the end
+// ---------------------------------------------------------------------------
+
+/**
+ * Step until the corpus is done, the lease is lost, the budget runs out, or a
+ * step fails too many times.
+ *
+ * WHAT IT NEVER DOES IS THROW. A sweep that ends badly ends as a RECORD saying
+ * so — `status: 'failed'`, `error`, and a cursor still pointing at the work that
+ * is left. The caller of this is a fire-and-forget background task; an exception
+ * there goes nowhere a person will ever read it.
+ *
+ * THE LEASE IS RE-READ BETWEEN STEPS, and a driver that has lost it stops
+ * immediately. The tenant store has no compare-and-set, so this is a check
+ * rather than a guarantee — the guarantee is the layer underneath: identity is
+ * `(subject, action)`, mentions key on `(documentId, originKey)`, and the upsert
+ * is keyed on `external_id`, so even a genuine overlap converges rather than
+ * duplicating. The lease is what stops two drivers PAYING for the same
+ * documents, and it closes the one window — two model calls over one document
+ * wording the same item two ways — where convergence is not automatic.
+ */
+export async function driveExtractionRun(
+  run: ExtractionRun,
+  deps: ExtractionRunDeps,
+  options: ExtractionRunOptions = {},
+): Promise<ExtractionRun> {
+  const config = { ...RUN_DEFAULTS, ...options };
+  const now = options.now ?? Date.now;
+  const driver = run.leaseOwner ?? options.driverId ?? defaultDriverId();
+  const deadline = now() + config.maxRunMs;
+
+  let current = run;
+  let attempts = 0;
+
+  while (!TERMINAL.has(current.status)) {
+    if (now() > deadline) {
+      // Not a failure: everything done is written and the cursor is honest. A
+      // later start adopts it.
+      return current;
+    }
+
+    const latest = await deps.runs.readRun().catch(() => null);
+    if (latest && latest.runId === current.runId) {
+      if (TERMINAL.has(latest.status)) return latest;
+      if (latest.leaseOwner && latest.leaseOwner !== driver && isLeaseLive(latest, now())) {
+        // Somebody else is driving this sweep. Two drivers over one cursor is
+        // the duplication mode this whole record exists to prevent.
+        return latest;
+      }
+      current = { ...latest, leaseOwner: driver };
+    } else if (latest && latest.runId !== current.runId) {
+      // The workspace moved on to a different sweep. This driver's is over.
+      return current;
+    }
+
+    try {
+      current = await advanceExtractionRun(current, deps, { ...options, driverId: driver });
+      attempts = 0;
+    } catch (err) {
+      if (isAuthFailure(err)) {
+        // Hand the run back, still running, with no lease and a stated reason.
+        // Whoever presents a live credential next picks it up at the cursor.
+        const paused: ExtractionRun = {
+          ...current,
+          status: 'running',
+          leaseUntil: null,
+          updatedAt: new Date(now()).toISOString(),
+          error: 'The session that started this run expired before it finished. Resuming continues from where it stopped.',
+        };
+        return deps.runs.writeRun(paused).catch(() => paused);
+      }
+      attempts += 1;
+      if (attempts >= config.maxSliceAttempts) {
+        const failed: ExtractionRun = {
+          ...current,
+          status: 'failed',
+          // THE CURSOR IS NOT REWOUND. Everything already extracted stays
+          // extracted, and a resume starts at the document that failed.
+          error: (err as { message?: string })?.message || 'The sweep could not continue.',
+          updatedAt: new Date(now()).toISOString(),
+          finishedAt: new Date(now()).toISOString(),
+          leaseUntil: null,
+        };
+        return deps.runs.writeRun(failed).catch(() => failed);
+      }
+      if (config.retryDelayMs > 0) await sleep(config.retryDelayMs);
+    }
+  }
+
+  return current;
+}
+
+// ---------------------------------------------------------------------------
+
+function capped<T>(rows: T[], limit: number): T[] {
+  return rows.length > limit ? rows.slice(rows.length - limit) : rows;
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function randomSuffix(): string {
+  return Math.random().toString(36).slice(2, 8);
+}
+
+/**
+ * Identifies THIS process for the life of it.
+ *
+ * One value per module instance, so "did somebody else take the lease" is a
+ * meaningful question between replicas and between restarts of the same one.
+ */
+let processDriverId = '';
+export function defaultDriverId(): string {
+  if (!processDriverId) processDriverId = `driver-${randomSuffix()}${randomSuffix()}`;
+  return processDriverId;
+}
+
+export type { ExtractionModelFailure };