@startsimpli/integrations 0.2.14 → 0.2.17

package/src/corpus/types.ts

@@ -127,6 +127,19 @@ export interface SourceDocument {
   scope: CorpusScope;
   /** When the connector last wrote this row. Drives the freshness claim. */
   ingestedAt?: string | null;
+  /**
+   * Attribute names the SERVER withheld from this reader
+   * (bd startsim-768w.38.33). The tenant's per-field read gate strips them from
+   * `data` and names them back in `redacted_fields`; this carries that fact all
+   * the way to the screen and into the prompt.
+   *
+   * IT HAS TO TRAVEL, and that is the whole reason it is on the document rather
+   * than swallowed at the mapper. A record whose transcript was withheld is NOT
+   * a record with no transcript. Anything that cannot tell the two apart — a
+   * reader, or a model asked "was this discussed?" — answers "no" with total
+   * confidence about text it was simply never shown.
+   */
+  withheldFields?: string[];
 }
 
 /**
@@ -299,6 +312,14 @@ export interface AskResult {
    * returned something it should not have, and is worth seeing.
    */
   droppedOutOfScope: number;
+  /**
+   * Attribute names the server withheld from this reader on at least one record
+   * that was considered (bd startsim-768w.38.33). Reported for exactly the
+   * reason `droppedOutOfScope` is: an answer built from a partial record has to
+   * say so, or "your corpus has nothing about that" and "you were not shown the
+   * part that does" become the same sentence.
+   */
+  withheldFields?: string[];
   /** Which provider/model actually answered, and whether it was a real one. */
   provider: LlmDescription;
   /** Freshness at answer time — an answer over a stale corpus says so. */
@@ -323,6 +344,13 @@ export interface PromptDocument {
   occurredAt: string | null;
   participants: string[];
   text: string;
+  /**
+   * Fields the reader was not allowed to see (bd startsim-768w.38.33). Rendered
+   * into the document block so the model states a limit instead of inventing a
+   * certainty: without it, a summary-only view of a meeting whose transcript was
+   * withheld reads as proof the thing was never discussed.
+   */
+  withheldFields?: string[];
 }
 
 export interface AnswerRequest {
@@ -627,8 +655,14 @@ export interface ExtractionDraft {
  */
 export interface CommitmentTrackerApi extends CommitmentApi {
   /**
-   * Run extraction over documents that have not been processed. Optional: a
-   * read-only deployment omits it and the control does not render.
+   * Run ONE slice of extraction and wait for it.
+   *
+   * KEPT, but no longer what the button uses. It is the primitive a job is made
+   * of, and it is what a deployment small enough to finish inside a request can
+   * still call directly. Anything with a real corpus must use the pair below:
+   * one document is about twenty seconds of model time, so a seventeen-document
+   * sweep is six minutes and no slice size fits a browser request
+   * (bd startsim-768w.38.28).
    */
   extract?(input: {
     scope: CorpusScope;
@@ -637,11 +671,170 @@ export interface CommitmentTrackerApi extends CommitmentApi {
     offset?: number;
     limit?: number;
   }): Promise<ExtractionResult>;
+
+  /**
+   * Start — or resume — a sweep, and return WITHOUT doing the work.
+   *
+   * The composer polls `getExtractionRun` for progress. Starting must be cheap
+   * and must return a durable run, because the whole point is that the thing
+   * doing the work outlives the request that asked for it.
+   */
+  startExtraction?(input: {
+    scope: CorpusScope;
+    /** Ignore a resumable sweep and begin again from the first document. */
+    restart?: boolean;
+  }): Promise<ExtractionRun>;
+
+  /** The current sweep, from the server's own record of it. */
+  getExtractionRun?(): Promise<ExtractionRun | null>;
+}
+
+/**
+ * WHY THE MODEL HALF DID NOT RUN.
+ *
+ * This type exists because of a specific lie. `extractCommitments` deliberately
+ * wraps the model call so that a provider outage costs the extras and not the
+ * list — which is right — but the old code turned every such failure into
+ * `fromModel: 0` and nothing else, and `fromModel: 0` reads exactly like "the
+ * model found nothing to add". On the live tandem tenant that meant a week of
+ * rules-only runs being reported as model runs, with a blown sixty-second client
+ * timeout as the invisible cause. A degraded run is fine. A degraded run that
+ * presents itself as a complete one is not.
+ */
+export interface ExtractionModelFailure {
+  /** Which model call failed — proposing items, or checking them. */
+  stage: 'extract' | 'verify';
+  message: string;
+  /**
+   * Whether the provider client's own deadline was hit, as opposed to a 401, a
+   * 429 or a bug. Worth separating because it is the only one the operator fixes
+   * by giving the run more time rather than by fixing a credential.
+   */
+  timedOut: boolean;
+}
+
+export type ExtractionRunStatus = 'queued' | 'running' | 'succeeded' | 'failed' | 'cancelled';
+
+/** Everything a sweep has accumulated so far, summed across its slices. */
+export interface ExtractionRunTotals {
+  documentsProcessed: number;
+  fromRules: number;
+  fromModel: number;
+  ungrounded: number;
+  unconfirmed: number;
+  created: number;
+  merged: number;
+  resolved: number;
+  droppedOutOfScope: number;
+  /** Slices whose model half answered. */
+  modelCalls: number;
+  /** Slices whose model half did NOT answer. The number that used to be hidden. */
+  modelFailures: number;
+}
+
+/** A finished sweep, kept so the screen can say what the last one did. */
+export interface ExtractionRunSummary {
+  runId: string;
+  status: ExtractionRunStatus;
+  startedAt: string;
+  finishedAt: string | null;
+  totals: ExtractionRunTotals;
+  error: string | null;
+}
+
+/**
+ * A SWEEP, as a durable record rather than as a request.
+ *
+ * Stored in the tenant's own generic entity store, which is what makes every
+ * property this design owes true at once: partial progress survives the process
+ * that made it, a poll from any replica sees the same run, and a driver that
+ * dies leaves a cursor rather than a hole. Nothing about it is Tandem-specific
+ * or meeting-specific — it is "a long, resumable, idempotent pass over the
+ * corpus", which is what every foundry's connector framework eventually needs.
+ */
+export interface ExtractionRun {
+  /** Identifies this SWEEP. The row it lives in is a singleton per workspace. */
+  runId: string;
+  /**
+   * The workspace the sweep is over.
+   *
+   * Stored, because a background driver has no request to read it from once the
+   * request that started it has returned — that is the whole point of the
+   * record. It cannot be forged: the row lives in the tenant entity store, which
+   * only ever returns rows in the caller's own org, and every slice re-runs
+   * `enforceScope` over the documents it read anyway. Adopting a stalled sweep
+   * overwrites it with the adopter's freshly proven scope.
+   */
+  scope: CorpusScope;
+  status: ExtractionRunStatus;
+  /**
+   * The next document offset to process. THE resumption mechanism: a sweep
+   * interrupted at nine of seventeen restarts at ten, not at one.
+   */
+  cursor: number;
+  /** In-scope documents at the last slice, so progress is a fraction of something. */
+  totalDocuments: number;
+  totals: ExtractionRunTotals;
+  /**
+   * Documents this sweep has already handed to the extractor, bounded.
+   *
+   * The cursor is an offset into a list whose order can shift when a document
+   * arrives mid-sweep. Recording the ids is what lets the run notice that it
+   * finished with fewer documents read than the corpus holds and SAY so, rather
+   * than reporting a complete sweep that quietly skipped one.
+   */
+  processedIds: string[];
+  /** Documents that got the deterministic pass only, because the model did not answer. */
+  modelIncompleteIds: string[];
+  needsReview: Array<{ commitmentId: string; documentId: string; quote: string }>;
+  provider: LlmDescription;
+  startedAt: string;
+  updatedAt: string;
+  finishedAt: string | null;
+  /**
+   * WHO IS DRIVING, AND UNTIL WHEN.
+   *
+   * Not an optimisation. Two drivers over one cursor means two model calls over
+   * the same document, and two model calls over one document produce two
+   * differently-worded items that identity resolution cannot see are the same —
+   * which is precisely how a live re-run once turned 82 commitments into 98.
+   */
+  leaseOwner: string | null;
+  leaseUntil: string | null;
+  /** Why the sweep stopped, when it stopped badly. */
+  error: string | null;
+  lastModelFailure: ExtractionModelFailure | null;
+  /** Previous sweeps, newest first, bounded. */
+  history: ExtractionRunSummary[];
+  /** True when this start adopted a sweep whose driver had gone away. Transient. */
+  adopted?: boolean;
+}
+
+/**
+ * Where a run record lives. ONE ROW PER WORKSPACE, rewritten in place.
+ *
+ * A row per sweep would be more history, and it would also be unbounded growth
+ * in the tenant's own corpus plus a scan on every poll — the tenant entity API
+ * filters on `?type=` and nothing else, so "the newest run" cannot be asked for
+ * directly. A singleton keyed on a fixed `external_id` is one deterministic
+ * read, never grows, and keeps the last few sweeps inline where the screen
+ * actually wants them.
+ */
+export interface ExtractionRunStore {
+  readRun(): Promise<ExtractionRun | null>;
+  writeRun(run: ExtractionRun): Promise<ExtractionRun>;
 }
 
 export interface ExtractionResult {
   /** Documents read and sent to the extractor IN THIS CALL. */
   documentsProcessed: number;
+  /**
+   * Which ones, by id. A sweep needs this to record what it has really covered
+   * — an offset cursor alone cannot tell a resumed run whether the list moved
+   * under it — and the screen needs it to name the meetings a degraded slice
+   * only half-read.
+   */
+  documentIds: string[];
   /** Documents in scope altogether, so a partial sweep can say what it is of. */
   totalDocuments: number;
   /**
@@ -684,6 +877,15 @@ export interface ExtractionResult {
   needsReview: Array<{ commitmentId: string; documentId: string; quote: string }>;
   droppedOutOfScope: number;
   provider: LlmDescription;
+  /**
+   * `null` when the model half ran. Populated when it did not, WITH THE REASON.
+   *
+   * Read this before reading `fromModel`. Zero findings and a failed call are
+   * different facts and only one of them is about the corpus; conflating them is
+   * what let a rules-only run be reported as a model run for a week
+   * (bd startsim-768w.38.28).
+   */
+  modelFailure: ExtractionModelFailure | null;
 }
 
 /**