@startsimpli/integrations 0.2.14 → 0.2.17

package/src/components/pages/TrackerPage.tsx

@@ -26,6 +26,7 @@
  */
 import * as React from 'react';
 
+import { isStalled, runCoverage, runProgress } from '../../corpus/runs';
 import { stripMarkdown } from '../Markdown';
 import type {
   Commitment,
@@ -34,6 +35,7 @@ import type {
   CommitmentTrackerApi,
   CorpusScope,
   ExtractionResult,
+  ExtractionRun,
 } from '../../corpus/types';
 
 export interface TrackerPageProps {
@@ -51,12 +53,29 @@ export interface TrackerPageProps {
    * supply a clock, one is frozen ONCE for the life of the component.
    */
   now?: number;
+  /**
+   * How often the queued sweep is polled while it is going.
+   *
+   * A six-minute job does not need a second-by-second poll, and the tenant data
+   * API pays for every one of them. Three seconds is fast enough that the bar
+   * moves and slow enough that watching costs nothing.
+   */
+  pollMs?: number;
+  /**
+   * How many times a stalled sweep may be picked back up without anybody
+   * pressing anything, per mount.
+   *
+   * Bounded so a workspace that cannot be reached ends up stopped and legible
+   * rather than retrying forever behind an open tab. Twenty-four stints is well
+   * over a day of ten-minute sweeps.
+   */
+  maxAutoResumes?: number;
   classNames?: Partial<Record<TrackerSlot, string>>;
 }
 
 type TrackerSlot =
   | 'root' | 'section' | 'heading' | 'card' | 'badge' | 'button' | 'secondaryButton'
-  | 'input' | 'muted' | 'error' | 'warning' | 'row' | 'due';
+  | 'input' | 'muted' | 'error' | 'warning' | 'row' | 'due' | 'progressTrack' | 'progressBar';
 
 const DEFAULTS: Record<TrackerSlot, string> = {
   root: 'space-y-6',
@@ -72,6 +91,8 @@ const DEFAULTS: Record<TrackerSlot, string> = {
   warning: 'rounded-md border border-amber-300 bg-amber-50 p-3 text-sm text-amber-900',
   row: 'rounded-lg border border-gray-200 p-4',
   due: 'text-sm font-medium text-red-700',
+  progressTrack: 'h-2 w-full overflow-hidden rounded-full bg-gray-200',
+  progressBar: 'h-2 rounded-full bg-gray-900 transition-all',
 };
 
 const STATUS_TABS: Array<{ key: 'open' | 'overdue' | 'recent' | 'done'; label: string }> = [
@@ -83,14 +104,20 @@ const STATUS_TABS: Array<{ key: 'open' | 'overdue' | 'recent' | 'done'; label: s
 
 const WEEK_MS = 7 * 24 * 3600 * 1000;
 
+/** States the poll stops at. A finished run is a record, not a subscription. */
+const TERMINAL_RUN = new Set(['succeeded', 'failed', 'cancelled']);
+
 export function TrackerPage({
   api,
   scope,
   onOpenDocument,
   now,
+  pollMs = 3_000,
+  maxAutoResumes = 24,
   classNames = {},
 }: TrackerPageProps) {
   const cn = { ...DEFAULTS, ...classNames };
+  const [sweepEpoch, setSweepEpoch] = React.useState(0);
 
   // Frozen once. See the note on the prop.
   const [mountedAt] = React.useState(() => Date.now());
@@ -113,6 +140,18 @@ export function TrackerPage({
   const [error, setError] = React.useState('');
   const [run, setRun] = React.useState<ExtractionResult | null>(null);
   const [running, setRunning] = React.useState(false);
+  /**
+   * The SWEEP, when this deployment runs extraction as a job
+   * (bd startsim-768w.38.28).
+   *
+   * `run` above is one synchronous slice's result and is what a deployment small
+   * enough to finish inside a request still gets. This is the durable record of a
+   * long one, polled from the server. They are different objects because they
+   * describe different things — a result, and a job — and collapsing them would
+   * put a cursor on something that has no cursor.
+   */
+  const [sweep, setSweep] = React.useState<ExtractionRun | null>(null);
+  const [starting, setStarting] = React.useState(false);
   /**
    * Does this workspace hold ANY commitments at all?
    *
@@ -168,6 +207,98 @@ export function TrackerPage({
   // Real input only: the query the user asked for, and the workspace they are in.
   React.useEffect(() => { void load(); }, [load, scope.orgId]);
 
+  /**
+   * WATCH THE SWEEP, and pick one up that was already going.
+   *
+   * The poll starts on mount rather than on the button, because the run outlives
+   * the tab that started it: somebody who closes this page and comes back has to
+   * see the job that is still going, not a fresh idle screen with a button that
+   * would start a second one. Polling STOPS the moment the run reaches a
+   * terminal state — a finished job is a record, not a subscription.
+   *
+   * `now` is read from the frozen clock, never from `Date.now()` in a dependency
+   * array: that is what turned this component into 917 requests in ten minutes
+   * once already.
+   */
+  const canQueue = Boolean(api.startExtraction && api.getExtractionRun);
+  /**
+   * The reader, and the last state we saw, held in REFS.
+   *
+   * Neither is an input to the poll. `load` changes whenever the tab or the
+   * owner filter changes, and a poll that restarts on a tab click is a poll that
+   * fires an extra request per click; the last-seen status is compared inside
+   * the tick rather than through a state updater, because a side effect inside a
+   * `setState` updater runs twice under StrictMode. Same discipline as the
+   * frozen clock above, and for the same reason: this component has already
+   * shipped one runaway.
+   */
+  const loadRef = React.useRef(load);
+  loadRef.current = load;
+  const clockRef = React.useRef(clock);
+  clockRef.current = clock;
+  const scopeRef = React.useRef(scope);
+  scopeRef.current = scope;
+  const lastSeen = React.useRef<{ runId: string; status: string } | null>(null);
+  const resumes = React.useRef(0);
+
+  React.useEffect(() => {
+    if (!canQueue) return undefined;
+    let live = true;
+    let timer: ReturnType<typeof setTimeout> | undefined;
+
+    const tick = async () => {
+      if (!live) return;
+      const latest = await apiRef.current.getExtractionRun!().catch(() => null);
+      if (!live) return;
+      if (latest) {
+        const previous = lastSeen.current;
+        lastSeen.current = { runId: latest.runId, status: latest.status };
+        setSweep(latest);
+        // A sweep that has just reached the end is a reason to re-read the list:
+        // the rows it wrote are the entire point of having watched it.
+        const settled = latest.status === 'succeeded' || latest.status === 'failed';
+        if (settled && (!previous || previous.runId !== latest.runId || previous.status !== latest.status)) {
+          void loadRef.current();
+        }
+      }
+      /**
+       * A STALLED RUN IS PICKED BACK UP, and this is the one place the browser
+       * is allowed near the job.
+       *
+       * It is NOT the scheduler: the server owns the cursor, the ordering, the
+       * lease and the work, and closing this tab does not lose any of it. What
+       * the browser uniquely has is a LIVE CREDENTIAL. A sweep is about ten
+       * minutes and the token that starts it lives thirty, so a run that began
+       * late in a session stops mid-way with its cursor intact; re-presenting a
+       * fresh bearer is all it needs to carry on, and asking somebody to click
+       * for that would be asking them to do the machine's bookkeeping.
+       *
+       * Bounded by a ref, not by state, so it can never drive a render. A run
+       * that FAILED is not auto-resumed — it said why it stopped and a person
+       * should read that before it is tried again.
+       */
+      if (
+        latest &&
+        !TERMINAL_RUN.has(latest.status) &&
+        isStalled(latest, clockRef.current) &&
+        resumes.current < maxAutoResumes
+      ) {
+        resumes.current += 1;
+        await apiRef.current.startExtraction!({ scope: scopeRef.current }).catch(() => undefined);
+      }
+
+      const busy = latest ? latest.status === 'queued' || latest.status === 'running' : false;
+      if (busy) timer = setTimeout(() => { void tick(); }, pollMs);
+    };
+
+    void tick();
+    return () => {
+      live = false;
+      if (timer) clearTimeout(timer);
+    };
+    // `sweepEpoch` re-arms the poll after the button starts a new one.
+  }, [canQueue, pollMs, maxAutoResumes, scope.orgId, sweepEpoch]);
+
   /**
    * The tab, applied in the browser as well as sent to the server.
    *
@@ -194,6 +325,11 @@ export function TrackerPage({
     return [...byOwner].sort(compareForScreen(clock));
   }, [rows, tab, owner, clock]);
 
+  const sweepBusy = Boolean(sweep && (sweep.status === 'queued' || sweep.status === 'running'));
+  // A run whose lease has lapsed is not running, whatever its status field says.
+  // Rendering it as "in progress" is a spinner that never ends.
+  const sweepStalled = Boolean(sweep && isStalled(sweep, clock));
+
   const unfiltered = (['status', 'owner', 'dueBefore', 'changedAfter'] as const).filter(
     (key) => Boolean(query[key]) && !applied.includes(key),
   );
@@ -225,7 +361,41 @@ export function TrackerPage({
               onChange={(event) => setOwner(event.target.value)}
             />
           </label>
-          {api.extract ? (
+          {canQueue ? (
+            /**
+             * THE BUTTON NO LONGER WAITS, because it cannot: seventeen meetings
+             * is about six minutes of model time and the tenant gateway closes a
+             * request at sixty seconds. It starts the job and the screen watches
+             * it (bd startsim-768w.38.28).
+             */
+            <button
+              type="button"
+              className={cn.secondaryButton}
+              disabled={starting || sweepBusy}
+              onClick={async () => {
+                setStarting(true);
+                setError('');
+                try {
+                  const started = await apiRef.current.startExtraction!({ scope });
+                  setSweep(started);
+                  setSweepEpoch((epoch) => epoch + 1);
+                } catch (err) {
+                  setError(failureMessage(err));
+                } finally {
+                  setStarting(false);
+                }
+              }}
+            >
+              {sweepStalled
+                ? 'Resume reading your meetings'
+                : sweepBusy
+                  ? 'Reading your meetings…'
+                  : 'Pull from meetings'}
+            </button>
+          ) : api.extract ? (
+            // A deployment with no run store still gets the old synchronous
+            // call. It is the same primitive; it just has nowhere to keep a
+            // cursor, so it is only honest on a corpus that fits in a request.
             <button
               type="button"
               className={cn.secondaryButton}
@@ -260,6 +430,9 @@ export function TrackerPage({
         ) : null}
 
         {error ? <p className={cn.error} role="alert">{error}</p> : null}
+        {sweep ? (
+          <SweepPanel sweep={sweep} now={clock} classNames={cn} onOpenDocument={onOpenDocument} />
+        ) : null}
         {run ? <RunSummary result={run} classNames={cn} onOpenDocument={onOpenDocument} /> : null}
       </section>
 
@@ -380,6 +553,157 @@ function CommitmentRow({
   );
 }
 
+/**
+ * A SWEEP IN FLIGHT, and afterwards what it did (bd startsim-768w.38.28).
+ *
+ * The old button blocked on a call that could not finish, so the only states it
+ * ever had were "spinning" and "504". A six-minute job has four states a person
+ * actually needs, and every one of them is a sentence rather than a spinner:
+ *
+ *   GOING     how far, out of what, and that it survives leaving the page. A
+ *             progress bar with no denominator is a spinner with extra steps.
+ *   STALLED   the driver went away — a container replacement, usually. Said out
+ *             loud with a Resume, because a sweep that silently stops looking
+ *             busy is the worst of the four.
+ *   DONE      what it wrote, AND whether the model actually answered.
+ *   FAILED    why, and what it kept. Partial progress is progress and nobody
+ *             should start over by hand because the screen did not say so.
+ *
+ * THE MODEL LINE IS THE POINT OF THE BEAD. `fromModel: 0` used to be the entire
+ * report of a blown provider timeout, and it reads exactly like "the model found
+ * nothing to add". It is not allowed to render that way any more: if the model
+ * did not answer, this says so, says it timed out when it did, and names how many
+ * meetings got only the deterministic pass.
+ */
+function SweepPanel({
+  sweep, now, classNames: cn, onOpenDocument,
+}: {
+  sweep: ExtractionRun;
+  now: number;
+  classNames: Record<TrackerSlot, string>;
+  onOpenDocument?: (documentId: string) => void;
+}) {
+  const progress = runProgress(sweep);
+  const coverage = runCoverage(sweep);
+  const stalled = isStalled(sweep, now);
+  const busy = (sweep.status === 'queued' || sweep.status === 'running') && !stalled;
+  const totals = sweep.totals;
+  const failure = sweep.lastModelFailure;
+
+  return (
+    <div className={cn.card} role="status" aria-live="polite">
+      {busy || stalled ? (
+        <>
+          <p className="text-sm text-gray-800">
+            Reading your meetings — {progress.done} of {progress.total} meetings
+            {totals.created > 0 ? `, ${totals.created} items so far` : ''}.
+          </p>
+          <div className={`mt-2 ${cn.progressTrack}`}>
+            <div className={cn.progressBar} style={{ width: `${Math.max(progress.percent, 2)}%` }} />
+          </div>
+        </>
+      ) : null}
+
+      {busy ? (
+        <p className={`mt-2 ${cn.muted}`}>
+          {/* The single most useful sentence on the screen: it removes the
+              reason somebody sits and watches, or presses the button again. */}
+          This keeps going if you close this page. Come back and it will still be here.
+        </p>
+      ) : null}
+
+      {stalled ? (
+        <p className={`mt-2 ${cn.warning}`}>
+          This run stopped before it finished — the process that was doing the work went
+          away. Everything it had already read is saved. Resume picks up at meeting{' '}
+          {progress.done + 1}, not at the beginning.
+        </p>
+      ) : null}
+
+      {sweep.status === 'succeeded' ? (
+        <p className="text-sm text-gray-800">
+          Read {totals.documentsProcessed} meeting{totals.documentsProcessed === 1 ? '' : 's'}:{' '}
+          {totals.created} new, {totals.merged} matched to something already tracked,{' '}
+          {totals.resolved} closed by a later meeting.
+        </p>
+      ) : null}
+
+      {sweep.status === 'failed' ? (
+        <p className={cn.error}>
+          This run stopped: {sweep.error || 'the workspace could not be reached'}. It kept
+          everything it had already read — {progress.done} of {progress.total} meetings,{' '}
+          {totals.created} items. Running it again carries on from there.
+        </p>
+      ) : null}
+
+      {/* WHAT THE MODEL DID, AND WHETHER IT ANSWERED AT ALL. */}
+      {sweep.status !== 'queued' && totals.documentsProcessed > 0 ? (
+        <p className={cn.muted}>
+          {totals.fromRules} came from the meetings’ own next-step lists, {totals.fromModel} were
+          read out of the notes by{' '}
+          {sweep.provider.live
+            ? `${sweep.provider.provider} · ${sweep.provider.model}`
+            : 'no live model'}
+          .
+        </p>
+      ) : null}
+
+      {totals.modelFailures > 0 ? (
+        <p className={`mt-2 ${cn.warning}`} role="status">
+          {/* NOT `fromModel: 0` on its own. That reading — "nothing to add" —
+              is what hid a week of degraded runs. */}
+          On {totals.modelFailures} of {totals.modelFailures + totals.modelCalls} passes the model
+          did not answer
+          {failure?.timedOut ? ' — it timed out' : failure ? ` — ${failure.message}` : ''}. Those{' '}
+          {sweep.modelIncompleteIds.length} meeting
+          {sweep.modelIncompleteIds.length === 1 ? '' : 's'} were read by the deterministic pass
+          only, so anything stated in the notes but not written into a next-step list is still
+          missing from them. Running it again gives the model another go; nothing is duplicated.
+        </p>
+      ) : null}
+
+      {sweep.status === 'succeeded' && coverage.missed > 0 ? (
+        <p className={`mt-2 ${cn.warning}`}>
+          {/* A document that arrived mid-sweep shifts the offsets under it. Said
+              out loud rather than reported as a complete pass. */}
+          {coverage.missed} meeting{coverage.missed === 1 ? '' : 's'} arrived while this was
+          running and were not read. Run it again to pick them up.
+        </p>
+      ) : null}
+
+      {sweep.needsReview.length ? (
+        <div className={`mt-2 ${cn.warning}`} role="status">
+          <p>
+            {sweep.needsReview.length} meeting statement
+            {sweep.needsReview.length === 1 ? '' : 's'} looked like they closed something, but not
+            closely enough to act on. Nothing was closed automatically.
+          </p>
+          <ul className="mt-1 space-y-1">
+            {sweep.needsReview.map((entry) => (
+              <li key={`${entry.commitmentId}:${entry.documentId}`}>
+                <button
+                  type="button"
+                  className="text-sm underline"
+                  onClick={() => onOpenDocument?.(entry.documentId)}
+                >
+                  “{stripMarkdown(entry.quote).slice(0, 120)}”
+                </button>
+              </li>
+            ))}
+          </ul>
+        </div>
+      ) : null}
+
+      {totals.droppedOutOfScope > 0 ? (
+        <p className={cn.error} role="status">
+          {totals.droppedOutOfScope} record{totals.droppedOutOfScope === 1 ? '' : 's'} from another
+          workspace were returned by the data API and excluded before extracting.
+        </p>
+      ) : null}
+    </div>
+  );
+}
+
 /**
  * What the last run did, in the terms that make it arguable.
  *

package/src/corpus/ask.ts

@@ -75,6 +75,7 @@ function systemPrompt(mode: AskRequest['mode']): string {
     'You answer questions about a company using only that company\'s own records.',
     ...shared,
     'Be specific and short. Prefer naming the record and the date over hedging.',
+    'A document may list "withheld_from_this_reader". Those parts were deliberately not shown to you: never claim something was not said or did not happen on the basis of a document that lists them — say the record is partly withheld instead.',
   ].join('\n');
 }
 
@@ -94,6 +95,7 @@ export function buildAnswerRequest(
     occurredAt: document.occurredAt,
     participants: document.participants.map((party) => party.name),
     text: documentText(document, maxChars),
+    ...(document.withheldFields?.length ? { withheldFields: document.withheldFields } : {}),
   }));
   return {
     question: request.question,
@@ -126,6 +128,13 @@ export function renderAnswerPrompt(input: AnswerRequest): string {
       `source: ${document.source}`,
       `occurred_at: ${document.occurredAt ?? 'unknown'}`,
       document.participants.length ? `participants: ${document.participants.join(', ')}` : '',
+      // The model is told what it was NOT given. Otherwise a summary-only view of
+      // a meeting whose transcript was withheld is indistinguishable from a
+      // meeting where the thing was never said, and the answer comes back as a
+      // confident negative about text nobody was allowed to show it.
+      document.withheldFields?.length
+        ? `withheld_from_this_reader: ${document.withheldFields.join(', ')}`
+        : '',
       '',
       document.text,
       '</document>',
@@ -279,6 +288,8 @@ export async function askCorpus(
     };
   }
 
+  const withheldFields = collectWithheld(retrieved);
+
   const draft = await llm.answer(
     buildAnswerRequest(request, retrieved, maxCharsPerDocument),
   );
@@ -294,11 +305,21 @@ export async function askCorpus(
     considered: retrieved,
     unresolvedCitations: unresolved,
     droppedOutOfScope,
+    ...(withheldFields.length ? { withheldFields } : {}),
     provider: llm.describe(),
     freshness,
   };
 }
 
+/** Every field withheld on any record that reached the prompt, deduped. */
+function collectWithheld(retrieved: RetrievedDocument[]): string[] {
+  const names = new Set<string>();
+  for (const item of retrieved) {
+    for (const name of item.document.withheldFields ?? []) names.add(name);
+  }
+  return [...names].sort();
+}
+
 /**
  * `uncited` is a real outcome, not a failure to report. The model produced text
  * and nothing traceable; the surface renders that as a warning over the text

package/src/corpus/client.ts

@@ -27,10 +27,13 @@
  */
 import {
   DEFAULT_PARTICIPANT_EDGES,
+  DEFAULT_RUN_KEY,
   DEFAULT_TYPE_KEYS,
   commitmentToEntity,
+  extractionRunToEntity,
   indexParticipants,
   toCommitment,
+  toExtractionRun,
   toMoneyEvent,
   toParty,
   toSourceDocument,
@@ -44,6 +47,8 @@ import type {
   CorpusPage,
   CorpusScope,
   DocumentQuery,
+  ExtractionRun,
+  ExtractionRunStore,
   MoneyEvent,
   Party,
   PartyRef,
@@ -105,6 +110,11 @@ export interface CreateCorpusApiOptions {
   maxGraphPages?: number;
   /** How long a loaded graph is reused before it is read again. */
   graphTtlMs?: number;
+  /**
+   * The `external_id` the singleton extraction-run row lives at. One per
+   * workspace; see `ExtractionRunStore` for why it is a singleton.
+   */
+  runKey?: string;
 }
 
 interface DrfPage<T> {
@@ -236,7 +246,9 @@ function appliedFilters(
   return requested.filter((key) => supported.includes(key));
 }
 
-export function createCorpusApi(options: CreateCorpusApiOptions): CorpusApi & CommitmentApi {
+export function createCorpusApi(
+  options: CreateCorpusApiOptions,
+): CorpusApi & CommitmentApi & ExtractionRunStore {
   const {
     http,
     scope,
@@ -249,6 +261,7 @@ export function createCorpusApi(options: CreateCorpusApiOptions): CorpusApi & Co
     graphTtlMs = 30_000,
   } = options;
   const types: CorpusTypeKeys = { ...DEFAULT_TYPE_KEYS, ...options.typeKeys };
+  const runKey = options.runKey ?? DEFAULT_RUN_KEY;
   const entities = options.paths?.entities ?? 'api/v1/entities';
   const connections = options.paths?.connections ?? 'api/v1/connections';
   const relationships = options.paths?.relationships ?? 'api/v1/relationships';
@@ -427,6 +440,49 @@ export function createCorpusApi(options: CreateCorpusApiOptions): CorpusApi & Co
       }
     },
 
+    /**
+     * The sweep record, read back (bd startsim-768w.38.28).
+     *
+     * ONE ROW PER WORKSPACE, at a fixed `external_id`, because the tenant entity
+     * API filters on `?type=` and nothing else — there is no way to ask for "the
+     * newest run", so a row per sweep would mean scanning a list that grows
+     * forever on every poll. A singleton is one deterministic read that never
+     * grows, and the last few sweeps ride along inside it.
+     *
+     * FAIL-SOFT ON ABSENCE, NOT ON FAILURE. No run yet is `null`, which is a
+     * fact. A 5xx is thrown, because "we could not reach the workspace" and
+     * "nothing has ever run" are different answers and the screen renders them
+     * differently.
+     */
+    async readRun() {
+      const response = await http
+        .get<DrfPage<TenantEntity>>(entities, {
+          params: { type: types.extractionRun, page: 1, page_size: 5 },
+        })
+        .catch((err: { status?: number }) => {
+          // An undeclared type is a 403 from the read gate, and it means exactly
+          // "no sweep has ever run here" — the type is declared by the first one.
+          if (err?.status === 403 || err?.status === 404) return EMPTY_PAGE as DrfPage<TenantEntity>;
+          throw err;
+        });
+      const row = (response.results ?? []).find(
+        (entity) => String(entity.external_id ?? '') === runKey,
+      );
+      return row ? toExtractionRun(row) : null;
+    },
+
+    async writeRun(run: ExtractionRun) {
+      if (!http.post) {
+        throw new Error('This corpus client is read-only — no write method was supplied.');
+      }
+      await http.post(`${entities}/upsert`, extractionRunToEntity(run, types.extractionRun, runKey));
+      // The record just written is the record, not whatever the server chose to
+      // echo — the upsert response shape is the tenant's, and re-parsing it here
+      // would make every field depend on a transform this package does not own.
+      const { adopted: _adopted, ...durable } = run;
+      return durable as ExtractionRun;
+    },
+
     async listSources(): Promise<SourceFreshness[]> {
       const [connectionPage, documentPage] = await Promise.all([
         http