@startsimpli/integrations 0.2.14 → 0.2.17

package/src/__tests__/corpus-runs.test.ts

@@ -0,0 +1,631 @@
+/**
+ * EXTRACTION AS A JOB, not as a request (bd startsim-768w.38.28).
+ *
+ * The measurement that forced this: on the live tandem tenant `POST
+ * /corpus/extract` returns 504 at EVERY slice size, `limit=1` included, because
+ * one document takes about twenty seconds against claude-sonnet-5 and there are
+ * two independent sixty-second walls in front of it — the tenant gateway's, and
+ * the provider client's own. Seventeen documents is roughly six minutes of model
+ * time. No slice size fits a browser request, so no synchronous design works.
+ *
+ * This file tests the four properties that make an asynchronous one worth having,
+ * in the order in which getting them wrong destroys the product:
+ *
+ *   1. RESUMPTION. A sweep interrupted at document nine keeps those nine and
+ *      starts at ten. A "resumable" job that redoes finished work is a job that
+ *      never finishes on a corpus large enough to need one.
+ *   2. IDEMPOTENCE UNDER RETRY. A queue retries. A retry that duplicates is
+ *      worse than a queue that gives up — and this exact failure is on the
+ *      record: a second live run created sixteen duplicates (82 → 98) because
+ *      the model reworded its own subject.
+ *   3. LEGIBLE FAILURE. A run whose model call timed out has to SAY so. The old
+ *      code swallowed it and reported `fromModel: 0`, which reads exactly like
+ *      "the model found nothing to add" — that lie is what hid this bug for a
+ *      week, so killing it is part of the fix, not a nicety.
+ *   4. ONE DRIVER. Two loops over the same cursor race, and two model calls over
+ *      the same document produce two differently-worded items that dedupe cannot
+ *      see. The lease is a correctness mechanism, not an optimisation.
+ */
+import { describe, expect, it, vi } from 'vitest';
+
+import {
+  advanceExtractionRun,
+  driveExtractionRun,
+  isLeaseLive,
+  isStalled,
+  newExtractionRun,
+  runProgress,
+  startExtractionRun,
+} from '../corpus/runs';
+import {
+  createFixtureCorpusApi,
+  createFixtureExtractor,
+  createFixtureRunStore,
+  FIXTURE_ORG,
+  FIXTURE_TRACKED_DOCUMENTS,
+} from '../corpus/fixtures';
+import { extractCommitments } from '../corpus/extract';
+import type { Commitment, CommitmentApi, CorpusApi, CorpusExtractorLlm } from '../corpus/types';
+
+const TANDEM = { orgId: FIXTURE_ORG };
+
+/**
+ * A corpus that stores what it is given, so a sweep's effect on the workspace is
+ * observable — which is the only way "run it twice and the count holds" can be
+ * asserted rather than asserted about.
+ */
+function trackingCorpus(options: { documents?: typeof FIXTURE_TRACKED_DOCUMENTS } = {}) {
+  const corpus = createFixtureCorpusApi({
+    scope: TANDEM,
+    documents: options.documents ?? FIXTURE_TRACKED_DOCUMENTS,
+  }) as CorpusApi & CommitmentApi;
+  const tracked = new Map<string, Commitment>();
+
+  corpus.listCommitments = async () => ({
+    count: tracked.size,
+    next: null,
+    previous: null,
+    results: [...tracked.values()],
+  });
+  corpus.saveCommitments = async (items: Commitment[]) => {
+    for (const item of items) tracked.set(item.externalId, item);
+  };
+
+  return { corpus, tracked };
+}
+
+/**
+ * A model that proposes the SAME thing every time, whatever it is told is
+ * already found.
+ *
+ * The stock fixture extractor deliberately skips anything in ALREADY_FOUND and
+ * reaches for the next bullet instead, which makes a second sweep DISCOVER more
+ * — legitimate behaviour, and useless for testing duplication, because a growing
+ * count is then the correct answer. These two extractors hold the findings fixed
+ * so that any growth at all is a duplicate.
+ */
+function stableExtractor(): CorpusExtractorLlm {
+  return {
+    describe: () => ({ provider: 'fixture', model: 'stable', live: false }),
+    async extract(input) {
+      return {
+        items: input.documents.map((document) => ({
+          documentId: document.id,
+          subtype: 'action',
+          title: 'Send the revised schedule to the client',
+          detail: '',
+          status: 'open',
+          subject: 'schedule',
+          quote: firstSentence(document.text),
+        })),
+        resolutions: [],
+      };
+    },
+  };
+}
+
+/**
+ * THE PRODUCTION FAILURE MODE, reproduced.
+ *
+ * On the live tandem tenant a second run created sixteen duplicates — 82 rows
+ * became 98 — for one reason: the model reworded its own subject, so the item it
+ * had already written did not resolve to the same identity. This extractor does
+ * exactly that, and the count still has to hold.
+ */
+function rewordingExtractor(): CorpusExtractorLlm {
+  return {
+    describe: () => ({ provider: 'fixture', model: 'rewording', live: false }),
+    async extract(input) {
+      return {
+        items: input.documents.map((document) => ({
+          documentId: document.id,
+          subtype: 'action',
+          title: 'Send the client the revised scheduling document',
+          detail: '',
+          status: 'open',
+          subject: 'scheduling',
+          quote: firstSentence(document.text),
+        })),
+        resolutions: [],
+      };
+    },
+  };
+}
+
+function firstSentence(text: string): string {
+  const line = text.split('\n').map((row) => row.replace(/^[#\-*\s]+/, '').trim())
+    .find((row) => row.split(/\s+/).length >= 6);
+  return line ?? text.slice(0, 120);
+}
+
+/** An extractor whose model half fails the way a blown timeout really fails. */
+function timingOutExtractor(): CorpusExtractorLlm {
+  const base = createFixtureExtractor();
+  return {
+    describe: base.describe,
+    extract: async () => {
+      throw Object.assign(new Error('Request timed out after 60000ms'), {
+        name: 'LLMProviderError',
+        code: 'TIMEOUT',
+      });
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+//  1. resumption
+// ---------------------------------------------------------------------------
+
+describe('a sweep is resumable, so partial progress is real progress', () => {
+  it('starts queued at nothing, and does no model work in the call that starts it', async () => {
+    const { corpus } = trackingCorpus();
+    const store = createFixtureRunStore();
+    const extractor = createFixtureExtractor();
+    const spy = vi.spyOn(extractor, 'extract');
+
+    const run = await startExtractionRun({ scope: TANDEM }, { corpus, extractor, runs: store });
+
+    expect(run.status).toBe('queued');
+    expect(run.cursor).toBe(0);
+    // THE WHOLE POINT. Starting a run must return before a single 20-second
+    // model call, or the gateway kills the thing that was supposed to survive it.
+    expect(spy).not.toHaveBeenCalled();
+    // And it is durable the moment it exists, so a poll from another replica —
+    // or after this process dies — can see it.
+    expect((await store.readRun())?.runId).toBe(run.runId);
+  });
+
+  it('advances the cursor one slice at a time and persists it', async () => {
+    const { corpus } = trackingCorpus();
+    const store = createFixtureRunStore();
+    const extractor = createFixtureExtractor();
+
+    const started = await startExtractionRun({ scope: TANDEM }, { corpus, extractor, runs: store });
+    const after = await advanceExtractionRun(started, { corpus, extractor, runs: store }, {
+      documentsPerSlice: 1,
+    });
+
+    expect(after.cursor).toBe(1);
+    expect(after.totals.documentsProcessed).toBe(1);
+    expect(after.status).toBe('running');
+    expect((await store.readRun())?.cursor).toBe(1);
+  });
+
+  it('resumes from where it stopped instead of starting over', async () => {
+    const { corpus } = trackingCorpus();
+    const store = createFixtureRunStore();
+    const extractor = createFixtureExtractor();
+
+    // Two documents' worth of progress, then the process "dies".
+    let run = await startExtractionRun({ scope: TANDEM }, { corpus, extractor, runs: store });
+    run = await advanceExtractionRun(run, { corpus, extractor, runs: store }, { documentsPerSlice: 1 });
+    run = await advanceExtractionRun(run, { corpus, extractor, runs: store }, { documentsPerSlice: 1 });
+    const processed = [...run.processedIds];
+    expect(processed).toHaveLength(2);
+
+    // The lease dies with the process.
+    await store.writeRun({ ...run, leaseUntil: new Date(Date.now() - 60_000).toISOString() });
+
+    const resumed = await startExtractionRun({ scope: TANDEM }, { corpus, extractor, runs: store });
+    expect(resumed.runId).toBe(run.runId); // the SAME sweep, not a new one
+    expect(resumed.cursor).toBe(2);
+
+    const spy = vi.spyOn(extractor, 'extract');
+    await advanceExtractionRun(resumed, { corpus, extractor, runs: store }, { documentsPerSlice: 1 });
+    const sent = spy.mock.calls[0][0].documents.map((document) => document.id);
+    // Not one of the two already done. Redoing them would cost twenty seconds
+    // each for a guaranteed no-op.
+    expect(sent.some((id) => processed.includes(id))).toBe(false);
+  });
+
+  it('reports progress as a fraction of the corpus, so a spinner never has to lie', async () => {
+    const { corpus } = trackingCorpus();
+    const store = createFixtureRunStore();
+    const extractor = createFixtureExtractor();
+
+    let run = await startExtractionRun({ scope: TANDEM }, { corpus, extractor, runs: store });
+    run = await advanceExtractionRun(run, { corpus, extractor, runs: store }, { documentsPerSlice: 1 });
+
+    const progress = runProgress(run);
+    expect(progress.total).toBe(FIXTURE_TRACKED_DOCUMENTS.length);
+    expect(progress.done).toBe(1);
+    expect(progress.percent).toBeGreaterThan(0);
+  });
+
+  it('finishes the corpus and says so', async () => {
+    const { corpus } = trackingCorpus();
+    const store = createFixtureRunStore();
+    const extractor = createFixtureExtractor();
+
+    const started = await startExtractionRun({ scope: TANDEM }, { corpus, extractor, runs: store });
+    const done = await driveExtractionRun(started, { corpus, extractor, runs: store }, {
+      documentsPerSlice: 1,
+    });
+
+    expect(done.status).toBe('succeeded');
+    expect(done.cursor).toBe(FIXTURE_TRACKED_DOCUMENTS.length);
+    expect(done.totals.documentsProcessed).toBe(FIXTURE_TRACKED_DOCUMENTS.length);
+    expect(done.finishedAt).toBeTruthy();
+    expect(done.leaseUntil).toBeNull();
+  });
+});
+
+// ---------------------------------------------------------------------------
+//  2. idempotence under retry
+// ---------------------------------------------------------------------------
+
+describe('a queue retries, and a retry must not duplicate', () => {
+  it('a full second sweep leaves the workspace exactly as the first did', async () => {
+    const { corpus, tracked } = trackingCorpus();
+    const store = createFixtureRunStore();
+    const extractor = stableExtractor();
+
+    const first = await driveExtractionRun(
+      await startExtractionRun({ scope: TANDEM }, { corpus, extractor, runs: store }),
+      { corpus, extractor, runs: store },
+      { documentsPerSlice: 1 },
+    );
+    const afterFirst = tracked.size;
+    expect(afterFirst).toBeGreaterThan(0);
+    expect(first.totals.created).toBe(afterFirst);
+
+    const second = await driveExtractionRun(
+      await startExtractionRun({ scope: TANDEM, restart: true }, { corpus, extractor, runs: store }),
+      { corpus, extractor, runs: store },
+      { documentsPerSlice: 1 },
+    );
+
+    // THE NUMBER THAT MATTERS. 82 → 98 is what happened on the live tenant the
+    // last time this was got wrong.
+    expect(tracked.size).toBe(afterFirst);
+    expect(second.totals.created).toBe(0);
+  });
+
+  it('holds the count even when the model rewords its own findings between sweeps', async () => {
+    const { corpus, tracked } = trackingCorpus();
+    const store = createFixtureRunStore();
+
+    await driveExtractionRun(
+      await startExtractionRun(
+        { scope: TANDEM },
+        { corpus, extractor: stableExtractor(), runs: store },
+      ),
+      { corpus, extractor: stableExtractor(), runs: store },
+      { documentsPerSlice: 1 },
+    );
+    const afterFirst = tracked.size;
+
+    const second = await driveExtractionRun(
+      await startExtractionRun(
+        { scope: TANDEM, restart: true },
+        { corpus, extractor: rewordingExtractor(), runs: store },
+      ),
+      { corpus, extractor: rewordingExtractor(), runs: store },
+      { documentsPerSlice: 1 },
+    );
+
+    expect(tracked.size).toBe(afterFirst);
+    expect(second.totals.created).toBe(0);
+    // It did not just refuse the reworded item — it recognised it.
+    expect(second.totals.merged).toBeGreaterThan(0);
+  });
+
+  it('re-running an already-processed slice creates nothing', async () => {
+    const { corpus, tracked } = trackingCorpus();
+    const store = createFixtureRunStore();
+    const extractor = createFixtureExtractor();
+
+    const started = await startExtractionRun({ scope: TANDEM }, { corpus, extractor, runs: store });
+    const once = await advanceExtractionRun(started, { corpus, extractor, runs: store }, {
+      documentsPerSlice: 1,
+    });
+    const afterOne = tracked.size;
+
+    // The same slice again — what a retry after a lost acknowledgement looks like.
+    const again = await advanceExtractionRun(
+      { ...once, cursor: 0 },
+      { corpus, extractor, runs: store },
+      { documentsPerSlice: 1 },
+    );
+
+    expect(tracked.size).toBe(afterOne);
+    expect(again.totals.created).toBe(once.totals.created);
+  });
+
+  it('a slice that throws keeps the cursor, so the retry does not restart the sweep', async () => {
+    const { corpus } = trackingCorpus();
+    const store = createFixtureRunStore();
+    const extractor = createFixtureExtractor();
+
+    let run = await startExtractionRun({ scope: TANDEM }, { corpus, extractor, runs: store });
+    run = await advanceExtractionRun(run, { corpus, extractor, runs: store }, { documentsPerSlice: 1 });
+    expect(run.cursor).toBe(1);
+
+    const exploding = {
+      ...corpus,
+      listDocuments: async () => {
+        throw Object.assign(new Error('the workspace is unreachable'), { status: 502 });
+      },
+    } as CorpusApi & CommitmentApi;
+
+    const failed = await driveExtractionRun(run, { corpus: exploding, extractor, runs: store }, {
+      documentsPerSlice: 1,
+      maxSliceAttempts: 2,
+      retryDelayMs: 0,
+    });
+
+    expect(failed.status).toBe('failed');
+    expect(failed.error).toContain('unreachable');
+    // Everything already done is still done.
+    expect(failed.cursor).toBe(1);
+    expect((await store.readRun())?.cursor).toBe(1);
+  });
+});
+
+// ---------------------------------------------------------------------------
+//  3. legible failure
+// ---------------------------------------------------------------------------
+
+describe('a run that the model did not answer says so', () => {
+  it('reports the timeout instead of reporting zero findings', async () => {
+    const { corpus } = trackingCorpus();
+    const result = await extractCommitments(
+      { scope: TANDEM, limit: 1 },
+      { corpus, extractor: timingOutExtractor() },
+    );
+
+    // The old behaviour: fromModel 0 and nothing else, which reads as "the model
+    // found nothing to add" and is how a rules-only run got reported as a model
+    // run for a week.
+    expect(result.fromModel).toBe(0);
+    expect(result.modelFailure).not.toBeNull();
+    expect(result.modelFailure?.stage).toBe('extract');
+    expect(result.modelFailure?.timedOut).toBe(true);
+    expect(result.modelFailure?.message).toMatch(/timed out/i);
+    // The deterministic half still landed — a provider outage costs the extras.
+    expect(result.fromRules).toBeGreaterThan(0);
+  });
+
+  it('distinguishes a verifier that did not answer from a verifier that refused', async () => {
+    const { corpus } = trackingCorpus();
+    const base = createFixtureExtractor();
+    const result = await extractCommitments(
+      { scope: TANDEM },
+      {
+        corpus,
+        extractor: {
+          describe: base.describe,
+          extract: base.extract,
+          verify: async () => {
+            throw Object.assign(new Error('Request timed out after 60000ms'), { code: 'TIMEOUT' });
+          },
+        },
+      },
+      // Enough budget that the slice really carries candidates for the verifier
+      // to be asked about — with none, no check is NEEDED and none failing is
+      // the honest answer.
+      { maxCharsPerRun: 200_000 },
+    );
+
+    expect(result.verified).toBe(false);
+    expect(result.modelFailure?.stage).toBe('verify');
+    expect(result.modelFailure?.timedOut).toBe(true);
+  });
+
+  it('says which meetings got only the deterministic pass', async () => {
+    const { corpus } = trackingCorpus();
+    const store = createFixtureRunStore();
+
+    const run = await driveExtractionRun(
+      await startExtractionRun(
+        { scope: TANDEM },
+        { corpus, extractor: timingOutExtractor(), runs: store },
+      ),
+      { corpus, extractor: timingOutExtractor(), runs: store },
+      { documentsPerSlice: 1 },
+    );
+
+    expect(run.status).toBe('succeeded');
+    expect(run.totals.modelFailures).toBe(FIXTURE_TRACKED_DOCUMENTS.length);
+    expect(run.totals.modelCalls).toBe(0);
+    expect(run.lastModelFailure?.timedOut).toBe(true);
+    // Named, so a later run can be pointed at them.
+    expect(run.modelIncompleteIds.length).toBe(FIXTURE_TRACKED_DOCUMENTS.length);
+    // And the rules half still ran on every one of them.
+    expect(run.totals.fromRules).toBeGreaterThan(0);
+  });
+
+  it('a clean run carries no failure at all', async () => {
+    const { corpus } = trackingCorpus();
+    const store = createFixtureRunStore();
+    const extractor = createFixtureExtractor();
+
+    const run = await driveExtractionRun(
+      await startExtractionRun({ scope: TANDEM }, { corpus, extractor, runs: store }),
+      { corpus, extractor, runs: store },
+      { documentsPerSlice: 1 },
+    );
+
+    expect(run.lastModelFailure).toBeNull();
+    expect(run.totals.modelFailures).toBe(0);
+    expect(run.totals.modelCalls).toBe(FIXTURE_TRACKED_DOCUMENTS.length);
+  });
+});
+
+// ---------------------------------------------------------------------------
+//  4. one driver
+// ---------------------------------------------------------------------------
+
+describe('exactly one driver at a time', () => {
+  it('a second start hands back the running sweep rather than a second one', async () => {
+    const { corpus } = trackingCorpus();
+    const store = createFixtureRunStore();
+    const extractor = createFixtureExtractor();
+
+    const first = await startExtractionRun({ scope: TANDEM }, { corpus, extractor, runs: store });
+    const second = await startExtractionRun({ scope: TANDEM }, { corpus, extractor, runs: store });
+
+    expect(second.runId).toBe(first.runId);
+    expect(second.adopted).toBe(false);
+    expect(isLeaseLive(second, Date.now())).toBe(true);
+  });
+
+  it('a lease that has expired is adoptable, and the run says it stalled', async () => {
+    const { corpus } = trackingCorpus();
+    const store = createFixtureRunStore();
+    const extractor = createFixtureExtractor();
+
+    const first = await startExtractionRun({ scope: TANDEM }, { corpus, extractor, runs: store });
+    const stale = { ...first, leaseUntil: new Date(Date.now() - 1000).toISOString() };
+    await store.writeRun(stale);
+
+    expect(isStalled(stale, Date.now())).toBe(true);
+
+    const adopted = await startExtractionRun({ scope: TANDEM }, { corpus, extractor, runs: store });
+    expect(adopted.runId).toBe(first.runId);
+    expect(adopted.adopted).toBe(true);
+    expect(isStalled(adopted, Date.now())).toBe(false);
+  });
+
+  it('a driver that loses its lease stops instead of racing the one that took it', async () => {
+    const { corpus } = trackingCorpus();
+    const store = createFixtureRunStore();
+    const extractor = createFixtureExtractor();
+
+    const run = await startExtractionRun({ scope: TANDEM }, { corpus, extractor, runs: store });
+    // Somebody else takes it between slices.
+    await store.writeRun({
+      ...run,
+      leaseOwner: 'another-replica',
+      leaseUntil: new Date(Date.now() + 60_000).toISOString(),
+    });
+
+    const stopped = await driveExtractionRun(run, { corpus, extractor, runs: store }, {
+      documentsPerSlice: 1,
+    });
+
+    expect(stopped.leaseOwner).toBe('another-replica');
+    expect(stopped.status).not.toBe('succeeded');
+  });
+
+  it('a sweep that FAILED resumes from its cursor, it does not start over', async () => {
+    // Thirty minutes is the life of the token that starts a sweep and ten is
+    // the length of one, so "the session expired at meeting twelve" is a normal
+    // event rather than an exotic one. Starting over would throw away twelve
+    // meetings of model time to redo work that is already written.
+    const { corpus } = trackingCorpus();
+    const store = createFixtureRunStore();
+    const extractor = createFixtureExtractor();
+
+    let run = await startExtractionRun({ scope: TANDEM }, { corpus, extractor, runs: store });
+    run = await advanceExtractionRun(run, { corpus, extractor, runs: store }, { documentsPerSlice: 1 });
+    await store.writeRun({ ...run, status: 'failed', error: 'api/v1/entities responded 502', leaseUntil: null });
+
+    const resumed = await startExtractionRun({ scope: TANDEM }, { corpus, extractor, runs: store });
+    expect(resumed.runId).toBe(run.runId);
+    expect(resumed.cursor).toBe(1);
+    expect(resumed.status).toBe('running');
+    expect(resumed.error).toBeNull();
+  });
+
+  it('an expired session pauses the sweep rather than failing it', async () => {
+    // A 401 retried is still a 401, and it is not a fact about the corpus. The
+    // stint ends, the lease is dropped, the cursor stays, and whoever presents a
+    // live credential next carries on.
+    const { corpus } = trackingCorpus();
+    const store = createFixtureRunStore();
+    const extractor = createFixtureExtractor();
+
+    let run = await startExtractionRun({ scope: TANDEM }, { corpus, extractor, runs: store });
+    run = await advanceExtractionRun(run, { corpus, extractor, runs: store }, { documentsPerSlice: 1 });
+
+    const expired = {
+      ...corpus,
+      listDocuments: async () => {
+        throw Object.assign(new Error('api/v1/entities/ responded 401'), { status: 401 });
+      },
+    } as CorpusApi & CommitmentApi;
+
+    const paused = await driveExtractionRun(run, { corpus: expired, extractor, runs: store }, {
+      documentsPerSlice: 1,
+      retryDelayMs: 0,
+    });
+
+    expect(paused.status).toBe('running');
+    expect(paused.cursor).toBe(1);
+    expect(paused.leaseUntil).toBeNull();
+    expect(isStalled(paused, Date.now())).toBe(true);
+    expect(paused.error).toMatch(/expired/i);
+
+    // And it is adoptable, from the cursor, by a caller with a live session.
+    const resumed = await startExtractionRun({ scope: TANDEM }, { corpus, extractor, runs: store });
+    expect(resumed.cursor).toBe(1);
+    expect(resumed.error).toBeNull();
+  });
+
+  it('a finished run is never resumed by accident', async () => {
+    const { corpus } = trackingCorpus();
+    const store = createFixtureRunStore();
+    const extractor = createFixtureExtractor();
+
+    const done = await driveExtractionRun(
+      await startExtractionRun({ scope: TANDEM }, { corpus, extractor, runs: store }),
+      { corpus, extractor, runs: store },
+      { documentsPerSlice: 1 },
+    );
+    expect(done.status).toBe('succeeded');
+
+    // Pressing the button again is a NEW sweep, from the top, not a resume of a
+    // finished one — otherwise "run it again" silently does nothing.
+    const next = await startExtractionRun({ scope: TANDEM }, { corpus, extractor, runs: store });
+    expect(next.runId).not.toBe(done.runId);
+    expect(next.cursor).toBe(0);
+    expect(next.history[0]?.runId).toBe(done.runId);
+  });
+});
+
+// ---------------------------------------------------------------------------
+//  the run record itself
+// ---------------------------------------------------------------------------
+
+describe('the run record', () => {
+  it('round-trips through the tenant entity store without losing a field', async () => {
+    const store = createFixtureRunStore();
+    const run = {
+      ...newExtractionRun({ totalDocuments: 17 }),
+      cursor: 9,
+      status: 'running' as const,
+      totals: {
+        documentsProcessed: 9, fromRules: 40, fromModel: 12, ungrounded: 3, unconfirmed: 7,
+        created: 44, merged: 8, resolved: 1, droppedOutOfScope: 0, modelFailures: 2, modelCalls: 7,
+      },
+      processedIds: ['a', 'b'],
+      modelIncompleteIds: ['b'],
+      lastModelFailure: { stage: 'extract' as const, message: 'Request timed out after 60000ms', timedOut: true },
+    };
+
+    await store.writeRun(run);
+    const read = await store.readRun();
+
+    expect(read).toEqual(expect.objectContaining({
+      runId: run.runId,
+      cursor: 9,
+      status: 'running',
+      processedIds: ['a', 'b'],
+      modelIncompleteIds: ['b'],
+    }));
+    expect(read?.totals).toEqual(run.totals);
+    expect(read?.lastModelFailure).toEqual(run.lastModelFailure);
+  });
+
+  it('refuses a run with no workspace, the same way extraction does', async () => {
+    const { corpus } = trackingCorpus();
+    const store = createFixtureRunStore();
+    await expect(
+      startExtractionRun(
+        { scope: { orgId: '' } },
+        { corpus, extractor: createFixtureExtractor(), runs: store },
+      ),
+    ).rejects.toThrow(/workspace/i);
+  });
+});