@qodeca/xezar 0.13.0 → 0.14.0

package/dist/workflows/run.d.ts

@@ -14,6 +14,22 @@ import { type WorkflowDef } from './types.ts';
  * `DEFAULT_IDLE_TIMEOUT_MINUTES`.
  */
 export declare const IDLE_TIMEOUT_MS: number;
+/**
+ * Why a NON-FINAL agent step may not be marked `done` (#317), or null when it may.
+ *
+ * Such a step runs one turn and its session closes by itself, so it has no way to wait for an
+ * answer — and until #317 nothing decided it was done except "the session ended without an
+ * error". Run b86c6066's implement step ended on an `XEZ:ASK` design question, was marked done,
+ * and readiness, gates and the seal all ran on a branch with no work in it.
+ *
+ * The decision now belongs to the agent contract every step is given (`HANDOFF_INSTRUCTIONS`):
+ * `XEZ:DONE` means "my goal is achieved", and a turn that ends without it is "waiting on the
+ * user". A turn that ends on anything else — a question, the monitoring marker, or plain prose —
+ * stops the workflow here instead of carrying on without an answer. Deliberately fail-closed on
+ * plain prose: that is the case no agent-side marker can report, and the one the BLOCKED file
+ * could not catch either. `turnText` is null when the session ended without finishing a turn.
+ */
+export declare function unfinishedStepReason(turnText: string | null): string | null;
 /**
  * Preserve boundaries between complete assistant text blocks while a turn is
  * accumulated for marker parsing. The runners join these same v1 blocks with
@@ -62,6 +78,17 @@ export declare const MAX_AUTO_RESUMES = 12;
  * note instead of fired, so the only tasks a sweep can revive are ones someone is still waiting on.
  */
 export declare const AUTO_RESUME_MISSED_WINDOW_MS: number;
+/**
+ * How long a resumed turn must stay live before it counts as proof that the limit lifted (#285).
+ *
+ * The in-flight hold exists for the window where a resume is TESTING the account: a limit that
+ * has not lifted refuses the turn at its first API call, and the measured doomed run lives about
+ * 200 ms. A turn still running minutes later got past that call, so holding every other task on
+ * the account until it COMPLETES — which a long turn may not do for hours — is the hold outliving
+ * its purpose. Not the first streamed event: a refused Claude turn still streams its session init
+ * and a synthetic assistant frame before the error, so "it said something" proves nothing.
+ */
+export declare const AUTO_RESUME_PROOF_MS: number;
 /**
  * How often the queue checks that it is not wedged.
  *
@@ -244,6 +271,12 @@ export declare class RunManager {
      *  from the record rather than losing the wait. Runs here are `failed` and therefore NOT in
      *  `active`, which is why the timer cannot live on an `ActiveRun` like the monitoring one. */
     private readonly autoResumeTimers;
+    /** Wake-ups for the instant a live resume's proof window closes (#285). The hold lifts by
+     *  derivation, and a derived release is not an event, so these are only the pump that notices;
+     *  the hold itself stays on the records. */
+    private readonly resumeProofTimers;
+    /** `AUTO_RESUME_PROOF_MS`, overridable only so a test need not wait two minutes. */
+    private readonly resumeProofMs;
     private pumping;
     /** A pump that arrived while one was in flight — replayed by `pump()`'s own
      *  loop so a slot freed mid-sweep is never a lost wakeup. */
@@ -265,15 +298,77 @@ export declare class RunManager {
     private readonly offUsage;
     /** The stalled-queue watchdog (see `rescueStalledQueue`). */
     private readonly queueWatchdog;
-    /** The rescue sweeps that have STARTED and not settled yet (#125). `clearInterval` cancels the
-     *  next tick; it cannot cancel the tick already running, and a sweep is async — it awaits
-     *  `reviveWorkflow` before appending NDJSON. dispose() settles these so a torn-down manager is
-     *  provably finished writing before its caller removes the data root under it. */
-    private readonly rescuesInFlight;
+    /**
+     * Every background WRITE this manager started on its own initiative and has not settled yet.
+     * Two producers, one set: the queue-watchdog rescue sweeps (#125) and the terminal-transition
+     * worktree retention sweeps (#200). Neither is cancellable and both cross an await before they
+     * touch disk — a rescue appends NDJSON after `reviveWorkflow`, a retention sweep spawns
+     * `git worktree remove` after `resolveWorktreeRetention` — so `clearInterval` and the
+     * `disposed` flag between them still leave work running.
+     *
+     * dispose() settles this set, which is what makes its promise mean "provably finished writing"
+     * before the caller removes the data root. Retention was the half that was NOT tracked, and
+     * the harm differs by caller — be precise about which, because only one of them deletes
+     * anything. In a TEST teardown a disposed manager could still be spawning git inside a
+     * directory the case was already `rmSync`-ing, which is the reported ENOTEMPTY. In a LIVE
+     * server nothing removes the project directory at all: the sweep instead stamps
+     * `worktreeReclaimedAt` onto a record, scheduling a debounced `runs.json` write from a store
+     * whose lifecycle has ended — so a re-add inside that window can see the stale in-memory index
+     * overwrite the fresh one. Every producer therefore enrols through a wrapper
+     * (`rescueStalledQueue`, `enforceRetention`), so nothing can start a background write teardown
+     * cannot see.
+     *
+     * The live-server half needed the OTHER end fixed too, and the promise is worthless without it:
+     * `ProjectContexts.dispose` used to drop this promise on the floor (`teardown()` was
+     * synchronous), so the guarantee held only for callers that awaited — which was tests. It now
+     * awaits the manager before `store.flush()` + `removeAllListeners()`, and `DELETE /projects/:id`
+     * awaits that in turn.
+     */
+    private readonly writesInFlight;
+    /**
+     * The run BODIES — `execute()` / `runContinuation()` — that have started and not settled yet,
+     * plus the turn-end bookkeeping (`recordTurnEnd`) that a body fires and does not await.
+     *
+     * Deliberately NOT `writesInFlight`. dispose() is documented as "the manager makes no further
+     * moves ON ITS OWN" and is explicitly not a run-stopper, so awaiting a live agent session in
+     * its promise would silently change what every existing caller gets. `quiesce()` is the opt-in
+     * that cancels first and then awaits this set; dispose()'s semantics are untouched.
+     */
+    private readonly runsInFlight;
+    /**
+     * Cancellations that arrived while a run was still MATERIALIZING — dequeued into `starting`,
+     * or floated by the direct Continue path — and had nowhere to land.
+     *
+     * `cancel()` stops an active run by writing `state.cancelled` and calling `state.interrupt()`,
+     * and neither exists until the body has built its `ActiveRun`. Before this set, `cancel()` for
+     * such a run simply returned false and delivered NOTHING, so the run went on to spawn an agent
+     * turn that no one had asked for and that `quiesce()` then waited out — up to the runner's
+     * 30-minute default (#200). The body consumes the request through `adoptActive`, at the first
+     * instant a cancellation has somewhere to go.
+     *
+     * `dropActive` clears the entry too, because a body can leave without ever reaching
+     * `adoptActive` (the continuation whose isolation is gone), and a stale entry would cancel the
+     * NEXT run to reuse that id — which, for a Continue, is the same id again.
+     */
+    private readonly cancelRequested;
     /** Set by dispose(): this manager makes no further moves. Re-checked at every await boundary a
      *  rescue crosses, because a sweep that was already past its first check when dispose() landed
      *  would otherwise write an event — and re-populate the queue dispose() had just emptied. */
     private disposed;
+    /**
+     * Set by `quiesce()` for the length of its drain: the scheduler starts nothing new.
+     *
+     * `disposed` cannot cover this window, because `quiesce()` disposes LAST — the whole point is
+     * to stop the runs while `cancel()` can still see them. During the drain the manager is
+     * otherwise fully live, and every settling run pumps the entire workspace on its way out
+     * (`dropActive` → `releaseSlot` → `semaphore.release()`, which awaits `pump()` on every
+     * registered participant, this one included). A pump reconciles auto-resumes from the RECORDS,
+     * and a record whose `autoResumeAt` has already passed arms at zero delay — so the drain could
+     * fire a resume, spawn a fresh agent turn into the very repo root the caller is about to
+     * delete, and then dutifully wait for it. Bounded by `MAX_AUTO_RESUMES`, and the exact opposite
+     * of what the method promises.
+     */
+    private quiescing;
     /** Set by the watchdog for exactly one sweep: ignore the usage-limit hold and make progress. */
     private forceNextPump;
     /** Runs the watchdog started despite the hold. The spawn-time gate (`requeueWhileHeld`) would
@@ -288,6 +383,7 @@ export declare class RunManager {
     private readonly offSemaphore;
     constructor(store: RunStore, repoRoot: string, options?: {
         semaphore?: WorkspaceSemaphore;
+        resumeProofMs?: number;
     });
     /**
      * Release everything this manager owns without touching run records
@@ -300,17 +396,110 @@ export declare class RunManager {
      * dispose only guarantees the manager makes no further moves on its own.
      *
      * Every side effect below is SYNCHRONOUS, so a caller that ignores the return value behaves
-     * exactly as it did before #125. The returned promise settles the one thing dispose cannot do
-     * synchronously: a queue-watchdog rescue that had already started. `clearInterval` stops the
-     * next tick, never the running one — the same discipline `AGENTS.md` records for the e2e
+     * exactly as it did before #125. The returned promise settles the things dispose cannot do
+     * synchronously: the background writes already in flight — a queue-watchdog rescue (#125) and
+     * a worktree retention sweep (#200), both tracked in `writesInFlight`. `clearInterval` stops
+     * the next tick, never the running one — the same discipline `AGENTS.md` records for the e2e
      * fixture servers, where `kill()` only delivers the signal and the helper awaits the exit.
      * Await it whenever the data root is about to be removed; a test that deletes its temp
      * directory without awaiting is the ENOENT in #125.
      *
+     * It does NOT wait for running runs, and that is the point of `quiesce()`: dispose clears
+     * `active`/`starting`/`queue` without stopping anything, so after it returns a leaked run is
+     * invisible to `cancel()` and can no longer be stopped at all. Reach for `quiesce()` whenever
+     * the runs are yours to end; reach for `dispose()` when they are not.
+     *
+     * ## The states a disposed manager leaves behind, and how they end
+     *
+     * - A run that was `active` keeps running to its own terminal status and writes it to the
+     *   record; only the manager's registries forgot it. `cancel()` answers false for it from here
+     *   on, which is the leak `quiesce()` exists to avoid.
+     * - A run that was `queued` — or one `startRun()` accepts AFTER dispose, which still writes a
+     *   `queued` record and still floats a `pump()` — never starts in this process: `pump()` bails
+     *   on `disposed`, so nothing dequeues it. Its only exit is the next process: `recover()`
+     *   re-queues every `queued` record at boot, and `rescueStalledQueue` re-adopts one the engine
+     *   is holding no work item for. That is deliberate — a disposed manager's project has been
+     *   removed from the registry, and starting its work would be the surprise — but it does mean
+     *   the record outlives the process as `queued` rather than `cancelled`.
+     *
      * It never rejects: a sweep that fails still belongs to whoever started it (the watchdog floats
      * it exactly as before), and teardown must not become a second place that error surfaces.
      */
     dispose(): Promise<void>;
+    /**
+     * Stop every run this manager owns, then dispose — "the data root is about to disappear".
+     *
+     * dispose() alone cannot deliver that: it is not a run-stopper by design, and because it
+     * clears `active`/`starting`/`queue` first, a run left behind is afterwards invisible to
+     * `cancel()` (which looks only in `queue` then `active`) and can never be stopped. The order
+     * here is the whole fix — cancel, drain, dispose — and it is the order four test files were
+     * already hand-rolling three different ways.
+     *
+     * ## Why the cancel is INSIDE the loop, and why termination needs `quiescing`
+     *
+     * Cancelling once up front is not enough, because the population is not fixed while the drain
+     * runs. `starting` promotes to `active` (the `ActiveRun` a `cancel()` needs did not exist a
+     * moment ago), a queued continuation crosses `rematerializeReclaimedWorktree` before it
+     * registers anywhere at all, and every settling body pumps the whole workspace on its way out.
+     * So each pass re-asks `activeRunIds()` and re-issues `cancel()`; `cancelRequested` carries a
+     * cancellation into the one window where neither registry can hold it.
+     *
+     * Termination then rests on `quiescing`, not on "the queue is empty". The manager has three
+     * self-starting entry points — `pump()`, `fireAutoResume()` and the queue watchdog's
+     * `rescueStalledQueue()` — and all three bail while it is set, so no pass can enrol a run the
+     * previous pass did not cancel. Ask "who fires this?" of any new one: the watchdog is an
+     * unref'd 60 s interval, so it is invisible in a short drain and lands squarely in a long one
+     * (an agent session that will not close is bounded only by the runner's 30-minute default),
+     * where it would re-adopt queued records — NDJSON, `pendingJobs`, a `queue` push — inside the
+     * window this method exists to make quiet.
+     *
+     * What a pass CAN still enrol is bounded bookkeeping: a body fires one `recordTurnEnd` per turn
+     * it finishes, and a cancelled run finishes no further turns (`autoContinueTurn` refuses a
+     * cancelled state). Each iteration therefore awaits a strictly older generation of work than the
+     * last, and the generations are finite.
+     *
+     * An external caller CAN still enrol during the drain — `startRun()` and `continueRun()` are
+     * not gated by `quiescing`, because refusing a user's request is the HTTP layer's decision and
+     * not a teardown helper's. The loop absorbs one: the next pass finds it, cancels it and waits
+     * for it, which is exactly what the "an agent turn enrolled after the drain took its snapshot"
+     * case pins. What is NOT bounded is an unbroken STREAM of them — a caller that keeps starting
+     * work into a project it is tearing down gets what it asked for.
+     *
+     * ## Each pass delivers its cancel ONCE — the bodies are what make that enough
+     *
+     * Re-issuing `cancel()` per pass reaches runs whose REGISTRY membership changed; it does not
+     * reach a run that was already in `active` when the pass ran, because the next re-issue waits
+     * on the same `Promise.allSettled` the stuck body is holding open. So the guarantee has to live
+     * in the body: every point at which a run body can park must consume a cancellation that
+     * arrived while it was elsewhere. Those points are `adoptActive` (the `starting` window),
+     * `acquireRepoRoot` (the lease wait, which races its own abort), the step loop's
+     * `if (state.cancelled) break`, `publishSession` (the gap between the loop's check and a live
+     * `state.interrupt`, which is what #199 hung on), and `state.interrupt()` itself once a session
+     * is up. Adding a new `await` inside a run body means asking which of those covers it.
+     *
+     * ## If a tracked promise never settles
+     *
+     * `quiesce()` has no deadline of its own and stays pending — the caller's timeout is the only
+     * bound. That is deliberate. Cancellation is delivered, an agent step's own wall clock still
+     * applies, and a session that refuses to close is a real writer: returning early would hand the
+     * caller a promise that says "nothing is writing any more" while something is, which is the
+     * exact lie this method exists to remove. An invented deadline would only move the ENOTEMPTY a
+     * few hundred milliseconds later — and, in the #199 hang, would have shipped a teardown that
+     * deleted a repository out from under a live agent CLI instead of failing loudly at 90 s.
+     *
+     * Never rejects, the same contract as dispose(): teardown must not become a second place a run's
+     * error surfaces.
+     */
+    quiesce(): Promise<void>;
+    /**
+     * Enrol a run body in `runsInFlight` so `quiesce()` can wait for it.
+     *
+     * The body arrives with its own `.catch(…)` already attached — the failure policy stays at the
+     * call site, where it can name the run — and the extra `.catch` here only keeps the tracked
+     * copy from ever rejecting, so a handler that itself throws cannot turn teardown into an
+     * unhandled rejection.
+     */
+    private trackRun;
     /**
      * Pause any active run whose whole process tree exceeds the WORKSPACE
      * `resources.memoryLimitMb`, freeing its slot so the queue advances
@@ -503,7 +692,7 @@ export declare class RunManager {
      *
      * Public so a test can drive the wedge directly instead of waiting out the interval.
      *
-     * The sweep itself is `sweepStalledQueue`; this wrapper only publishes it to `rescuesInFlight`
+     * The sweep itself is `sweepStalledQueue`; this wrapper only publishes it to `writesInFlight`
      * so dispose() can settle it (#125). Every entry point goes through here — the interval and the
      * tests alike — so nothing can start a sweep that teardown cannot see. The schedule, the
      * conditions and the effects are untouched.
@@ -532,13 +721,48 @@ export declare class RunManager {
      * which is the route's 404.
      */
     cancelAutoResume(runId: string): boolean;
+    /**
+     * Pump the whole workspace once a live resume's proof window has closed (#285). The hold is
+     * workspace-wide — one account can be queued in several projects — so this is `release()`,
+     * not this manager's own pump. It carries no state: whether the hold has lifted is decided by
+     * `accountHolds()` from the record when the pump reads it, so a turn that failed or finished
+     * first just makes this a no-op pump. The slack keeps a timer that fires a millisecond early
+     * from reading a window that has not quite closed yet.
+     */
+    private armResumeProofPump;
     /** Retire a pending resume — timer, deadline and counter. The counter goes too because every
      *  caller is a fresh epoch: a human Continue, or a resume that re-stamps its own count. */
     private clearAutoResume;
-    /** Reclaim finished worktrees beyond the keep-limit (#483) — directory only,
-     *  `xez/<id8>` branch kept. Best-effort; a failure never affects run
-     *  lifecycle. `review`/live runs are excluded by the selector. */
+    /**
+     * Reclaim finished worktrees beyond the keep-limit (#483) — directory only, `xez/<id8>` branch
+     * kept. Best-effort; a failure never affects run lifecycle. `review`/live runs are excluded by
+     * the selector.
+     *
+     * The sweep itself is `reclaimRetiredWorktrees`; this wrapper only publishes it to
+     * `writesInFlight` so dispose() can settle it, the same shape `rescueStalledQueue` already
+     * uses. Callers still float it — retention must never delay or throw into a terminal
+     * transition — but a floated promise nobody holds is one teardown cannot wait for, and this
+     * one spawns `git worktree remove` and `git worktree prune` inside `repoRoot` (#200).
+     */
     private enforceRetention;
+    /**
+     * The retention sweep proper. `disposed` is re-read after `resolveWorktreeRetention` — the one
+     * await before any git spawn — for the reason the flag exists: a sweep already past its first
+     * check when dispose() landed would otherwise start removing directories under a root its
+     * caller is deleting. `shouldStop` carries the same question INTO the loop, because a sweep
+     * removes one directory per iteration and a single check before the first one stops nothing on
+     * a five-worktree pass that straddles a dispose.
+     *
+     * Bailing there reclaims nothing, and nothing is lost by it: retention is count-based and
+     * re-enforced from scratch by the boot sweep (`index.ts`) and by the Settings → Worktrees
+     * reclaim route, so a skipped pass costs one over-limit directory until the next boot. The
+     * alternative — finishing the pass — spends two git spawns inside a directory whose owner has
+     * just said it is done with it, plus a `worktreeReclaimedAt` stamp through `store.updateRun`
+     * that schedules a debounced `runs.json` write from a store nobody owns any more. That is the
+     * whole of #200, and the same hazard `armRepoHandle` is guarded against in
+     * `server/project-context.ts`.
+     */
+    private reclaimRetiredWorktrees;
     /** Last live-refresh namer inputs per run — unchanged inputs skip the call. */
     private lastNamerKey;
     /**
@@ -561,7 +785,48 @@ export declare class RunManager {
      */
     private acquireRepoRoot;
     cancel(runId: string): boolean;
+    /**
+     * Publish a run's freshly built `ActiveRun` and adopt any cancellation that arrived while it
+     * was being built. Returns true when the caller must abandon the run.
+     *
+     * One helper for both bodies, because the two `active.set` sites are the two halves of the same
+     * moment and had already drifted once (`state.autonomous`, #141). Ordering inside is
+     * load-bearing: the flag is consumed BEFORE `active.set`, so a `cancel()` landing after this
+     * returns finds a state to write to and takes the ordinary path.
+     */
+    private adoptActive;
+    /**
+     * Publish a freshly started agent session on the run's `ActiveRun`, and deliver any
+     * cancellation that arrived while the session was being BUILT.
+     *
+     * The twin of `adoptActive`, one phase later and for the same reason. `cancel()` stops a live
+     * turn by calling `state.interrupt()`, and between the step loop's own `state.cancelled` check
+     * and this moment that function is still the `() => undefined` placeholder: everything in
+     * between — `configuredModelProvider`, `agentEnvForStep`, a team skill's `materializeSkillDir`
+     * — is `await`ed, and a cancel landing in any of those gaps set the flag and delivered nothing.
+     *
+     * What that cost is a run that never ends, not a turn that runs one step too long (#199/#200).
+     * The session spawns anyway; an interactive step passes `timeoutMs: 0`, so the runner has no
+     * wall clock at all; and the turn-end handler computes `sessionOpen` as
+     * `!state.cancelled && session.open`, so a cancelled run neither parks at `waiting` nor closes
+     * the session on `XEZ:DONE`. The body then sits in `await session.result` for as long as the
+     * agent CLI keeps stdin open — which for the bundled mock is forever. `quiesce()` issues its
+     * cancel once per drain generation and only re-issues it after `Promise.allSettled` resolves,
+     * so a single missed delivery is a permanent hang: measured as the 90 s teardown timeout on a
+     * 2-core CI runner, where the widened gap makes the window easy to land in.
+     *
+     * ONE helper for both construction sites — `runAgentStep` and `runContinuation` — because they
+     * are the same moment written twice and had already drifted (only one of them re-stamped
+     * `currentStepId`). A third session site must come through here too, or it re-opens the hole.
+     */
+    private publishSession;
     isActive(runId: string): boolean;
+    /** The id set behind `isActive` — `active ∪ starting ∪ queue`. `quiesce()` needs the whole
+     *  population rather than one membership test, and a second spelling of that union is exactly
+     *  how the two drift apart: `run-quiesce.test.ts` pins the observable half by putting one run
+     *  in `queue` and one in `active`, asserting `isActive` for both before `quiesce()` and for
+     *  neither after. A registry this set forgets would leave a live writer behind. */
+    private activeRunIds;
     /**
      * Fold a queued run's persisted prompt — `run.task` plus everything stacked
      * onto it (#472) — into the job input that is about to execute.