@edspencer/paddock 0.65.0 → 0.66.1

package/packages/server/dist/subagents.js

@@ -88,7 +88,7 @@ export const SAFE_SEGMENT = /^[A-Za-z0-9._-]+$/;
  */
 export const MTIME_CACHE_MAX = 1024;
 const taskUsesCache = new Map();
-const durationCache = new Map();
+const factsCache = new Map();
 const usageCache = new Map();
 /** The file's mtime in epoch ms, or undefined if it can't be stat'd. */
 export async function statMtimeMs(file) {
@@ -137,7 +137,7 @@ export async function readTaskToolUses(projectDir, sessionId) {
  */
 async function readTaskUsesForSession(projectDir, sessionId) {
     if (!SAFE_SEGMENT.test(sessionId))
-        return { paired: [], pending: [] };
+        return emptyTaskToolUses();
     return readTaskUsesFromFile(path.join(projectChatsDir(projectDir), `${sessionId}.jsonl`));
 }
 /**
@@ -158,10 +158,32 @@ async function readTaskUsesFromFile(file) {
         mtimeCacheSet(taskUsesCache, file, mtimeMs, value);
     return value;
 }
+function emptyTaskToolUses() {
+    return { paired: [], pending: [], sidechain: { uuids: new Set(), taskIds: new Set() } };
+}
+/**
+ * True when a raw transcript LINE was written on a sub-agent's own lane.
+ *
+ * Two markers, because the writers disagree: Claude Code stamps `isSidechain:
+ * true` (plus an `agentId`) on the line, while the SDK message stream — which
+ * `isSidechainMessage` in `ws-turn.ts` reads live, and which the test harness
+ * mirrors onto disk — carries `parent_tool_use_id`. Either is conclusive.
+ *
+ * Exported because EVERY file-order join over a main transcript has to agree with
+ * {@link dropSidechainMessages} about which lines exist: a scan that keeps a
+ * sidechain line while the parsed message list has dropped it shifts the whole
+ * join by one (#727).
+ */
+export function isSidechainLine(parsed) {
+    if (parsed.isSidechain === true)
+        return true;
+    return Boolean(parsed.parent_tool_use_id ?? parsed.message?.parent_tool_use_id);
+}
 async function readTaskUsesFromFileUncached(file) {
     const byId = new Map();
     const order = [];
     const resultIds = new Set();
+    const sidechain = { uuids: new Set(), taskIds: new Set() };
     const stream = createReadStream(file, { encoding: "utf8" });
     stream.on("error", () => undefined);
     const rl = createInterface({ input: stream, crlfDelay: Infinity });
@@ -177,12 +199,20 @@ async function readTaskUsesFromFileUncached(file) {
             catch {
                 continue;
             }
+            // A sub-agent's OWN transcript is sidechain top to bottom, so this only ever
+            // marks anything on a MAIN session file — which is the only place the
+            // distinction means something (issue #727).
+            const onSidechain = isSidechainLine(parsed);
+            if (onSidechain && typeof parsed.uuid === "string")
+                sidechain.uuids.add(parsed.uuid);
             const content = parsed.message?.content;
             if (!Array.isArray(content))
                 continue;
             for (const block of content) {
                 const b = block;
                 if (b?.type === "tool_use" && b.name && SUBAGENT_TOOL_NAMES.has(b.name) && b.id) {
+                    if (onSidechain)
+                        sidechain.taskIds.add(b.id);
                     if (!byId.has(b.id)) {
                         order.push(b.id);
                         byId.set(b.id, {
@@ -200,7 +230,7 @@ async function readTaskUsesFromFileUncached(file) {
         }
     }
     catch {
-        return { paired: [], pending: [] };
+        return emptyTaskToolUses();
     }
     finally {
         rl.close();
@@ -209,8 +239,43 @@ async function readTaskUsesFromFileUncached(file) {
     return {
         paired: order.filter((id) => resultIds.has(id)).map((id) => byId.get(id)),
         pending: order.filter((id) => !resultIds.has(id)).map((id) => byId.get(id)),
+        sidechain,
     };
 }
+/**
+ * Drop a session's sub-agent sidechain steps from its parsed history (issue
+ * #727) — the history-path counterpart of the live path's `isSidechainMessage`
+ * filter, and the same invariant: a sub-agent's steps render INSIDE its Task
+ * card (via the subagents endpoint), never as top-level rows of the parent.
+ *
+ * Joins on `uuid`, the source line's own id, which `@herdctl/core` preserves on
+ * every parsed message (and, for a paired tool message, takes from the
+ * originating `tool_use` entry — so a sidechain tool call matches on the same
+ * line that made it sidechain). Rides the mtime-cached transcript scan, and
+ * short-circuits to the identity for the overwhelmingly common transcript with
+ * no sidechain lines at all, so this costs nothing on the normal path.
+ *
+ * A transcript that is sidechain top to bottom is not a parent with leaked steps
+ * — it IS a sub-agent's own session, the shape Claude Code writes as its own
+ * `agent-<hex>.jsonl`. herdctl keeps those out of session discovery, but if one
+ * is ever reached directly it should render its content, not an empty chat, so
+ * a drop that would empty the history is refused.
+ */
+export async function dropSidechainMessages(projectDir, sessionId, messages) {
+    const { sidechain } = await readTaskUsesForSession(projectDir, sessionId);
+    if (sidechain.uuids.size === 0)
+        return messages;
+    const kept = messages.filter((m) => !(m.uuid && sidechain.uuids.has(m.uuid)));
+    return kept.length === 0 ? messages : kept;
+}
+/** The same launches minus any a sub-agent made on its own sidechain lane. */
+function withoutSidechainLaunches(uses) {
+    const { taskIds } = uses.sidechain;
+    if (taskIds.size === 0)
+        return uses;
+    const keep = (u) => !taskIds.has(u.toolUseId);
+    return { ...uses, paired: uses.paired.filter(keep), pending: uses.pending.filter(keep) };
+}
 /**
  * Live sub-agent enrichment for a tool frame (issue #429). When the tool is a
  * `Task`/`Agent` launch this turn already recovered (via {@link extractSubagentLaunches}),
@@ -306,10 +371,16 @@ export async function readSubagentMessages(projectDir, sessionId, toolUseId) {
 export async function enrichWithSubagents(projectDir, sessionId, messages) {
     if (!hasSubagentTool(messages))
         return messages;
-    const [taskUses, subagents] = await Promise.all([
+    const [rawTaskUses, subagents] = await Promise.all([
         readTaskUsesForSession(projectDir, sessionId),
         listSubagents(projectDir, sessionId),
     ]);
+    // The join is by FILE ORDER, so it has to see the same launches the caller's
+    // message list does. {@link dropSidechainMessages} has already removed the
+    // sidechain rows from that list, so a nested launch a sub-agent made on its own
+    // lane must come out of the cursor too — leaving it in would shift every
+    // following top-level sub-agent's enrichment onto the wrong card (#727).
+    const taskUses = withoutSidechainLaunches(rawTaskUses);
     const [durations, costs] = await Promise.all([
         subagentDurations(subagents),
         subagentCosts(subagents),
@@ -317,42 +388,85 @@ export async function enrichWithSubagents(projectDir, sessionId, messages) {
     return attachSubagentFields(messages, taskUses, subagents, durations, costs);
 }
 /**
- * Compute each sub-agent's run time (ms) from the first→last timestamp in its
- * own transcript. Keyed by toolUseId. Runs once per session on chat open (only
- * over that session's sub-agent files), so the cost is bounded to the open chat.
+ * How long a sub-agent transcript with no terminal `end_turn` may stay quiet
+ * before we stop calling it live (issue #725).
+ *
+ * A sub-agent that was interrupted — its keeper killed, the box restarted — never
+ * writes an `end_turn`, so without this fallback its card would advertise "still
+ * running" forever and its duration would never appear. The window has to be
+ * comfortably longer than the longest gap a WORKING sub-agent can leave between
+ * transcript lines, which is one slow tool call (a full test suite, a long build),
+ * not one slow model response. Ten minutes clears that with room to spare, and the
+ * `end_turn` check below means the overwhelming majority of finished sub-agents
+ * (392 of the 483 real transcripts this was measured against) never wait for it at
+ * all — they are recognised as finished the instant their last line lands.
+ */
+const SUBAGENT_STALE_MS = 10 * 60_000;
+/**
+ * Compute each FINISHED sub-agent's run time (ms) from the first→last timestamp
+ * in its own transcript. Keyed by toolUseId. Runs once per session on chat open
+ * (only over that session's sub-agent files), so the cost is bounded to the open
+ * chat.
+ *
+ * A sub-agent that still looks live is deliberately left OUT of the map (issue
+ * #725). `subagentDurationMs` is not merely cosmetic — it is the client's
+ * finished signal (`useRunningSubagents` drops any card that carries one) — and
+ * it is derived from a transcript that is still growing, so publishing it for a
+ * running sub-agent both freezes a bogus part-way number onto the card and
+ * evicts it from the running-sub-agents bar for the rest of the run. #622 fixed
+ * that for the `pending` branch of {@link attachSubagentFields} by omitting the
+ * field; the `paired` branch below it kept stamping one, and because the SDK
+ * backgrounds sub-agents, the launching `Task` pairs within milliseconds while
+ * the sub-agent keeps working — so a live sub-agent reaches the paired branch as
+ * a matter of course. Gating at the source fixes both branches by construction.
  */
 async function subagentDurations(subagents) {
     const out = new Map();
+    const now = Date.now();
     await Promise.all([...subagents.values()].map(async (m) => {
-        const ms = await readSubagentDurationMs(m.transcriptPath);
+        const ms = await readFinishedSubagentDurationMs(m.transcriptPath, now);
         if (ms !== undefined)
             out.set(m.toolUseId, ms);
     }));
     return out;
 }
 /**
- * The elapsed wall-clock of a sub-agent transcript: the delta between its first
- * and last `timestamp`. Scans for timestamps without a full JSON parse; returns
- * undefined if the file is missing or has < 2 timestamps.
+ * The elapsed wall-clock of a FINISHED sub-agent transcript, or undefined when it
+ * is missing, has < 2 timestamps, or still looks live (see {@link SUBAGENT_STALE_MS}).
  */
-async function readSubagentDurationMs(file) {
+async function readFinishedSubagentDurationMs(file, now) {
     const mtimeMs = await statMtimeMs(file);
+    const facts = await readSubagentFacts(file, mtimeMs);
+    if (facts.durationMs === undefined)
+        return undefined;
+    // No mtime (the file vanished between the stat and the read) ⇒ nothing is
+    // writing to it, so fall back to the structural check alone.
+    const quietMs = mtimeMs === undefined ? Infinity : now - mtimeMs;
+    if (!facts.endsWithEndTurn && quietMs < SUBAGENT_STALE_MS)
+        return undefined;
+    return facts.durationMs;
+}
+async function readSubagentFacts(file, mtimeMs) {
     if (mtimeMs !== undefined) {
-        const cached = mtimeCacheGet(durationCache, file, mtimeMs);
+        const cached = mtimeCacheGet(factsCache, file, mtimeMs);
         if (cached.hit)
             return cached.value;
     }
-    const value = await readSubagentDurationMsUncached(file);
-    // Cache even undefined (missing/malformed) so a repeat open of an unchanged,
-    // duration-less transcript doesn't re-read the whole file every time.
+    const value = await readSubagentFactsUncached(file);
+    // Cache even the empty result (missing/malformed) so a repeat open of an
+    // unchanged, duration-less transcript doesn't re-read the whole file every time.
     if (mtimeMs !== undefined)
-        mtimeCacheSet(durationCache, file, mtimeMs, value);
+        mtimeCacheSet(factsCache, file, mtimeMs, value);
     return value;
 }
-async function readSubagentDurationMsUncached(file) {
+async function readSubagentFactsUncached(file) {
     const raw = await fs.readFile(file, "utf8").catch(() => null);
     if (raw === null)
-        return undefined;
+        return { endsWithEndTurn: false };
+    return { durationMs: transcriptSpanMs(raw), endsWithEndTurn: endsWithEndTurn(raw) };
+}
+/** First→last `timestamp` delta. Scans for timestamps without a full JSON parse. */
+function transcriptSpanMs(raw) {
     const matches = [...raw.matchAll(/"timestamp":"([^"]+)"/g)];
     if (matches.length < 2)
         return undefined;
@@ -362,6 +476,25 @@ async function readSubagentDurationMsUncached(file) {
         return undefined;
     return last - first;
 }
+/** True when the transcript's last non-empty line is an assistant entry that
+ *  stopped with `end_turn`. Only that line is JSON-parsed. */
+function endsWithEndTurn(raw) {
+    const lines = raw.split("\n");
+    for (let i = lines.length - 1; i >= 0; i--) {
+        const trimmed = lines[i].trim();
+        if (!trimmed)
+            continue;
+        try {
+            const parsed = JSON.parse(trimmed);
+            return parsed.type === "assistant" && parsed.message?.stop_reason === "end_turn";
+        }
+        catch {
+            // A torn final line means the writer is mid-append — definitively not settled.
+            return false;
+        }
+    }
+    return false;
+}
 /**
  * Compute each sub-agent's estimated API-rate cost (USD), keyed by toolUseId, as
  * the RECURSIVE total of the sub-agent PLUS every sub-agent it (transitively)
@@ -611,6 +744,13 @@ function attachSubagentFields(messages, taskUses, subagents, durations, costs) {
                 description: use.description,
                 prompt: use.prompt,
                 hasSubagent: subagents.has(use.toolUseId),
+                // Same hazard as the `pending` branch above, and for the same reason:
+                // a PAIRED `Task` does NOT mean the sub-agent is done — the SDK
+                // backgrounds it, so the tool_result lands in milliseconds while the
+                // sub-agent keeps working. `durations` is therefore gated on liveness at
+                // its source ({@link subagentDurations}) and holds an entry only for a
+                // sub-agent whose transcript has actually settled, so a still-running one
+                // gets `undefined` here and keeps its place in the bar (#725).
                 subagentDurationMs: durations.get(use.toolUseId),
                 subagentCostUsd: costs.get(use.toolUseId),
             },

package/packages/server/dist/tooldetails.d.ts

@@ -24,10 +24,16 @@ export declare function readToolDetails(projectDir: string, sessionId: string):
  */
 export declare function attachToolDetails(projectDir: string, sessionId: string, messages: EnrichedMessage[]): Promise<EnrichedMessage[]>;
 /**
- * The single history-enrichment orchestrator (issue #237). Composes the three
- * server-side passes so the routes call one function: sub-agent recovery (#37) →
- * generalized per-tool detail (#237, subsuming #232's edit diff) → background /
- * task-notification folding (#230) → local-command stdout recovery (#158).
+ * The single history-enrichment orchestrator (issue #237). Composes the
+ * server-side passes so the routes call one function: sidechain drop (#727) →
+ * sub-agent recovery (#37) → generalized per-tool detail (#237, subsuming #232's
+ * edit diff) → background / task-notification folding (#230) → local-command
+ * stdout recovery (#158).
+ *
+ * The sidechain drop goes FIRST and is the history-path mirror of the live path's
+ * `isSidechainMessage` filter: a sub-agent's steps belong inside its Task card,
+ * and every later pass here joins by file order, so they have to be gone before
+ * anything counts positions.
  */
 export declare function enrichWithToolDetails(projectDir: string, sessionId: string, messages: EnrichedMessage[]): Promise<EnrichedMessage[]>;
 export {};

package/packages/server/dist/tooldetails.js

@@ -33,7 +33,7 @@ import { createInterface } from "node:readline";
 import path from "node:path";
 import { projectChatsDir } from "./transcripts.js";
 import { fileKind } from "./projects.js";
-import { enrichWithSubagents } from "./subagents.js";
+import { dropSidechainMessages, enrichWithSubagents, isSidechainLine } from "./subagents.js";
 import { enrichWithBackground } from "./background.js";
 import { enrichWithLocalCommands } from "./localcommand.js";
 import { SAFE_SEGMENT, statMtimeMs, mtimeCacheGet, mtimeCacheSet, } from "./subagents.js";
@@ -355,6 +355,16 @@ async function readToolDetailsFromFileUncached(file) {
             catch {
                 continue;
             }
+            // A sub-agent's steps are not rows of THIS transcript — `dropSidechainMessages`
+            // has already removed them from the message list this list joins against, so
+            // keeping them would shift every following tool's detail by one (#727). Their
+            // detail is derived separately, off the sub-agent's own transcript. (A file
+            // that is sidechain top to bottom — a sub-agent's OWN transcript — therefore
+            // recovers nothing and its messages pass through unenriched, which is a
+            // degrade rather than a misjoin; `dropSidechainMessages` likewise refuses to
+            // empty such a history.)
+            if (isSidechainLine(parsed))
+                continue;
             const content = parsed.message?.content;
             if (!Array.isArray(content))
                 continue;
@@ -453,13 +463,20 @@ export async function attachToolDetails(projectDir, sessionId, messages) {
     });
 }
 /**
- * The single history-enrichment orchestrator (issue #237). Composes the three
- * server-side passes so the routes call one function: sub-agent recovery (#37) →
- * generalized per-tool detail (#237, subsuming #232's edit diff) → background /
- * task-notification folding (#230) → local-command stdout recovery (#158).
+ * The single history-enrichment orchestrator (issue #237). Composes the
+ * server-side passes so the routes call one function: sidechain drop (#727) →
+ * sub-agent recovery (#37) → generalized per-tool detail (#237, subsuming #232's
+ * edit diff) → background / task-notification folding (#230) → local-command
+ * stdout recovery (#158).
+ *
+ * The sidechain drop goes FIRST and is the history-path mirror of the live path's
+ * `isSidechainMessage` filter: a sub-agent's steps belong inside its Task card,
+ * and every later pass here joins by file order, so they have to be gone before
+ * anything counts positions.
  */
 export async function enrichWithToolDetails(projectDir, sessionId, messages) {
-    const withSubagents = await enrichWithSubagents(projectDir, sessionId, messages);
+    const topLevel = await dropSidechainMessages(projectDir, sessionId, messages);
+    const withSubagents = await enrichWithSubagents(projectDir, sessionId, topLevel);
     const withDetails = await attachToolDetails(projectDir, sessionId, withSubagents);
     const withBackground = enrichWithBackground(withDetails);
     return enrichWithLocalCommands(projectDir, sessionId, withBackground);

package/packages/server/dist/trusted-proxy.js

@@ -40,7 +40,7 @@
  * A request from another host to a `0.0.0.0`-published port is SNAT'd to that
  * same address, so "peer is the gateway" does not prove the traffic stayed on
  * the box. Smoke-test from INSIDE the container instead (`docker compose exec
- * paddock curl http://127.0.0.1:4000/mcp …`), where the peer really is loopback.
+ * paddock curl http://127.0.0.1:7233/mcp …`), where the peer really is loopback.
  */
 import proxyAddr from "proxy-addr";
 /**

package/packages/server/dist/turn-interlock.d.ts

@@ -0,0 +1,101 @@
+/**
+ * Turn interlock (issue #731) — stop a chat's in-flight turn BEFORE a destructive
+ * operation mutates its transcript.
+ *
+ * ## The bug this exists to make impossible
+ *
+ * A chat's transcript is not something paddock writes: `claude` writes
+ * `<sessionId>.jsonl` itself, straight through the symlink `ensureProjectAgent`
+ * plants (see `transcripts.ts`). So a destructive operation — delete, promote,
+ * revert, project delete — is a race against a live subprocess that still holds
+ * that path and will append to it whenever it next has something to say.
+ *
+ * Deleting the file mid-turn therefore did not delete the chat. The unlink
+ * landed, the chat left the list, and ~45s later the still-running process
+ * re-created the SAME file containing only its own tail: a 3-line transcript
+ * opening on an orphan `tool_result` whose `tool_use` no longer existed. The
+ * chat came back, named after its raw session id, with every prior turn
+ * permanently gone. Promote lost the chat from BOTH projects the same way.
+ *
+ * ## The guarantee
+ *
+ * Nothing here tries to out-race the writer, because that race is unwinnable.
+ * Instead {@link quiesceSession} refuses to return "go ahead" until the session
+ * is verifiably dead — and a caller that cannot get that answer within its
+ * timeout must NOT mutate (the routes return 409 instead). A destructive op
+ * therefore only ever runs against a transcript no process holds, and a dead
+ * session has nothing left to write itself back with.
+ *
+ * ## Why two liveness signals
+ *
+ * `hub.isRunning` sees the WS layer's turn, and `herdctl.isSdkSessionLive` sees
+ * herdctl's `SessionReaper` — a session held open past its turn (a killed
+ * background task's keepAlive, or the re-invocation grace, #397). Either one
+ * means a `claude` process may still write. The recovery engine already pairs
+ * them for exactly this reason (`ws-turn.ts`); so do we.
+ *
+ * ## Why cancel is not enough on its own
+ *
+ * `herdctl.cancel(jobId)` ends the model TURN. On the session runtime that is an
+ * `interrupt()`, which deliberately leaves the session — and its subprocess —
+ * alive for the next message. That is right for the Stop button and wrong for a
+ * delete, so we also reap the managed session. Cancel and reap are both
+ * best-effort; the polled liveness check is what actually decides.
+ */
+import type { SessionHub } from "./session-hub.js";
+import type { HerdctlService } from "./herdctl.js";
+/** How long to wait for a cancelled turn to actually die before giving up. */
+export declare const DEFAULT_QUIESCE_TIMEOUT_MS = 10000;
+/**
+ * The dependencies the interlock needs. Both come off the route context: `hub`
+ * via `ctx.managementOpsContext?.hub` (hence optional — a server built without
+ * the WS layer has no hub, and correctly reports nothing running).
+ */
+export interface TurnInterlockDeps {
+    hub: SessionHub | undefined;
+    herdctl: HerdctlService;
+}
+/**
+ * - `idle` — nothing was running; the caller may proceed.
+ * - `stopped` — a turn WAS running, was stopped, and is now verifiably dead;
+ *   the caller may proceed.
+ * - `stuck` — something is still alive after the timeout. The caller MUST NOT
+ *   mutate the transcript: that is precisely the resurrection case.
+ */
+export type QuiesceOutcome = "idle" | "stopped" | "stuck";
+/**
+ * Is anything alive on this session right now — a live WS turn, or a session
+ * herdctl's reaper is still holding open?
+ */
+export declare function sessionIsBusy(deps: TurnInterlockDeps, sessionId: string): boolean;
+/**
+ * Stop whatever is running against `sessionId` and WAIT for it to actually be
+ * gone. Returns `stuck` rather than throwing, so each caller can pick its own
+ * failure shape (a 409, or a `failed` bucket entry in a batch).
+ *
+ * A running turn is not always cancellable the instant we look at it: `jobId`
+ * is null early in a turn (it arrives with the first frames). That is why this
+ * polls rather than cancelling once — each pass re-reads `activeInfo` and
+ * cancels any job id it has not cancelled yet, so a turn that arms its job id a
+ * beat later is still caught inside the same call.
+ */
+export declare function quiesceSession(deps: TurnInterlockDeps, sessionId: string, opts?: {
+    timeoutMs?: number;
+}): Promise<QuiesceOutcome>;
+/**
+ * Quiesce every session with a running turn in `projectSlug`, ahead of a project
+ * delete. Returns the ids that could NOT be stopped — empty means it is safe to
+ * remove the project directory.
+ *
+ * Only the hub can enumerate running turns by project, so a server with no hub
+ * simply has nothing to stop.
+ */
+export declare function quiesceProject(deps: TurnInterlockDeps, projectSlug: string, opts?: {
+    timeoutMs?: number;
+}): Promise<string[]>;
+/** The 409 body every refusing route sends, so clients can match one code. */
+export declare function turnRunningError(sessionIds: string[]): {
+    error: string;
+    code: "turn_running";
+    sessionIds: string[];
+};

package/packages/server/dist/turn-interlock.js

@@ -0,0 +1,84 @@
+/** How long to wait for a cancelled turn to actually die before giving up. */
+export const DEFAULT_QUIESCE_TIMEOUT_MS = 10_000;
+/** How often to re-check liveness while waiting. */
+const POLL_MS = 25;
+const delay = (ms) => new Promise((r) => setTimeout(r, ms));
+/**
+ * Is anything alive on this session right now — a live WS turn, or a session
+ * herdctl's reaper is still holding open?
+ */
+export function sessionIsBusy(deps, sessionId) {
+    if (deps.hub?.isRunning(sessionId) === true)
+        return true;
+    return deps.herdctl.isSdkSessionLive(sessionId);
+}
+/**
+ * Stop whatever is running against `sessionId` and WAIT for it to actually be
+ * gone. Returns `stuck` rather than throwing, so each caller can pick its own
+ * failure shape (a 409, or a `failed` bucket entry in a batch).
+ *
+ * A running turn is not always cancellable the instant we look at it: `jobId`
+ * is null early in a turn (it arrives with the first frames). That is why this
+ * polls rather than cancelling once — each pass re-reads `activeInfo` and
+ * cancels any job id it has not cancelled yet, so a turn that arms its job id a
+ * beat later is still caught inside the same call.
+ */
+export async function quiesceSession(deps, sessionId, opts = {}) {
+    if (!sessionIsBusy(deps, sessionId))
+        return "idle";
+    const deadline = Date.now() + (opts.timeoutMs ?? DEFAULT_QUIESCE_TIMEOUT_MS);
+    const cancelled = new Set();
+    let reaped = false;
+    // Mark the turn end this is about to cause as OURS, not a user's Stop. The WS
+    // layer treats the two oppositely for anything queued behind the turn: a Stop
+    // hands the message back to the user's composer, while this caller is about to
+    // delete, revert or promote the transcript — so the queue must be left strictly
+    // alone. Draining it here would start a fresh turn on a chat being removed (the
+    // #730 resurrection), and would make the session busy again so the poll below
+    // could never settle.
+    deps.hub?.noteCancel(sessionId, { reason: "quiesce", origin: null });
+    for (;;) {
+        const jobId = deps.hub?.activeInfo(sessionId)?.jobId ?? null;
+        if (jobId && !cancelled.has(jobId)) {
+            cancelled.add(jobId);
+            await deps.herdctl.cancel(jobId).catch(() => false);
+        }
+        // Close the managed session too — on the session runtime `cancel` only
+        // interrupts the turn and leaves the subprocess holding the transcript.
+        // Once is enough; a second reap of an already-closed session is a no-op.
+        if (!reaped) {
+            reaped = true;
+            deps.herdctl.reapSession(sessionId);
+        }
+        if (!sessionIsBusy(deps, sessionId))
+            return "stopped";
+        if (Date.now() >= deadline)
+            return "stuck";
+        await delay(POLL_MS);
+    }
+}
+/**
+ * Quiesce every session with a running turn in `projectSlug`, ahead of a project
+ * delete. Returns the ids that could NOT be stopped — empty means it is safe to
+ * remove the project directory.
+ *
+ * Only the hub can enumerate running turns by project, so a server with no hub
+ * simply has nothing to stop.
+ */
+export async function quiesceProject(deps, projectSlug, opts = {}) {
+    const running = (deps.hub?.runningSessions() ?? []).filter((i) => i.projectSlug === projectSlug);
+    // Concurrent: each turn dies independently, so the whole call is bounded by one
+    // timeout rather than N of them stacked end to end.
+    const outcomes = await Promise.all(running.map(async (info) => [info.sessionId, await quiesceSession(deps, info.sessionId, opts)]));
+    return outcomes.filter(([, outcome]) => outcome === "stuck").map(([sessionId]) => sessionId);
+}
+/** The 409 body every refusing route sends, so clients can match one code. */
+export function turnRunningError(sessionIds) {
+    return {
+        error: sessionIds.length === 1
+            ? `A turn is still running on chat ${sessionIds[0]} and could not be stopped; try again.`
+            : `Turns are still running on ${sessionIds.length} chats and could not be stopped; try again.`,
+        code: "turn_running",
+        sessionIds,
+    };
+}

package/packages/server/dist/ws-protocol.d.ts

@@ -214,10 +214,23 @@ export interface ChatSetQueueMessage {
         /** The queued message text, or null/empty to clear. */
         text?: string | null;
         /**
-         * Client-stamped identity of this queued message (#245): the ms timestamp of
-         * when it was first enqueued, stable across reloads. The server records the ts
-         * of the message it last drained per session so it can skip re-sending a stale
-         * copy a reloaded client re-asserts. Absent → the server stamps `Date.now()`.
+         * OPAQUE identity of this client's queue (#245 stable identity, #736). Minted
+         * once when the queue is first created and kept across edits, appends and
+         * reloads, so the server can tell (a) this client updating its own queue apart
+         * from another client queueing alongside it (#629) and (b) a stale re-assert of
+         * an already-drained message apart from a new one — by EQUALITY, never ordering.
+         *
+         * Absent from an older client, which sends only `ts`; the server folds that
+         * into an id.
+         */
+        qid?: string | null;
+        /**
+         * Legacy client-stamped enqueue time (#245). Superseded by `qid`: it was used
+         * as an ORDERED identity, so one client with a fast clock poisoned the dedup
+         * marker and every later queued message on that chat — from any client — was
+         * silently destroyed (#736). Still accepted (older clients send it, and it
+         * still identifies a message) but only ever compared for equality, and never
+         * stored as the enqueue time — the server stamps that itself.
          */
         ts?: number | null;
     };
@@ -382,6 +395,64 @@ export interface ChatQueuedFlushedMessage {
         text?: string;
     };
 }
+/**
+ * The chat's queued message changed (#629). Broadcast to EVERY socket attached to
+ * the session whenever the slot is written, so the queue is shared chat state that
+ * all clients render identically instead of invisible per-client state.
+ *
+ * Before this, a queue was only ever written and never announced: a second client
+ * queueing on the same chat overwrote the first client's text with no signal to
+ * anyone. The first user's chip sat there showing a message that no longer existed,
+ * and when the drain finally fired, their transcript rendered — as their own user
+ * bubble — a message they had never typed. The slot now MERGES contributions, and
+ * this frame is what makes the merge visible.
+ */
+export interface ChatQueuedStateMessage {
+    type: "chat:queued_state";
+    payload: {
+        projectSlug: string;
+        sessionId: string;
+        /** The chat's full queued text, or null when the slot is now empty. */
+        text: string | null;
+        /**
+         * The slot's identity. A client adopts it as its own queue id, so its next
+         * edit updates this shared slot in place rather than appending beside it.
+         */
+        qid?: string;
+        /**
+         * Why the slot changed, when it is worth explaining. `returned` means a user
+         * pressed Stop and the message went back to THEIR composer — sent to the
+         * other clients so a chip that just vanished from under them has a reason
+         * attached, rather than looking like the message evaporated. The client that
+         * pressed Stop is excluded from this broadcast; it gets
+         * {@link ChatQueuedReturnedMessage} instead.
+         */
+        reason?: "returned";
+    };
+}
+/**
+ * A user pressed Stop, so the message queued behind that turn is handed BACK to
+ * them (#751 follow-up). Sent only to the socket that asked to cancel.
+ *
+ * Stop means "give me control back". Auto-sending the follow-up would have the
+ * agent start working again the instant the user stopped it, and holding the
+ * message server-side is what stranded it (#627) — so it returns to the composer,
+ * where it is visible, editable, and sent only when the user says so.
+ *
+ * Deliberately NOT `chat:queued_flushed` with a flag: that frame means "this text
+ * was sent, render it as the user's bubble", and a returned message is precisely
+ * one that was NOT sent. Overloading it risks a phantom user turn in the
+ * transcript for a message the agent never received.
+ */
+export interface ChatQueuedReturnedMessage {
+    type: "chat:queued_returned";
+    payload: {
+        projectSlug: string;
+        sessionId: string;
+        /** The queued text, for the client to put back in its composer. */
+        text: string;
+    };
+}
 /**
  * A background task was killed at the turn boundary and the keeper is idle
  * (issue #347). Broadcast LIVE by the recovery engine the moment the kill is
@@ -418,6 +489,6 @@ export interface ChatNoticeMessage {
 export interface PongMessage {
     type: "pong";
 }
-export type ServerMessage = ChatResponseMessage | ChatToolCallMessage | ChatToolStartMessage | ChatMessageBoundaryMessage | ChatCompleteMessage | ChatErrorMessage | ChatResyncMessage | ChatActiveMessage | ChatQueuedFlushedMessage | ChatKilledTaskMessage | ChatNoticeMessage | PongMessage;
+export type ServerMessage = ChatResponseMessage | ChatToolCallMessage | ChatToolStartMessage | ChatMessageBoundaryMessage | ChatCompleteMessage | ChatErrorMessage | ChatResyncMessage | ChatActiveMessage | ChatQueuedFlushedMessage | ChatQueuedStateMessage | ChatQueuedReturnedMessage | ChatKilledTaskMessage | ChatNoticeMessage | PongMessage;
 export declare function isClientMessage(data: unknown): data is ClientMessage;
 export {};

package/packages/server/dist/ws-protocol.js

@@ -245,6 +245,8 @@ export function isClientMessage(data) {
             return false;
         if (p.ts !== undefined && p.ts !== null && typeof p.ts !== "number")
             return false;
+        if (p.qid !== undefined && p.qid !== null && typeof p.qid !== "string")
+            return false;
         return true;
     }
     return false;