@songsid/agend 2.3.0 → 2.3.1

package/dist/fleet-manager.js

@@ -217,6 +217,8 @@ import { assertExplicitInstanceRemoval } from "./instance-removal.js";
 import { KNOWN_BACKENDS, validateFleetConfig } from "./config-validator.js";
 import { isRemovedBackend, removedBackendMessage } from "./backend/removed.js";
 import { presentationState, interactionSummary, sameInteractionOwner } from "./interaction-observation.js";
+import { describeSelfOom } from "./session-oom.js";
+import { assertNoSessionOomStop, readSessionOomStop, captureSessionOomRetry, clearSessionOomStopForRetry, SessionOomStoppedError, sessionOomStopHint } from "./session-oom-stop.js";
 import { idleGateTimeoutLog } from "./idle-gate-log.js";
 import { DEFAULT_STUCK_TIMEOUT_MS, readLastInboundAt } from "./daemon.js";
 import { clearPausedMarker, readPausedAt, readPauseReason, writePausedMarker } from "./pause-marker.js";
@@ -460,6 +462,7 @@ const EFFORT_SELECT_CALLBACK_PREFIX = "effort-select:";
 const BACKEND_SELECT_CALLBACK_PREFIX = "backend-select:";
 const INTERACTIVE_ASSIST_CALLBACK_PREFIX = "interactive-assist:";
 const EXIT_RESTART_CALLBACK_PREFIX = "exit-restart:";
+const OOM_RETRY_CALLBACK_PREFIX = "oom-retry:";
 /** #1386: how long a paused instance's marker reason is reused before it is read again (transitions drop it at once). */
 const NEEDS_PAUSE_TTL_MS = 10_000;
 const HANG_CALLBACK_PREFIX = "hang:";
@@ -471,7 +474,7 @@ const CLEAR_CONFIRM_CALLBACK_PREFIX = "clear-confirm:";
  * group's approval, tips, and the per-user /model and /effort menus (which are not nonce entries at all).
  */
 const WEB_MIRRORED_PROMPT_PREFIXES = new Set([
-    HANG_CALLBACK_PREFIX, EXIT_RESTART_CALLBACK_PREFIX, INTERACTIVE_ASSIST_CALLBACK_PREFIX,
+    HANG_CALLBACK_PREFIX, EXIT_RESTART_CALLBACK_PREFIX, OOM_RETRY_CALLBACK_PREFIX, INTERACTIVE_ASSIST_CALLBACK_PREFIX,
 ]);
 /**
  * Where a reply goes on a fleet with no chat platform (web track C4): it is "sent" by being shown in the
@@ -759,6 +762,8 @@ export class FleetManager {
      * backend outage during a post-update herd). instanceName → pending retry.
      */
     startupRetries = new Map();
+    /** #2484: instances whose automatic startup retry is running now (between its timer and its outcome). */
+    startupRetriesRunning = new Set();
     /** Bumped by every explicit stop (incl. the stop half of a restart); fences the outage hand-off. */
     explicitStopGeneration = new Map();
     /** instanceName → outage hand-off currently executing; a repeat joins it. */
@@ -1792,7 +1797,7 @@ export class FleetManager {
                     ...(web && by.guest ? { web_guest: "1" } : {}), // #1920: "[user:… via web (guest), id:…]", as /ui/send
                     ...reactions.meta,
                 },
-            }, by.humanOrigin, button.label)) !== false;
+            }, by.humanOrigin, button.label, this.startRetryHoldFor(set.instance, inbound))) !== false;
         }
         finally {
             // Both false and throw reopen the choice in the controller. Finish the
@@ -1841,8 +1846,18 @@ export class FleetManager {
         }
         this.deliveryOutbox = outbox;
         this.deliveryOutboxRecovered = true;
+        outbox.on("working_references", (names) => {
+            for (const name of names)
+                this.observeWorkingState(name, "obligations");
+        });
         outbox.on("admitted", () => { this.scheduleDeliveryOutboxPump(); this.wakeCoordinator?.kick(); });
         outbox.on("state", (event) => {
+            if (event.deliveryId) {
+                const row = outbox.get(event.deliveryId);
+                if (row)
+                    for (const name of new Set([row.sourceInstance, row.targetInstance]))
+                        this.observeWorkingState(name, "obligations");
+            }
             this.scheduleDeliveryOutboxPump();
             this.wakeCoordinator?.kick();
             if (event.deliveryId && event.state === "delivered") {
@@ -2001,8 +2016,19 @@ export class FleetManager {
                 : item),
         };
     }
+    /** A terminal OOM stop is not an auto-wakeable pause, including after a fleet restart. */
+    sessionOomStopped(name) {
+        return this.daemons.get(name)?.isSessionOomStopped?.() === true || readSessionOomStop(this.getInstanceDir(name)).blocked;
+    }
+    sessionOomStopReason(name) {
+        const state = readSessionOomStop(this.getInstanceDir(name));
+        return state.blocked ? sessionOomStopHint(state)
+            : this.daemons.get(name)?.notAcceptingReason?.() ?? sessionOomStopHint();
+    }
     /** #1641: why a target is not taking queued work right now, or null when nothing known holds it. */
     deliveryWaitingReason(target) {
+        if (this.sessionOomStopped(target))
+            return this.sessionOomStopReason(target);
         if (this.lifecycle.isPaused(target))
             return t("delivery.hold_state", "paused");
         return this.daemons.get(target)?.notAcceptingReason?.() ?? null;
@@ -2015,6 +2041,9 @@ export class FleetManager {
     }
     /** Called after a replacement Daemon object is installed in lifecycle.daemons. */
     onDaemonReady(name, daemonBootId) {
+        const daemon = this.daemons.get(name);
+        if (daemon?.bootId === daemonBootId)
+            this.listenForWorkingState(name, daemon);
         const outbox = this.deliveryOutbox;
         if (!outbox)
             return;
@@ -2038,6 +2067,82 @@ export class FleetManager {
         }
         this.scheduleDeliveryOutboxPump();
     }
+    workingStateListeners;
+    workingStateErrors;
+    /** Attach only to the published owner. Nothing here sends or resumes work. */
+    listenForWorkingState(name, daemon) {
+        if (typeof daemon.getWorkingStateObservation !== "function")
+            return;
+        this.workingStateListeners ??= new WeakSet();
+        if (this.workingStateListeners.has(daemon))
+            return;
+        this.workingStateListeners.add(daemon);
+        daemon.on("working_state", (event) => {
+            try {
+                if (this.daemons.get(name) !== daemon || !sameInteractionOwner(event.observation.owner, daemon.getInteractionSnapshot().owner))
+                    return;
+                this.observeWorkingState(name, event.reason, event.observation);
+            }
+            catch { /* A malformed/stale observation cannot affect the daemon. */ }
+        });
+        this.observeWorkingState(name, "published");
+    }
+    observeWorkingState(name, reason, supplied) {
+        const outbox = this.deliveryOutbox, daemon = this.daemons.get(name);
+        if (!outbox || !daemon || typeof daemon.getWorkingStateObservation !== "function")
+            return;
+        try {
+            const observation = supplied ?? daemon.getWorkingStateObservation();
+            if (!sameInteractionOwner(observation.owner, daemon.getInteractionSnapshot().owner))
+                return;
+            const references = [];
+            const unavailable = [];
+            let totalReferences = 0;
+            const board = this.scheduler?.db.board;
+            if (!board)
+                unavailable.push("tasks");
+            else
+                try {
+                    const tasks = board.listCards({ assignee: name, status: ["claimed", "blocked", "review"], limit: 16 });
+                    totalReferences += tasks.total;
+                    for (const card of tasks.cards)
+                        references.push({ kind: "task", id: card.uuid, version: card.version,
+                            status: card.status });
+                }
+                catch {
+                    unavailable.push("tasks");
+                }
+            try {
+                const obligations = outbox.workingObligations(name);
+                references.push(...obligations.references);
+                totalReferences += obligations.total;
+            }
+            catch {
+                unavailable.push("obligations");
+            }
+            const entry = outbox.observeWorkingState(name, { version: 1, reason, observedAt: Date.now(), observation,
+                lastActivityAt: this.lastActivity.get(name) ?? null, references, totalReferences, unavailable });
+            if (!entry)
+                return;
+            const state = entry.current.observation.executionState ?? "unknown";
+            const counts = { tasks: entry.current.references.filter(r => r.kind === "task").length,
+                owed: entry.current.references.filter(r => r.kind === "owed").length,
+                waiting: entry.current.references.filter(r => r.kind === "waiting").length,
+                background: entry.current.observation.backgroundWork,
+                total: entry.current.totalReferences, unavailable: entry.current.unavailable };
+            this.logger.info({ name, state, reason, revision: entry.revision, ...counts }, "Working-state journal observed");
+            this.eventLog?.insert(name, "working_state", { state, reason, revision: entry.revision, ...counts,
+                message: `Observed ${state}; tasks ${counts.tasks}, replies owed ${counts.owed}, waiting ${counts.waiting}; ${counts.total} references${counts.background === true ? "; background work running" : ""}${unavailable.length ? "; some sources unavailable" : ""}. Journal only; no recovery action.` });
+            this.emitSseEvent("steps_tick", { instance: name });
+        }
+        catch {
+            this.workingStateErrors ??= new Set();
+            if (!this.workingStateErrors.has(name)) {
+                this.workingStateErrors.add(name);
+                this.logger.warn({ name }, "Could not persist working-state observation; delivery and restart behavior unchanged");
+            }
+        }
+    }
     /**
      * #926: the two reply-obligation safety nets. Idle owners with an unanswered
      * request get one reminder (after the grace); requesters get one overdue
@@ -3725,7 +3830,7 @@ export class FleetManager {
                 finish(false);
         });
     }
-    async deliverWithIdleGate(instanceName, payload, timeoutMs, deliveryEpoch, noInlineWake = false, stillCurrent) {
+    async deliverWithIdleGate(instanceName, payload, timeoutMs, deliveryEpoch, noInlineWake = false, stillCurrent, onStartRetryHold) {
         if (this.shuttingDown || !this.deliveryCurrent(instanceName, deliveryEpoch, stillCurrent))
             return false;
         await this.holdDeliveryForStorm(instanceName, deliveryEpoch);
@@ -3764,7 +3869,7 @@ export class FleetManager {
             const log = idleGateTimeoutLog(instanceName, timeoutMs, this.instanceStateCache.get(instanceName), Date.now());
             this.logger[log.level](log.fields, log.message);
         }
-        const sent = await this.sendWhenConnected(instanceName, payload, deliveryEpoch, stillCurrent);
+        const sent = await this.sendWhenConnected(instanceName, payload, deliveryEpoch, stillCurrent, onStartRetryHold);
         if (!sent)
             return false;
         if (this.deliveryCurrent(instanceName, deliveryEpoch, stillCurrent)) {
@@ -3793,8 +3898,11 @@ export class FleetManager {
      * Ordering is preserved by serialising behind any waiter already queued for this
      * instance, *including* when the socket happens to be up: otherwise a message
      * arriving after the reconnect could overtake one that has been waiting for it.
+     *
+     * `onStartRetryHold` (#2484, a person's message): an instance in its own start-retry backoff is waited for until that
+     * retry brings it up, however long the backoff, and fails only once the retries give up.
      */
-    async sendWhenConnected(instanceName, payload, deliveryEpoch = this.getDeliveryEpoch(instanceName), stillCurrent) {
+    async sendWhenConnected(instanceName, payload, deliveryEpoch = this.getDeliveryEpoch(instanceName), stillCurrent, onStartRetryHold) {
         if (!this.deliveryCurrent(instanceName, deliveryEpoch, stillCurrent))
             return false;
         const queued = this.ipcWaitTails.get(instanceName);
@@ -3808,7 +3916,7 @@ export class FleetManager {
             .then(() => {
             if (!this.deliveryCurrent(instanceName, deliveryEpoch, stillCurrent))
                 return false;
-            return this.sendAfterIpcReturns(instanceName, payload, deliveryEpoch, stillCurrent);
+            return this.sendAfterIpcReturns(instanceName, payload, deliveryEpoch, stillCurrent, onStartRetryHold);
         });
         // The chain stores a settled-either-way promise so one failed delivery cannot
         // wedge every later one, and so `queued` above is safe to await unguarded.
@@ -3826,9 +3934,10 @@ export class FleetManager {
         }
     }
     /** Poll for the instance's IPC to come back, then send. Throws if it does not. */
-    async sendAfterIpcReturns(instanceName, payload, deliveryEpoch = this.getDeliveryEpoch(instanceName), stillCurrent) {
-        const deadline = Date.now() + IPC_RECONNECT_GRACE_MS;
+    async sendAfterIpcReturns(instanceName, payload, deliveryEpoch = this.getDeliveryEpoch(instanceName), stillCurrent, onStartRetryHold) {
+        let deadline = Date.now() + IPC_RECONNECT_GRACE_MS;
         let warned = false;
+        let held = false;
         for (;;) {
             if (!this.deliveryCurrent(instanceName, deliveryEpoch, stillCurrent))
                 return false;
@@ -3837,6 +3946,22 @@ export class FleetManager {
             const ipc = this.instanceIpcClients.get(instanceName);
             if (ipc?.connected && ipc.send(this.bindHumanReceiptPayload(instanceName, payload)))
                 return true;
+            // #2484: while the instance waits for (or runs) its own start retry, the grace runs from now — so the message is
+            // delivered once a retry brings it up, and fails only after the retries stop (they gave up, or a person stopped
+            // or paused it) and a last grace passes without it.
+            if (onStartRetryHold && this.inStartRetryBackoff(instanceName)) {
+                deadline = Math.max(deadline, Date.now() + IPC_RECONNECT_GRACE_MS);
+                if (!held) {
+                    held = true;
+                    this.logger.info({ instanceName, retry: this.startupRetries.get(instanceName)?.attempt }, "Instance is waiting for its start retry — holding the message until it is up");
+                    try {
+                        onStartRetryHold();
+                    }
+                    catch (err) {
+                        this.logger.debug({ err, instanceName }, "Start-retry hold notice failed");
+                    }
+                }
+            }
             if (Date.now() >= deadline) {
                 throw new Error(`Instance '${instanceName}' IPC is unavailable`);
             }
@@ -4222,6 +4347,18 @@ export class FleetManager {
                 this.deliveryOutbox?.markHumanReceiptUncertain(id, instanceName);
         }
     }
+    /**
+     * #2484: a person's message waits for its instance's own start retry (DeliveryOptions.onStartRetryHold), shown as
+     * queued on the message meanwhile. A ClassicBot room message (no `msg`) has no delivery-status reactions: it only waits.
+     */
+    startRetryHoldFor(instanceName, msg) {
+        return {
+            onStartRetryHold: () => {
+                if (msg?.chatId && msg.messageId)
+                    this.reactMessageStatus(instanceName, msg.chatId, msg.messageId, "queued", msg.threadId || undefined);
+            },
+        };
+    }
     /** Synchronous steer/btw handoff, using the same evidence and owner binding. */
     sendHumanControl(instanceName, payload, origin, originalBody) {
         const ipc = this.instanceIpcClients.get(instanceName);
@@ -4262,7 +4399,7 @@ export class FleetManager {
                     return false;
                 this.enforceWarmCap(instanceName); // woke one → evict a different LRU idle if over cap
             }
-            const sent = await this.sendWhenConnected(instanceName, deliveryPayload, deliveryEpoch, options.stillCurrent);
+            const sent = await this.sendWhenConnected(instanceName, deliveryPayload, deliveryEpoch, options.stillCurrent, options.onStartRetryHold);
             if (!sent)
                 return false;
             // A cross-instance item arriving before the daemon observes this turn as
@@ -4273,7 +4410,7 @@ export class FleetManager {
             return true;
         }
         const previous = this.idleGatedDeliveryTails.get(instanceName) ?? Promise.resolve();
-        const delivery = previous.catch(() => { }).then(() => this.deliverWithIdleGate(instanceName, deliveryPayload, options.idleTimeoutMs ?? 60_000, deliveryEpoch, options.noInlineWake === true, options.stillCurrent));
+        const delivery = previous.catch(() => { }).then(() => this.deliverWithIdleGate(instanceName, deliveryPayload, options.idleTimeoutMs ?? 60_000, deliveryEpoch, options.noInlineWake === true, options.stillCurrent, options.onStartRetryHold));
         this.idleGatedDeliveryTails.set(instanceName, delivery);
         try {
             return await delivery;
@@ -4730,7 +4867,7 @@ export class FleetManager {
             mode: target => this.deliveryWorkerMode(target),
             available: () => !this.shuttingDown && this.deliveryOutbox?.isOpen === true,
             listPending: () => this.deliveryOutbox?.listPending() ?? [],
-            isPaused: target => this.lifecycle.isPaused(target),
+            isPaused: target => !this.sessionOomStopped(target) && this.lifecycle.isPaused(target),
             pauseReason: target => readPauseReason(this.getInstanceDir(target)),
             isRestarting: target => this.isInstanceRestarting(target),
             wake: async (target) => {
@@ -4742,7 +4879,7 @@ export class FleetManager {
             warmCap: () => this.fleetConfig?.defaults?.warm_cap ?? 0,
             warmOverflow: () => this.fleetConfig?.defaults?.warm_overflow ?? DEFAULT_WARM_OVERFLOW,
             wokeAt: target => this.lifecycle.wokeAtFor(target),
-            notAcceptingReason: target => this.daemons.get(target)?.notAcceptingReason?.() ?? null,
+            notAcceptingReason: target => this.sessionOomStopped(target) ? this.sessionOomStopReason(target) : this.daemons.get(target)?.notAcceptingReason?.() ?? null,
             notifyTarget: (target, text) => { this.notifyInstanceTopic(target, text); },
             notifySender: (source, text) => { this.notifyInstanceTopic(source, text); },
             kickPump: () => this.scheduleDeliveryOutboxPump(),
@@ -4797,7 +4934,12 @@ export class FleetManager {
      * is exactly the old lifecycle wake; otherwise it goes through the shared
      * slot admission and joins any wake already in flight for the target.
      */
-    async explicitWake(name, timeoutMs = 30_000) {
+    async explicitWake(name, timeoutMs = 30_000, retryStoppedSession = false) {
+        if (this.sessionOomStopped(name)) {
+            if (!retryStoppedSession)
+                throw new SessionOomStoppedError(false, readSessionOomStop(this.getInstanceDir(name)));
+            return this.restartSingleInstance(name, { explicit: true });
+        }
         const token = await this.acquireWakeSlot(name);
         try {
             await this.lifecycle.wake(name, timeoutMs);
@@ -4818,7 +4960,7 @@ export class FleetManager {
     }
     async changeInstancePauseState(name, action) {
         if (action === "wake") {
-            await this.explicitWake(name, 30_000);
+            await this.explicitWake(name, 30_000, true);
             this.enforceWarmCap(name); // manual wake still respects the fleet warm cap
             return "awake";
         }
@@ -4994,6 +5136,11 @@ export class FleetManager {
     /** Phase 2a: the caller's transition, for a start made from inside a restart or wake. */
     transition, execution) {
         execution?.assert();
+        if (this.sessionOomStopped(name)) {
+            if (resumePaused)
+                return this.restartSingleInstance(name, { explicit: true });
+            throw new SessionOomStoppedError(false, readSessionOomStop(this.getInstanceDir(name)));
+        }
         // CLI single-instance cold starts bypass startAll. Start diagnostics before
         // any lifecycle/wake work; a shutdown must not revive the stopped sampler.
         if (!this.shuttingDown)
@@ -5002,7 +5149,7 @@ export class FleetManager {
         // into a running instance — harmless, but noisy — or race this start).
         this.cancelStartupRetry(name);
         if (resumePaused && this.lifecycle.isPaused(name)) {
-            await this.explicitWake(name, 30_000);
+            await this.explicitWake(name, 30_000, true);
             // A successful wake clears the persisted pause marker and produces a
             // fresh instance_state snapshot.  Drop any stale process error left by a
             // pre-pause crash so /api/fleet and `agend ls` converge on running.
@@ -5022,7 +5169,7 @@ export class FleetManager {
             if (resumePaused) {
                 const status = this.getInstanceStatus(name);
                 if (status === "crashed" || status === "stopped") {
-                    await this.restartSingleInstance(name);
+                    await this.restartSingleInstance(name, { explicit: true });
                     return;
                 }
             }
@@ -5197,6 +5344,8 @@ export class FleetManager {
             // outcome. Retrying would start an instance the operator just stopped.
             if (err instanceof SupersededStartError)
                 return false;
+            if (err instanceof SessionOomStoppedError)
+                await this.notifySessionOomStop(name);
             if (this.stopOnUnsupportedCli(name, err, what))
                 return false;
             this.logger.error({ err, name }, `Failed to start ${what}`);
@@ -5217,6 +5366,11 @@ export class FleetManager {
     scheduleStartupRetry(name, attempt) {
         if (this.shuttingDown)
             return;
+        if (this.sessionOomStopped(name)) {
+            this.instanceProcessStatus.set(name, "stopped");
+            this.logger.warn({ name }, this.sessionOomStopReason(name));
+            return;
+        }
         if (this.startupRetries.has(name))
             return;
         const backoff = FleetManager.STARTUP_RETRY_BACKOFF_MS;
@@ -5244,6 +5398,12 @@ export class FleetManager {
         this.startupRetries.delete(name);
         this.logger.info({ name }, "Pending automatic startup retry cancelled");
     }
+    /** #2484: the instance's own start retry is scheduled or running, and no person paused it (a stop cancels the retry). */
+    inStartRetryBackoff(name) {
+        if (this.shuttingDown || this.lifecycle.isPaused(name))
+            return false;
+        return this.startupRetries.has(name) || this.startupRetriesRunning.has(name);
+    }
     /** Pending automatic retry, if any (status display / tests). */
     pendingStartupRetry(name) {
         const pending = this.startupRetries.get(name);
@@ -5294,6 +5454,8 @@ export class FleetManager {
         this.startupRetries.delete(name);
         if (this.shuttingDown)
             return;
+        if (this.sessionOomStopped(name))
+            return;
         if (!this.isConfiguredInstance(name))
             return; // removed from fleet.yaml / classic channel meanwhile
         if (this.lifecycle.isPaused(name) || this.daemons.has(name))
@@ -5309,6 +5471,7 @@ export class FleetManager {
         }
         const topicMode = this.fleetConfig?.channel?.mode === "topic"
             || !!this.fleetConfig?.channels?.some(channel => channel.mode === "topic");
+        this.startupRetriesRunning.add(name);
         try {
             await this.spawnGate.run({
                 instanceName: name,
@@ -5330,6 +5493,9 @@ export class FleetManager {
             this.logger.error({ err, name, attempt: attempt + 1 }, "Automatic startup retry failed");
             this.scheduleStartupRetry(name, attempt + 1);
         }
+        finally {
+            this.startupRetriesRunning.delete(name);
+        }
     }
     /** Fleet-topic (fleet.yaml) or ClassicBot (classic channel) — both are retryable; anything else is gone. */
     isConfiguredInstance(name) {
@@ -5655,12 +5821,29 @@ export class FleetManager {
     }
     /** Restart a single instance, reloading fleet.yaml first to pick up config changes. */
     async restartSingleInstance(name, opts) {
+        if (opts?.recoveryCurrent && !opts.recoveryCurrent())
+            throw new Error(t("exit.expired", name));
+        const retry = opts?.explicit ? opts.sessionOomRetry ?? captureSessionOomRetry(this.getInstanceDir(name)) : undefined;
+        if (opts?.sessionOomRetry) {
+            const current = captureSessionOomRetry(this.getInstanceDir(name));
+            // An unclear-cause stop intentionally has no tombstone. Only its live
+            // launch fence can authorize an absent-marker receipt; positive stops
+            // still require the offered durable revision.
+            if ((!retry?.revision && !opts.recoveryCurrent) || current.revision !== retry?.revision || current.legacyRevision !== retry?.legacyRevision)
+                throw new SessionOomStoppedError(true);
+        }
+        if (!opts?.explicit && this.sessionOomStopped(name))
+            throw new SessionOomStoppedError(false, readSessionOomStop(this.getInstanceDir(name)));
         // One restart at a time per instance. Multiple sources can ask concurrently
         // (MCP revival, /restart, pty_error, model failover); a second stop/start
         // interleaved with the first tears down the window the first just created.
         // Later callers join the in-flight restart instead — its opts win.
         const inFlight = this.restartsInFlight.get(name);
         if (inFlight) {
+            // Do not let a late explicit caller clear a stop belonging to the
+            // automatic flight it would join. That action must be retried afterwards.
+            if (retry && this.sessionOomStopped(name))
+                throw new SessionOomStoppedError(true);
             this.logger.info({ name }, "restartSingleInstance: joining the restart already in flight");
             // #1753: a credential-profile switch joining it still needs its conversation carried in that restart's stop →
             // start gap: handed to the restart while it has not reached the gap (and carries nothing else); otherwise not
@@ -5687,7 +5870,7 @@ export class FleetManager {
             workingDirectory,
             reason: "restart",
             stage: "lifecycle",
-        }, () => this.doRestartSingleInstance(name, opts, transition)))
+        }, () => this.doRestartSingleInstance(name, opts, transition, retry)))
             .finally(() => {
             this.restartsInFlight.delete(name);
             if (this.restartGaps.get(name) === gap)
@@ -5728,7 +5911,7 @@ export class FleetManager {
             this.lifecycle.clearActivitySeedNow(name);
         }
     }
-    async doRestartSingleInstance(name, opts, transition) {
+    async doRestartSingleInstance(name, opts, transition, retry) {
         if (this.configPath) {
             this.loadConfig(this.configPath);
             this.routing.rebuild(this.fleetConfig);
@@ -5746,7 +5929,7 @@ export class FleetManager {
         if (!wasPaused)
             this.restartsHoldingSlot.add(name);
         try {
-            await this.doRestartWithSlot(name, opts, transition, config, wasPaused);
+            await this.doRestartWithSlot(name, opts, transition, config, wasPaused, retry);
             // An explicit restart of a paused instance is an operator wake (§1.6):
             // its success lifts the coordinator's park/backoff (the level stays).
             // Automatic restarts never do.
@@ -5758,7 +5941,7 @@ export class FleetManager {
             this.releaseWakeSlot(name, slot);
         }
     }
-    async doRestartWithSlot(name, opts, transition, config, wasPaused) {
+    async doRestartWithSlot(name, opts, transition, config, wasPaused, retry) {
         const pausedAt = wasPaused ? (this.lifecycle.getLastPausedAt(name) ?? readPausedAt(this.getInstanceDir(name))) : null;
         const pauseReason = wasPaused ? readPauseReason(this.getInstanceDir(name)) : null;
         // #1753: the directory the CLI being stopped was launched in — its conversation is under that directory's key.
@@ -5766,6 +5949,12 @@ export class FleetManager {
         // Between the stop and the start: the old CLI's transcript is complete, the new one has not looked yet.
         /** `launching`: the login the start will use — read only when a switch is to be carried. */
         const betweenStopAndStart = async (launching) => {
+            // Clearing is synchronous, after the old daemon is stopped and inside
+            // this action's serialized transition; an unseen later stop is retained.
+            if (retry)
+                clearSessionOomStopForRetry(this.getInstanceDir(name), retry);
+            else
+                assertNoSessionOomStop(this.getInstanceDir(name));
             // #1753: this restart's own carry, or one a caller joined it with before this point (restartSingleInstance).
             const gap = this.restartGaps.get(name);
             const carry = gap?.carry ?? opts?.carry;
@@ -9702,7 +9891,7 @@ export class FleetManager {
                             ...generalReactions.meta,
                             ...extraMeta,
                         },
-                    }, msg.humanOrigin, msg.humanOriginalBody ?? msg.text);
+                    }, msg.humanOrigin, msg.humanOriginalBody ?? msg.text, this.startRetryHoldFor(generalInstance, msg));
                     generalReactions.consume();
                     this.afterUserInboundDelivered(generalInstance, msg);
                     this.logger.info(`${msg.username} → ${generalInstance}: ${(text ?? "").slice(0, 100)}`);
@@ -9798,7 +9987,7 @@ export class FleetManager {
                     ...reactions.meta,
                     ...extraMeta,
                 },
-            }, msg.humanOrigin, msg.humanOriginalBody ?? msg.text);
+            }, msg.humanOrigin, msg.humanOriginalBody ?? msg.text, this.startRetryHoldFor(instanceName, msg));
             // Only after the message actually went out. A failed delivery keeps the
             // reactions queued for the retry / the next message.
             reactions.consume();
@@ -15021,14 +15210,42 @@ export class FleetManager {
     /** A clean exit is intentional from the CLI's perspective, but often not from
      * the operator's. Keep the instance notice passive and put the action in the
      * same-world General topic where an administrator can make the choice. */
-    async notifyNormalExit(instanceName) {
-        this.notifyInstanceTopic(instanceName, t("exit.instance_notice", instanceName) + placeSuffix({ kind: "general" }, "General"));
+    async notifySessionOomStop(instanceName, unclear = false) {
+        await this.notifyNormalExit(instanceName, unclear ? "unclear" : true);
+    }
+    async notifyNormalExit(instanceName, oom = false) {
+        const daemon = oom === "unclear" ? this.daemons.get(instanceName) : undefined;
+        const owner = daemon ? { ...daemon.getInteractionSnapshot().owner } : undefined;
+        const current = daemon && owner ? () => this.daemons.get(instanceName) === daemon
+            && daemon.isUnclearOomStopped() && sameInteractionOwner(owner, daemon.getInteractionSnapshot().owner) : undefined;
+        if (oom === "unclear" && !current?.())
+            return;
+        const state = oom ? readSessionOomStop(this.getInstanceDir(instanceName)) : null;
+        if (oom === "unclear" && state?.blocked)
+            return;
+        const proof = state?.blocked ? state.record?.evidence ?? state.evidence : undefined;
+        if (oom === true && !proof)
+            return;
+        const sessionOomRetry = oom ? captureSessionOomRetry(this.getInstanceDir(instanceName)) : undefined;
+        if (oom && [...this.pendingNonceButtons.values()].some(p => p.prefix === OOM_RETRY_CALLBACK_PREFIX
+            && p.instanceName === instanceName && p.sessionOomRetry?.revision === sessionOomRetry?.revision
+            && (oom !== "unclear" || p.confirmationCurrent?.() === true)))
+            return;
+        const prefix = oom ? OOM_RETRY_CALLBACK_PREFIX : EXIT_RESTART_CALLBACK_PREFIX;
+        const message = oom === "unclear" ? t("inst.oom_unclear_choice", instanceName) : oom && proof
+            ? t("inst.session_oom_choice", instanceName, describeSelfOom(proof))
+            : t("exit.general_notice", placeRef({ kind: "instance", name: instanceName }, instanceName));
+        const choices = oom
+            ? [{ action: "resume", label: t("inst.session_oom_resume") }, { action: "fresh", label: t("inst.session_oom_fresh") }]
+            : [{ action: "restart", label: t("exit.restart") }, { action: "ignore", label: t("exit.ignore") }];
+        const extra = sessionOomRetry ? { sessionOomRetry, ...(current ? { confirmationCurrent: current } : {}) } : undefined;
+        this.notifyInstanceTopic(instanceName, (oom ? message : t("exit.instance_notice", instanceName)) + placeSuffix({ kind: "general" }, "General"));
         const web = this.webOnlyPromptPlace();
         if (web) {
             await this.postNonceButtonPrompt({
-                prefix: EXIT_RESTART_CALLBACK_PREFIX, alertType: "exit_restart", instanceName, ...web,
-                message: t("exit.general_notice", placeRef({ kind: "instance", name: instanceName }, instanceName)),
-                choices: [{ action: "restart", label: t("exit.restart") }, { action: "ignore", label: t("exit.ignore") }],
+                prefix, alertType: "exit_restart", instanceName, ...web,
+                message, extra,
+                choices,
                 expiredText: t("exit.expired", instanceName),
             });
             return;
@@ -15049,37 +15266,36 @@ export class FleetManager {
             return;
         }
         await this.postNonceButtonPrompt({
-            prefix: EXIT_RESTART_CALLBACK_PREFIX,
+            prefix,
             alertType: "exit_restart",
             instanceName,
             adapter,
             adapterId,
             chatId,
             threadId,
-            message: t("exit.general_notice", placeRef({ kind: "instance", name: instanceName }, instanceName)),
-            choices: [
-                { action: "restart", label: t("exit.restart") },
-                { action: "ignore", label: t("exit.ignore") },
-            ],
+            message, extra,
+            choices,
             expiredText: t("exit.expired", instanceName),
         });
     }
     /** Consume a clean-exit Restart/Ignore button exactly once. */
     async handleExitRestartPrompt(data, callbackAdapterId, receivingAdapter) {
-        const claimed = this.consumeNonceCallback(EXIT_RESTART_CALLBACK_PREFIX, /^exit-restart:([0-9a-f]+):(restart|ignore)$/, data, callbackAdapterId, receivingAdapter);
+        const oom = data.callbackData.startsWith(OOM_RETRY_CALLBACK_PREFIX);
+        const claimed = this.consumeNonceCallback(oom ? OOM_RETRY_CALLBACK_PREFIX : EXIT_RESTART_CALLBACK_PREFIX, oom ? /^oom-retry:([0-9a-f]+):(resume|fresh)$/ : /^exit-restart:([0-9a-f]+):(restart|ignore)$/, data, callbackAdapterId, receivingAdapter);
         if (claimed === null)
             return false;
         if (claimed === "consumed")
             return true;
         const { entry: pending, action } = claimed;
-        this.eventLog?.insert(pending.instanceName, "normal_exit_action", { action, userId: data.userId });
+        this.eventLog?.insert(pending.instanceName, oom ? "session_oom_action" : "normal_exit_action", { action, userId: data.userId });
         if (action === "ignore") {
             await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, t("exit.ignored", pending.instanceName));
             return true;
         }
         await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, t("exit.restarting", pending.instanceName));
         try {
-            await this.restartSingleInstance(pending.instanceName);
+            await this.restartSingleInstance(pending.instanceName, { explicit: true, ...(oom ? { freshStart: action === "fresh", sessionOomRetry: pending.sessionOomRetry,
+                    ...(pending.confirmationCurrent ? { recoveryCurrent: pending.confirmationCurrent } : {}) } : {}) });
             await pending.adapter.editMessage(pending.chatId, pending.messageId ?? data.messageId, t("exit.restarted", pending.instanceName), pending.threadId);
         }
         catch (err) {
@@ -16910,6 +17126,10 @@ export class FleetManager {
             log: (level, message, extra) => this.logger[level](extra ?? {}, message),
         });
         db.board.onChange(({ rev, ids, projects, wakes }) => {
+            // Refresh both old and new owners on reassignment; wakes are not an ownership index.
+            if (ids.length)
+                for (const name of this.daemons.keys())
+                    this.observeWorkingState(name, "tasks");
             this.emitSseEvent("tasks", { rev, ids, projects });
             waker.note(wakes);
             // step 3: cards waiting for the person are Needs you items; #2355 review: a project's lead routes them too.
@@ -17008,7 +17228,7 @@ export class FleetManager {
         }
         await this.retireNonceButtons(pending, pending.messageId ?? data.messageId, t("hang.restarting", pending.instanceName));
         try {
-            await this.restartSingleInstance(pending.instanceName);
+            await this.restartSingleInstance(pending.instanceName, { explicit: true });
             await pending.adapter.editMessage(pending.chatId, pending.messageId ?? data.messageId, t("hang.restarted", pending.instanceName), pending.threadId);
         }
         catch (err) {
@@ -17780,7 +18000,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
                 content: fullText,
                 targetSession: instanceName,
                 meta,
-            }, msg.humanOrigin, msg.humanOriginalBody ?? text);
+            }, msg.humanOrigin, msg.humanOriginalBody ?? text, this.startRetryHoldFor(instanceName));
             reactions.consume();
         }
         catch (err) {
@@ -19738,7 +19958,12 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
         // also sees the channel's messages.
         if (adapterId)
             this.bindInstanceAdapter(instanceName, adapterId);
-        await this.startClassicInstance(instanceName, classicChannels.getBackend(channelId, adapterId, this.fleetConfig?.defaults?.backend), classicChannels.getPreTaskCommand(channelId, adapterId), classicChannels.getModel(channelId, adapterId, this.fleetConfig?.defaults?.model), classicChannels.getAutoPauseAfter(channelId, adapterId, this.fleetConfig?.defaults?.auto_pause_after));
+        if (this.sessionOomStopped(instanceName)) {
+            await this.restartSingleInstance(instanceName, { explicit: true });
+        }
+        else {
+            await this.startClassicInstance(instanceName, classicChannels.getBackend(channelId, adapterId, this.fleetConfig?.defaults?.backend), classicChannels.getPreTaskCommand(channelId, adapterId), classicChannels.getModel(channelId, adapterId, this.fleetConfig?.defaults?.model), classicChannels.getAutoPauseAfter(channelId, adapterId, this.fleetConfig?.defaults?.auto_pause_after));
+        }
         this.reregisterClassicChannels();
         // Auto-enable collab for Discord classic channels (TG uses @mention directly without collab mode)
         if (guildId && !classicChannels.isCollab(channelId, adapterId)) {