@songsid/agend 2.3.0 → 2.3.1

package/dist/daemon.js

@@ -10,7 +10,8 @@ import { createHash, randomBytes, randomUUID } from "node:crypto";
 import { EventEmitter } from "node:events";
 import { rotateLogIfNeededAsync } from "./logger.js";
 import { coredumpFilterLaunchPrefix } from "./coredump-filter.js";
-import { observeCliOomIdentity, confirmedCliOom, consecutiveSessionOoms } from "./session-oom.js";
+import { observeCliOomIdentity, confirmedCliOom, consecutiveSessionOoms, describeSelfOom } from "./session-oom.js";
+import { assertNoSessionOomStop, readSessionOomStop, writeSessionOomStop, SessionOomStoppedError, sessionOomStopHint } from "./session-oom-stop.js";
 import { mcpServerState } from "./mcp-liveness.js";
 import { clearPausedMarker, writePausedMarker } from "./pause-marker.js";
 import { writeBackgroundWorkEndedMarker } from "./background-work-ended.js";
@@ -58,6 +59,7 @@ import { buildResumeContinuation, clearInFlightTurnMarker, consumeInFlightTurnMa
 import { codexLiveRowMatches } from "./backend/codex.js";
 import { codexPackageProblemMessage } from "./backend/codex-package.js";
 import { claudeConfigDirFor, claudeProfileConfigDir, claudeSwitchDialog, claudeSwitchTargetMatches } from "./backend/claude-code.js";
+import { MuseBackend } from "./backend/muse.js";
 import { InteractionObservation, INTERACTION_CONFIRM_MS, sameInteractionOwner } from "./interaction-observation.js";
 import { unrecognisedPromptShape } from "./unrecognised-prompt.js";
 /** #2140: deliveries that may carry an unwritten what's-new notice before it is recorded anyway. */
@@ -1163,6 +1165,25 @@ export class Daemon extends EventEmitter {
     crashCount = 0;
     cliOomObservation = null;
     consecutiveOoms = null;
+    collateralOomRecovery = false;
+    unclearOomStopped = false;
+    sessionOomStopUnpersisted = false;
+    assertSessionOomAdmission() {
+        if (this.sessionOomStopUnpersisted)
+            throw new SessionOomStoppedError(false, { blocked: true, record: null, revision: null, confirmedOom: true });
+        assertNoSessionOomStop(this.instanceDir);
+    }
+    isSessionOomStopped() {
+        return this.sessionOomStopUnpersisted || readSessionOomStop(this.instanceDir).blocked;
+    }
+    isUnclearOomStopped() {
+        return this.unclearOomStopped && this.healthCheckPaused && this.processStatus === "crashed";
+    }
+    stopUnclearOom(exitCode) {
+        this.unclearOomStopped = true;
+        this.healthCheckPaused = true;
+        this.emitSupervisionEnded(t("inst.oom_unclear_stopped"), t("inst.session_oom_remedy"), exitCode, "oom_unclear");
+    }
     oomSessionKey() {
         // Claude's per-instance statusline owns its ID. Other current getters
         // either return null or select by cwd (Muse/Grok): those are not proof
@@ -1810,6 +1831,12 @@ export class Daemon extends EventEmitter {
             model: config.model ?? "default",
         };
         this.logger = rootLogger.child({ instance: name }, { level: config.log_level });
+        if (backend instanceof MuseBackend) {
+            const log = this.logger;
+            backend.setPlaceholderDebug((rec) => {
+                log.debug(rec, "muse empty-composer placeholder classified as empty");
+            });
+        }
         this.tmuxSessionName = getTmuxSession();
         this.messageBus = new MessageBus();
         this.messageBus.setLogger(this.logger);
@@ -1822,7 +1849,10 @@ export class Daemon extends EventEmitter {
         // finished turn can never resume after a later restart. reset() is
         // deliberately NOT hooked: stop/spawn abandon the in-memory turn while
         // the on-disk interruption must survive for the resume gate.
-        this.turnReplyGuard.onComplete = () => clearInFlightTurnMarker(this.instanceDir);
+        this.turnReplyGuard.onComplete = () => {
+            clearInFlightTurnMarker(this.instanceDir);
+            this.publishWorkingState("turn_closed");
+        };
     }
     /** FleetManager and this daemon share a process, so durable begin/ACK is a direct synchronous store call. */
     setDeliveryOutboxPort(port) {
@@ -2117,6 +2147,11 @@ export class Daemon extends EventEmitter {
      * daemon's own state only; it never inspects the pane itself.
      */
     notAcceptingReason() {
+        if (this.sessionOomStopUnpersisted)
+            return t("inst.session_oom_hold");
+        const oomStop = readSessionOomStop(this.instanceDir);
+        if (oomStop.blocked)
+            return sessionOomStopHint(oomStop);
         if (this.pauseWakeState !== "active")
             return t("delivery.hold_state", this.pauseWakeState);
         if (this.inputBlockedDialogKey !== null)
@@ -2167,6 +2202,7 @@ export class Daemon extends EventEmitter {
         }
     }
     async startAdmitted() {
+        this.assertSessionOomAdmission();
         ensureInstanceDir(this.instanceDir);
         writeFileSync(join(this.instanceDir, "daemon.pid"), String(process.pid));
         this.logger.info(`Starting ${this.name}`);
@@ -3094,19 +3130,36 @@ export class Daemon extends EventEmitter {
                         return;
                     this.cliOomObservation = null;
                     const decisionKey = this.oomSessionKey();
-                    this.consecutiveOoms = consecutiveSessionOoms(this.consecutiveOoms, decisionKey, decisionKey === key && oom !== null);
+                    this.consecutiveOoms = consecutiveSessionOoms(this.consecutiveOoms, decisionKey, decisionKey === key && oom?.attribution === "self");
+                    const collateralOom = oom?.attribution === "collateral";
+                    if (collateralOom)
+                        this.logger.warn({ attribution: oom.attribution, pid: oom.pid }, "OOM victim without self-caused proof — preserving session; waiting for memory recovery");
                     if (this.consecutiveOoms && this.consecutiveOoms.count >= 2) {
                         // Stop supervision, not an idle pause: queued work must not auto-wake
                         // the same session. An explicit restart is the person's retry choice.
+                        // No await after the final captured health owner/session check. The
+                        // stop is authoritative before observers can see supervision end.
+                        try {
+                            writeSessionOomStop(this.instanceDir, { owner: healthOwner, sessionKey: decisionKey,
+                                configuration: this.getWorkingStateObservation().configuration, stoppedAt: Date.now(),
+                                count: this.consecutiveOoms.count, victim: { pid: oom.pid, at: oom.at }, evidence: oom.evidence });
+                        }
+                        catch {
+                            // Storage failure cannot revive this CLI in the current process.
+                            // Say explicitly that a fleet restart cannot yet rely on durability.
+                            this.sessionOomStopUnpersisted = true;
+                            this.logger.error("Could not persist session OOM stop; fix storage before restarting the fleet");
+                        }
                         this.healthCheckPaused = true;
                         try {
-                            this.appendCrashHistory({ exitCode, lastOutput, crashType, reason: "session_oom" });
+                            this.appendCrashHistory({ exitCode, lastOutput, crashType, reason: "session_oom", oomEvidence: oom.evidence });
                         }
                         catch (error) {
                             this.logger.warn({ error }, "Could not append session OOM history");
                         }
                         this.logger.error({ pid: oom.pid, count: this.consecutiveOoms.count }, "Repeated confirmed CLI OOM — automatic resume stopped; session preserved");
-                        this.emitSupervisionEnded(t("inst.session_oom_stopped"), t("inst.session_oom_remedy"), exitCode, "session_oom");
+                        this.emitSupervisionEnded(`${t("inst.session_oom_stopped")} (${describeSelfOom(oom.evidence)})`, t("inst.session_oom_remedy")
+                            + (this.sessionOomStopUnpersisted ? ` ${t("inst.session_oom_save_failed")}` : ""), exitCode, "session_oom");
                         return;
                     }
                     // Kill the dead window (remain-on-exit keeps it around) before respawn
@@ -3116,7 +3169,7 @@ export class Daemon extends EventEmitter {
                             return;
                     }
                     // Detect claude-code background session conflict — recover without counting as crash
-                    if (lastOutput && (lastOutput.includes("background agent") || lastOutput.includes("Session is currently running"))) {
+                    if (!oom && lastOutput && (lastOutput.includes("background agent") || lastOutput.includes("Session is currently running"))) {
                         if (!this.backgroundSessionRecoveryAttempted) {
                             this.backgroundSessionRecoveryAttempted = true;
                             this.logger.warn("Detected lingering background agent session — starting fresh (no resume)");
@@ -3156,7 +3209,7 @@ export class Daemon extends EventEmitter {
                     // --continue again and crash in the same way → loop. Clear the session id
                     // and skip resume so the next spawn starts fresh. (skipResume also stops
                     // saveSessionId below from resurrecting the id from statusline.json.)
-                    if (this.paneSaysNoConversation(lastOutput)) {
+                    if (!oom && this.paneSaysNoConversation(lastOutput)) {
                         this.setSessionAside("cli_reported_no_conversation");
                         this.skipResume = true;
                     }
@@ -3177,12 +3230,21 @@ export class Daemon extends EventEmitter {
                         return;
                     }
                     // Detect rapid crash: sliding window — 3+ crashes in 5 minutes
-                    this.crashTimestamps.push(Date.now());
+                    if (!collateralOom)
+                        this.crashTimestamps.push(Date.now());
+                    else
+                        this.crashTimestamps = [];
                     const crashWindowMs = 5 * 60_000;
                     this.crashTimestamps = this.crashTimestamps.filter(t => t > Date.now() - crashWindowMs);
                     if (this.crashTimestamps.length >= 3) {
                         this.healthCheckPaused = true;
                         this.logger.error({ crashesInWindow: this.crashTimestamps.length, lastExitCode: exitCode ?? null, lastOutput: paneTail(lastOutput) ?? null }, "3+ crashes in 5 minutes — pausing respawn");
+                        // Unknown cause is bounded like an ordinary crash, never a
+                        // self-OOM verdict. Preserve the session for the person's choice.
+                        if (oom?.attribution === "unknown") {
+                            this.stopUnclearOom(exitCode);
+                            return;
+                        }
                         // P1: Persist crash state so next process restart skips resume
                         try {
                             writeFileSync(join(this.instanceDir, "crash-state.json"), JSON.stringify({
@@ -3199,18 +3261,23 @@ export class Daemon extends EventEmitter {
                     if (reset_after > 0 && Date.now() - this.lastCrashAt > reset_after) {
                         this.crashCount = 0;
                     }
-                    this.crashCount++;
+                    if (!collateralOom)
+                        this.crashCount++;
                     this.lastCrashAt = Date.now();
-                    if (this.crashCount > max_retries) {
+                    if (!collateralOom && this.crashCount > max_retries) {
                         this.logger.error({ crashCount: this.crashCount, maxRetries: max_retries, lastExitCode: exitCode ?? null, lastOutput: paneTail(lastOutput) ?? null }, "Max crash retries exceeded — not respawning");
                         this.healthCheckPaused = true;
+                        if (oom?.attribution === "unknown") {
+                            this.stopUnclearOom(exitCode);
+                            return;
+                        }
                         this.emitSupervisionEnded(`it crashed ${this.crashCount} times, exceeding restart_policy.max_retries (${max_retries})`, "Check the logs for the cause, then restart it.");
                         return; // don't schedule next — given up
                     }
                     // Calculate backoff delay
                     const delay = backoff === "exponential"
-                        ? Math.min(1000 * Math.pow(2, this.crashCount - 1), 60_000)
-                        : 1000 * this.crashCount;
+                        ? Math.min(1000 * Math.pow(2, Math.max(1, this.crashCount) - 1), 60_000)
+                        : 1000 * Math.max(1, this.crashCount);
                     this.logger.warn({ crashCount: this.crashCount, delay }, `${cliLabel} window died — respawning after backoff`);
                     await new Promise(r => setTimeout(r, delay));
                     try {
@@ -3255,7 +3322,14 @@ export class Daemon extends EventEmitter {
                         // Write snapshot before spawn — consumed only if resume fails
                         this.writeRotationSnapshot("crash");
                         // Try --resume first; spawnClaudeWindow falls back to fresh session if resume fails
-                        const resumed = await this.spawnClaudeWindow();
+                        this.collateralOomRecovery = collateralOom;
+                        let resumed;
+                        try {
+                            resumed = await this.spawnClaudeWindow();
+                        }
+                        finally {
+                            this.collateralOomRecovery = false;
+                        }
                         if (!resumed) {
                             // Resume failed → fresh session → inject snapshot for context
                             await this.injectSnapshotMessage();
@@ -3349,6 +3423,7 @@ export class Daemon extends EventEmitter {
      * four, carrying a human-readable cause and the operator's next step.
      */
     emitSupervisionEnded(reason, remedy, exitCode, cause) {
+        this.publishWorkingState("supervision_ended");
         this.emit("supervision_ended", {
             name: this.name,
             reason,
@@ -3361,6 +3436,7 @@ export class Daemon extends EventEmitter {
         if (this.processStatus === status)
             return;
         this.processStatus = status;
+        this.publishWorkingState("process");
         if (status === "running") {
             // A successful crash respawn is a new pane generation. It is ready by
             // this point, so an error gate inherited from the dead pane must not keep
@@ -3909,6 +3985,7 @@ export class Daemon extends EventEmitter {
         this.deliveryEpoch = fleetEpoch === undefined
             ? this.deliveryEpoch + 1
             : Math.max(this.deliveryEpoch, fleetEpoch);
+        this.publishWorkingState("cancelled");
         this.logger.info({ deliveryEpoch: this.deliveryEpoch }, "Pending delivery queue cleared by user cancel");
     }
     /** Board 907c7899: the layout a web /compact's end can be read from on this pane (kiro classic or TUI V2), or null. */
@@ -4477,6 +4554,7 @@ export class Daemon extends EventEmitter {
         return true;
     }
     async stop() {
+        this.publishWorkingState("before_stop");
         this.logger.info("Stopping daemon instance");
         this.clearQuitRelaunchWatch();
         this.fenceDeliveryWritesForStop();
@@ -4625,6 +4703,31 @@ export class Daemon extends EventEmitter {
     getInteractionSnapshot() {
         return this.interactionObservation.snapshot(this.interactionOwner());
     }
+    workingTurnDelivery = null;
+    workingBackgroundObservation = null;
+    /** Pure metadata read: never discovers sessions, captures a pane or reads backend files. */
+    getWorkingStateObservation() {
+        const turn = this.turnReplyGuard.snapshot();
+        const backend = this.config.backend ?? this.backend?.binaryName ?? "unknown";
+        const profile = resolveCredentialProfile(this.config.backend_options?.[backend]);
+        return {
+            owner: this.interactionOwner(), backend,
+            configuration: createHash("sha256").update(JSON.stringify([backend, this.config.working_directory, profile])).digest("hex"),
+            executionState: this.processStatus === "running" ? this.isPaused ? "paused" : this.instanceState : null,
+            processStatus: this.getProcessStatus(), sessionId: null, backendTurnId: null,
+            backgroundWork: this.workingBackgroundObservation,
+            turn: turn ? { generation: turn.generation,
+                deliveryId: this.workingTurnDelivery?.generation === turn.generation ? this.workingTurnDelivery.deliveryId : null,
+                correlationId: turn.target.correlationId ?? null, messageId: turn.target.messageId ?? null,
+                cancelled: turn.cancelledByUser, completionDelivered: turn.completionDelivered } : null,
+        };
+    }
+    publishWorkingState(reason) {
+        try {
+            this.emit("working_state", { reason, observation: this.getWorkingStateObservation() });
+        }
+        catch { /* Journal observers cannot change daemon behavior. */ }
+    }
     /** Key of the last interaction snapshot actually emitted + broadcast (#1219). */
     lastPublishedInteractionKey = null;
     publishInteraction() {
@@ -4877,6 +4980,7 @@ export class Daemon extends EventEmitter {
                 this.pauseWakeState = "paused";
                 this.autoPauseController.markPaused();
                 writePausedMarker(this.instanceDir, this.lastPausedAt ?? Date.now(), reason);
+                this.publishWorkingState("execution");
                 this.logger.info({ pausedAt: this.lastPausedAt }, "Instance auto-paused");
                 this.ipcServer?.broadcast({
                     type: "instance_state", instanceName: this.name, state: "paused", pausedAt: this.lastPausedAt,
@@ -4902,6 +5006,7 @@ export class Daemon extends EventEmitter {
     }
     /** Respawn the CLI in the preserved window and block until its prompt is ready. */
     async wake(timeoutMs = 30_000) {
+        this.assertSessionOomAdmission();
         // An explicit wake (e.g. after the user re-logs in) cancels a deferred
         // auth pause — otherwise the instance would pause again the moment it idles.
         this.pausePending = false;
@@ -4969,6 +5074,7 @@ export class Daemon extends EventEmitter {
                 this.clearErrorRecoveryGate();
                 clearPausedMarker(this.instanceDir);
                 this.resumeRuntimeMonitors();
+                this.publishWorkingState("execution");
                 this.logger.info("Instance auto-woke");
                 this.ipcServer?.broadcast({
                     type: "instance_state", instanceName: this.name, state: this.instanceState, pausedAt: null,
@@ -5070,6 +5176,17 @@ export class Daemon extends EventEmitter {
         }
         this.instanceState = snapshot.state;
         this.acceptedInstanceSnapshot = { ...snapshot };
+        // Reuse an already-captured pane and the backend's pure parser, never capture for the journal.
+        if (typeof pane === "string" && this.backend?.backgroundWorkRunning) {
+            try {
+                const background = this.backend.backgroundWorkRunning(pane);
+                if (background !== this.workingBackgroundObservation) {
+                    this.workingBackgroundObservation = background;
+                    this.publishWorkingState("background");
+                }
+            }
+            catch { /* Observational parsing cannot change accepted execution state. */ }
+        }
         // OpenCode creates its session lazily on the first submitted message.
         // Waiting until stop/pause to persist that id loses resume state when the
         // fleet is SIGKILLed or the host reboots.
@@ -5114,6 +5231,7 @@ export class Daemon extends EventEmitter {
         }
         if (snapshot.state !== previous) {
             this.logStateEdge(previous, snapshot);
+            this.publishWorkingState("execution");
             this.emit("instance_state", { name: this.name, ...snapshot });
             this.ipcServer?.broadcast({ type: "instance_state", instanceName: this.name, ...snapshot });
             if (snapshot.state === "stuck" && pane && this.instanceStateReadyPattern) {
@@ -5196,6 +5314,8 @@ export class Daemon extends EventEmitter {
             deliveredText,
             deliveredAt,
         }, this.pendingWork.lastInboundTimestamp());
+        this.workingTurnDelivery = { generation: this.turnReplyGuard.snapshot().generation, deliveryId: meta.delivery_id || null };
+        this.publishWorkingState("turn_started");
         // #1209: persist the in-flight turn with a seam checkpoint. A marker
         // that survives a restart is an interrupted turn by construction —
         // completion (via the guard's onComplete) and cancel paths delete it.
@@ -10095,8 +10215,10 @@ export class Daemon extends EventEmitter {
                 // purposes even before the idle edge completes the guard. Without
                 // this, a restart in the reply-to-idle window would re-inject a
                 // continuation for an already-answered turn.
-                if (replyAttempt?.completionAction)
+                if (replyAttempt?.completionAction) {
                     clearInFlightTurnMarker(this.instanceDir);
+                    this.publishWorkingState("completion_delivered");
+                }
                 // #1377: a confirmed completion action (reply/react/edit_message) ends
                 // the human-facing obligation immediately — the idle edge does not need
                 // to confirm it.  Clear any pending confirmation window so a later
@@ -10759,6 +10881,9 @@ export class Daemon extends EventEmitter {
      * into exactly the window this exists to close.
      */
     beginSpawn() {
+        this.unclearOomStopped = false;
+        this.publishWorkingState("before_spawn");
+        this.workingBackgroundObservation = null;
         this.dialogAnswers = null;
         this.capacityNudge = null;
         this.lastErrorEpisode = null;
@@ -10974,6 +11099,7 @@ export class Daemon extends EventEmitter {
         catch { /* best effort */ }
     }
     async spawnClaudeWindow() {
+        this.assertSessionOomAdmission();
         this.beginSpawn();
         let resumedSuccessfully = false;
         try {
@@ -11364,11 +11490,20 @@ export class Daemon extends EventEmitter {
      * Returns true if CLI is ready, false if it failed or got stuck.
      */
     async trySpawn(reuseWindow = false, startupTimeoutMs) {
-        const ready = !this.spawnGate ? await this.trySpawnInsideGate(reuseWindow, startupTimeoutMs) : await this.spawnGate.run({
+        const recoveryOwner = this.collateralOomRecovery ? this.interactionOwner() : null;
+        const enter = () => {
+            if (recoveryOwner && (!sameInteractionOwner(recoveryOwner, this.interactionOwner())
+                || this.startupAborted || this.deliveryWritesStopping || this.runtimeMonitorsFrozen || this.healthCheckPaused)) {
+                throw new Error("OOM recovery cancelled: the instance was stopped, paused or replaced while memory was unavailable");
+            }
+            return this.trySpawnInsideGate(reuseWindow, startupTimeoutMs);
+        };
+        const ready = !this.spawnGate ? await enter() : await this.spawnGate.run({
             instanceName: this.name,
             workingDirectory: this.config.working_directory,
             reason: reuseWindow ? "wake" : this.lastSpawnAt > 0 ? "recovery" : "startup",
-        }, () => this.trySpawnInsideGate(reuseWindow, startupTimeoutMs));
+            requireNormalMemory: this.collateralOomRecovery,
+        }, enter);
         // #906: a resumed kiro conversation comes back as the agent it was saved under. Every launch path that reaches a
         // ready CLI (start, recovery, wake) switches it to this instance's own agent here, while the spawn still holds
         // deliveries — and outside the spawn gate, which it does not need.
@@ -11650,6 +11785,7 @@ export class Daemon extends EventEmitter {
         }
     }
     async trySpawnInsideGate(reuseWindow = false, startupTimeoutMs) {
+        this.assertSessionOomAdmission();
         this.startupAdmission?.();
         const launchFence = this.launchFenceEpoch;
         const backendConfig = this.buildBackendConfig();
@@ -11698,6 +11834,7 @@ export class Daemon extends EventEmitter {
         // Thrown, not `false`: a false verdict reads as "the CLI failed to start" and sends the startup path
         // into its resume-retry / set-the-session-aside handling, for an instance somebody just stopped.
         this.startupAdmission?.();
+        this.assertSessionOomAdmission();
         if (launchFence !== this.launchFenceEpoch)
             throw new Error("Launch cancelled: the instance was stopped or paused while the launch was being prepared");
         this.backend.writeConfig(backendConfig);
@@ -11784,6 +11921,7 @@ export class Daemon extends EventEmitter {
         }
         this.startupAdmission?.();
         let windowId;
+        this.assertSessionOomAdmission();
         if (reuseWindow) {
             this.controlClient?.unregisterWindow(this.tmux.getWindowId());
             await this.tmux.respawnWindow(cmd, resolvedCwd);
@@ -12418,6 +12556,7 @@ export class Daemon extends EventEmitter {
                 crashType: data.crashType,
                 exitCode: data.exitCode,
                 reason: data.reason,
+                ...(data.oomEvidence ? { oomEvidence: data.oomEvidence } : {}),
                 lastOutput: data.lastOutput,
                 crashCount: this.crashCount + 1,
                 crashesInWindow: this.crashTimestamps.length,