@ouro.bot/cli 0.1.0-alpha.92 → 0.1.0-alpha.920

package/dist/heart/daemon/process-manager.js

@@ -33,16 +33,34 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.DaemonProcessManager = void 0;
+exports.DaemonProcessManager = exports.PRIVATE_RUNTIME_DISPATCH_ACK_TIMEOUT_MS = exports.RESPAWN_GUARD_WINDOW_MS = exports.RESPAWN_GUARD_MAX_RESTARTS = void 0;
 const child_process_1 = require("child_process");
+const crypto_1 = require("crypto");
 const path = __importStar(require("path"));
 const identity_1 = require("../identity");
 const runtime_1 = require("../../nerves/runtime");
 function startOfHour(ms) {
     return ms - 60 * 60 * 1000;
 }
+/**
+ * Respawn-loop guard: refuse `restartAgent` if we've already orchestrated
+ * RESPAWN_GUARD_MAX_RESTARTS in the past RESPAWN_GUARD_WINDOW_MS.
+ *
+ * Calibrated for the 2026-05-11 BB sense incident: a misconfigured probe
+ * was triggering `restartAgent` every ~60s for hours. Five restarts in
+ * 10 minutes is well above the rate of legitimate operational restarts
+ * (a single human-initiated `ouro down && ouro up` produces one) and well
+ * below the rate of a death spiral (60/hr ⇒ 10/10min).
+ */
+exports.RESPAWN_GUARD_MAX_RESTARTS = 5;
+exports.RESPAWN_GUARD_WINDOW_MS = 10 * 60_000;
+const MAX_PENDING_IPC_MESSAGES = 20;
+exports.PRIVATE_RUNTIME_DISPATCH_ACK_TIMEOUT_MS = 8 * 60_000;
 class DaemonProcessManager {
     agents = new Map();
+    /** `stopAll()` is terminal for a manager instance. Once set, it fences
+     *  detached restart/start continuations until the owning daemon exits. */
+    shutdownRequested = false;
     maxRestartsPerHour;
     stabilityThresholdMs;
     initialBackoffMs;
@@ -51,33 +69,130 @@ class DaemonProcessManager {
     now;
     setTimeoutFn;
     clearTimeoutFn;
+    cooldownRecoveryMs;
+    maxCooldownRetries;
+    startupStaleAfterMs;
+    stopTimeoutMs;
     existsSyncFn;
+    configCheckFn;
+    statusWriterFn;
+    onSnapshotChangeFn;
+    /**
+     * Notify the snapshot-change observer (if registered). Swallows any
+     * errors from the observer so process lifecycle code never fails
+     * because the observer threw.
+     */
+    notifySnapshotChange(snapshot) {
+        if (!this.onSnapshotChangeFn)
+            return;
+        try {
+            this.onSnapshotChangeFn(snapshot);
+        }
+        catch (error) {
+            (0, runtime_1.emitNervesEvent)({
+                level: "warn",
+                component: "daemon",
+                event: "daemon.snapshot_change_observer_error",
+                message: "snapshot-change observer threw",
+                meta: {
+                    agent: snapshot.name,
+                    error: error instanceof Error ? error.message : /* v8 ignore next -- defensive: non-Error catch branch @preserve */ String(error),
+                },
+            });
+        }
+    }
+    writeStatus(agent, text) {
+        try {
+            this.statusWriterFn(text);
+        }
+        catch (error) {
+            (0, runtime_1.emitNervesEvent)({
+                level: "warn",
+                component: "daemon",
+                event: "daemon.status_writer_error",
+                message: "daemon status writer threw",
+                meta: {
+                    agent,
+                    error: error instanceof Error ? error.message : String(error),
+                },
+            });
+        }
+    }
+    currentTimeMs() {
+        const value = this.now();
+        return Number.isFinite(value) ? value : Date.now();
+    }
+    isStartAttemptCurrent(state, attemptId) {
+        return state.startAttemptId === attemptId;
+    }
+    markConfigCheckFailed(state, errorReason, fixHint) {
+        const agent = state.config.name;
+        state.snapshot.status = "crashed";
+        state.snapshot.errorReason = errorReason;
+        state.snapshot.fixHint = fixHint;
+        (0, runtime_1.emitNervesEvent)({
+            level: "error",
+            component: "daemon",
+            event: "daemon.agent_config_invalid",
+            message: errorReason,
+            meta: { agent, fix: fixHint },
+        });
+        this.writeStatus(agent, `[daemon] ${agent}: ${errorReason}\n` +
+            (fixHint ? `  Fix: ${fixHint}\n` : ""));
+        this.notifySnapshotChange(state.snapshot);
+    }
     constructor(options) {
         this.maxRestartsPerHour = options.maxRestartsPerHour ?? 10;
         this.stabilityThresholdMs = options.stabilityThresholdMs ?? 60_000;
         this.initialBackoffMs = options.initialBackoffMs ?? 1_000;
         this.maxBackoffMs = options.maxBackoffMs ?? 60_000;
+        this.cooldownRecoveryMs = options.cooldownRecoveryMs ?? 5 * 60 * 1_000;
+        this.maxCooldownRetries = options.maxCooldownRetries ?? 3;
+        this.startupStaleAfterMs = options.startupStaleAfterMs ?? 45_000;
+        this.stopTimeoutMs = options.stopTimeoutMs ?? 4_000;
         this.spawnFn = options.spawn ?? ((command, args, spawnOptions) => (0, child_process_1.spawn)(command, args, spawnOptions));
         this.now = options.now ?? (() => Date.now());
         this.setTimeoutFn = options.setTimeoutFn ?? ((cb, delay) => setTimeout(cb, delay));
         this.clearTimeoutFn = options.clearTimeoutFn ?? ((timer) => clearTimeout(timer));
         this.existsSyncFn = options.existsSync ?? null;
+        this.configCheckFn = options.configCheck ?? null;
+        this.statusWriterFn = options.statusWriter ?? ((text) => {
+            process.stderr.write(text);
+        });
+        this.onSnapshotChangeFn = options.onSnapshotChange ?? null;
         for (const agent of options.agents) {
             this.agents.set(agent.name, {
                 config: agent,
                 process: null,
+                startInFlight: false,
+                startAttemptedAtMs: null,
+                startAttemptId: 0,
+                pendingIpcMessages: [],
+                pendingDispatches: new Map(),
                 restartTimer: null,
                 crashTimestamps: [],
+                orchestratedRestartTimestamps: [],
+                respawnLoopTripped: false,
                 stopRequested: false,
+                cooldownTimer: null,
+                cooldownRetryCount: 0,
+                fastCrashCount: 0,
+                stopFailureActive: false,
+                displacedReason: null,
                 snapshot: {
                     name: agent.name,
                     channel: agent.channel,
+                    autoStart: agent.autoStart,
                     status: "stopped",
                     pid: null,
                     restartCount: 0,
                     startedAt: null,
                     lastCrashAt: null,
                     backoffMs: this.initialBackoffMs,
+                    lastExitCode: null,
+                    lastSignal: null,
+                    errorReason: null,
+                    fixHint: null,
                 },
             });
         }
@@ -89,85 +204,508 @@ class DaemonProcessManager {
             }
         }
     }
-    async startAgent(agent) {
+    triggerAutoStartAgents() {
+        for (const state of this.agents.values()) {
+            if (!state.config.autoStart)
+                continue;
+            void this.startAgent(state.config.name).catch((error) => {
+                const errorReason = error instanceof Error ? error.message : String(error);
+                this.markConfigCheckFailed(state, `agent startup threw before the worker could run: ${errorReason}`, "Run 'ouro doctor' for diagnostics, then retry 'ouro up'.");
+            });
+        }
+    }
+    async startAgent(agent, options = {}) {
         const state = this.requireAgent(agent);
-        if (state.process)
+        if (this.shutdownRequested)
             return;
-        this.clearRestartTimer(state);
-        state.stopRequested = false;
-        state.snapshot.status = "starting";
-        const runCwd = (0, identity_1.getRepoRoot)();
-        const entryScript = path.join((0, identity_1.getRepoRoot)(), "dist", state.config.entry);
-        if (this.existsSyncFn && !this.existsSyncFn(entryScript)) {
-            state.snapshot.status = "crashed";
+        if (state.displacedReason !== null)
+            return;
+        if (state.process || state.startInFlight)
+            return;
+        const attemptId = state.startAttemptId + 1;
+        state.startAttemptId = attemptId;
+        state.startInFlight = true;
+        state.startAttemptedAtMs = this.currentTimeMs();
+        try {
+            this.clearRestartTimer(state);
+            state.stopRequested = false;
+            state.snapshot.status = "starting";
+            if (this.configCheckFn && options.skipConfigCheck !== true) {
+                let result;
+                try {
+                    result = await this.configCheckFn(agent);
+                }
+                catch (error) {
+                    const errorReason = error instanceof Error ? error.message : String(error);
+                    this.markConfigCheckFailed(state, `agent config validation threw: ${errorReason}`, "Run 'ouro doctor' for diagnostics, then retry 'ouro up'.");
+                    return;
+                }
+                if (!this.isStartAttemptCurrent(state, attemptId))
+                    return;
+                if (state.stopRequested) {
+                    state.snapshot.status = "stopped";
+                    state.snapshot.pid = null;
+                    this.notifySnapshotChange(state.snapshot);
+                    return;
+                }
+                if (result.skip) {
+                    state.snapshot.status = "stopped";
+                    state.snapshot.errorReason = null;
+                    state.snapshot.fixHint = null;
+                    (0, runtime_1.emitNervesEvent)({
+                        component: "daemon",
+                        event: "daemon.agent_config_skipped",
+                        message: result.error ?? "agent start skipped by config check",
+                        meta: { agent, fix: result.fix ?? null },
+                    });
+                    this.notifySnapshotChange(state.snapshot);
+                    return;
+                }
+                if (!result.ok) {
+                    this.markConfigCheckFailed(state, result.error ?? "agent config validation failed", result.fix ?? null);
+                    return;
+                }
+                /* v8 ignore next -- defensive duplicate stale-start guard before spawn preparation @preserve */
+                if (!this.isStartAttemptCurrent(state, attemptId))
+                    return;
+                // Config check passed — clear any prior error so the pulse stops
+                // reporting the broken state. This is the recovery path: the user
+                // fixed their secrets/config, the next startAgent attempt sees a
+                // valid config, and the pulse goes quiet.
+                state.snapshot.errorReason = null;
+                state.snapshot.fixHint = null;
+            }
+            if (options.skipConfigCheck === true) {
+                state.snapshot.errorReason = null;
+                state.snapshot.fixHint = null;
+            }
+            const runCwd = (0, identity_1.getRepoRoot)();
+            const entryScript = path.join((0, identity_1.getRepoRoot)(), "dist", state.config.entry);
+            if (this.existsSyncFn && !this.existsSyncFn(entryScript)) {
+                state.snapshot.status = "crashed";
+                (0, runtime_1.emitNervesEvent)({
+                    level: "error",
+                    component: "daemon",
+                    event: "daemon.agent_entry_missing",
+                    message: "agent entry script does not exist — cannot spawn. Run 'ouro daemon install' from the correct location.",
+                    meta: { agent, entryScript },
+                });
+                this.notifySnapshotChange(state.snapshot);
+                return;
+            }
+            /* v8 ignore next -- defensive duplicate stale-start guard immediately before spawn @preserve */
+            if (!this.isStartAttemptCurrent(state, attemptId))
+                return;
+            if (state.stopRequested) {
+                state.snapshot.status = "stopped";
+                state.snapshot.pid = null;
+                this.notifySnapshotChange(state.snapshot);
+                return;
+            }
+            const agentArgs = state.config.getArgs?.() ?? state.config.args ?? [];
+            const args = [entryScript, "--agent", state.config.agentArg ?? agent, ...agentArgs];
+            const child = this.spawnFn("node", args, {
+                cwd: runCwd,
+                env: state.config.env ? { ...process.env, ...state.config.env } : process.env,
+                stdio: ["ignore", "ignore", "ignore", "ipc"],
+            });
+            /* v8 ignore next 7 -- defensive: spawn should always return a ChildProcess @preserve */
+            if (!child) {
+                state.snapshot.status = "crashed";
+                (0, runtime_1.emitNervesEvent)({ level: "error", component: "daemon", event: "daemon.agent_spawn_failed", message: "spawn returned null", meta: { agent } });
+                return;
+            }
+            if (!this.isStartAttemptCurrent(state, attemptId)) {
+                try {
+                    child.kill("SIGTERM");
+                }
+                catch {
+                    (0, runtime_1.emitNervesEvent)({
+                        level: "warn",
+                        component: "daemon",
+                        event: "daemon.agent_stop_error",
+                        message: "failed to stop stale managed agent startup",
+                        meta: { agent },
+                    });
+                }
+                return;
+            }
+            state.process = child;
+            state.snapshot.status = "running";
+            state.snapshot.pid = child.pid ?? null;
+            state.snapshot.startedAt = new Date(this.currentTimeMs()).toISOString();
+            child.on("message", (message) => {
+                if (state.process !== child || !message || typeof message !== "object")
+                    return;
+                const result = message;
+                if (result.type !== "ouro.privateRuntimeDispatchResult" || typeof result.dispatchId !== "string")
+                    return;
+                const pending = state.pendingDispatches.get(result.dispatchId);
+                if (!pending || pending.child !== child)
+                    return;
+                state.pendingDispatches.delete(result.dispatchId);
+                this.clearTimeoutFn(pending.timer);
+                if (result.ok === true)
+                    pending.resolve();
+                else
+                    pending.reject(new Error(typeof result.error === "string" && result.error.trim() ? result.error.slice(0, 1_000) : "private-runtime dispatch failed"));
+            });
+            child.once("disconnect", () => {
+                if (state.process !== child)
+                    return;
+                this.rejectPendingDispatchesForChild(state, child, new Error(`Managed agent '${agent}' disconnected before acknowledged dispatch completed.`));
+            });
+            const bootstrap = state.config.getRuntimeCredentialBootstrap?.() ?? null;
+            if (bootstrap) {
+                const message = {
+                    type: "ouro.runtimeCredentialBootstrap",
+                    agentName: bootstrap.agentName,
+                };
+                if (bootstrap.runtimeConfig)
+                    message.runtimeConfig = bootstrap.runtimeConfig;
+                if (bootstrap.machineRuntimeConfig)
+                    message.machineRuntimeConfig = bootstrap.machineRuntimeConfig;
+                if (bootstrap.machineId)
+                    message.machineId = bootstrap.machineId;
+                if (bootstrap.providerCredentialRecords)
+                    message.providerCredentialRecords = bootstrap.providerCredentialRecords;
+                try {
+                    child.send?.(message);
+                    (0, runtime_1.emitNervesEvent)({
+                        component: "daemon",
+                        event: "daemon.agent_runtime_credentials_bootstrap_sent",
+                        message: "sent runtime credential bootstrap to managed agent process",
+                        meta: {
+                            agent,
+                            runtimeConfig: !!bootstrap.runtimeConfig,
+                            machineRuntimeConfig: !!bootstrap.machineRuntimeConfig,
+                            providerCredentialRecords: bootstrap.providerCredentialRecords?.length ?? 0,
+                        },
+                    });
+                }
+                catch (error) {
+                    (0, runtime_1.emitNervesEvent)({
+                        level: "warn",
+                        component: "daemon",
+                        event: "daemon.agent_ipc_send_error",
+                        message: "failed to send runtime credential bootstrap to managed agent process",
+                        meta: { agent, error: error instanceof Error ? error.message : String(error) },
+                    });
+                }
+            }
+            const pendingIpcMessages = state.pendingIpcMessages.splice(0);
+            for (const pendingMessage of pendingIpcMessages) {
+                const { message, dispatchId } = pendingMessage;
+                if (dispatchId) {
+                    const pending = state.pendingDispatches.get(dispatchId);
+                    if (!pending)
+                        continue;
+                    pending.child = child;
+                    this.sendAcknowledgedMessage(state, child, dispatchId, message);
+                    continue;
+                }
+                try {
+                    child.send?.(message);
+                    (0, runtime_1.emitNervesEvent)({
+                        component: "daemon",
+                        event: "daemon.agent_pending_ipc_flushed",
+                        message: "flushed pending IPC message to managed agent",
+                        meta: { agent },
+                    });
+                }
+                catch (error) {
+                    (0, runtime_1.emitNervesEvent)({
+                        level: "warn",
+                        component: "daemon",
+                        event: "daemon.agent_ipc_send_error",
+                        message: "failed to flush pending IPC message to managed agent",
+                        meta: { agent, error: error instanceof Error ? error.message : String(error) },
+                    });
+                }
+            }
             (0, runtime_1.emitNervesEvent)({
-                level: "error",
                 component: "daemon",
-                event: "daemon.agent_entry_missing",
-                message: "agent entry script does not exist — cannot spawn. Run 'ouro daemon install' from the correct location.",
-                meta: { agent, entryScript },
+                event: "daemon.agent_started",
+                message: "daemon started managed agent process",
+                meta: { agent, pid: child.pid ?? null, cwd: runCwd },
+            });
+            this.notifySnapshotChange(state.snapshot);
+            /* v8 ignore start — child process error handler; requires real spawn to trigger */
+            child.on("error", (err) => {
+                (0, runtime_1.emitNervesEvent)({
+                    level: "warn",
+                    component: "daemon",
+                    event: "daemon.agent_process_error",
+                    message: "managed agent process emitted error",
+                    meta: { agent, error: err.message },
+                });
+            });
+            /* v8 ignore stop */
+            child.once("exit", (code, signal) => {
+                this.onExit(state, child, code, signal);
             });
-            return;
         }
-        const args = [entryScript, "--agent", state.config.agentArg ?? agent, ...(state.config.args ?? [])];
-        const child = this.spawnFn("node", args, {
-            cwd: runCwd,
-            env: state.config.env ? { ...process.env, ...state.config.env } : process.env,
-            stdio: ["ignore", "ignore", "ignore", "ipc"],
-        });
-        state.process = child;
-        state.snapshot.status = "running";
-        state.snapshot.pid = child.pid ?? null;
-        state.snapshot.startedAt = new Date(this.now()).toISOString();
-        (0, runtime_1.emitNervesEvent)({
-            component: "daemon",
-            event: "daemon.agent_started",
-            message: "daemon started managed agent process",
-            meta: { agent, pid: child.pid ?? null, cwd: runCwd },
-        });
-        child.once("exit", (code, signal) => {
-            this.onExit(state, code, signal);
-        });
+        finally {
+            if (this.isStartAttemptCurrent(state, attemptId)) {
+                state.startInFlight = false;
+                state.startAttemptedAtMs = null;
+                if (!state.process) {
+                    this.rejectPendingDispatchesForChild(state, null, new Error(`Managed agent '${agent}' startup ended before acknowledged dispatch completed.`));
+                }
+            }
+        }
     }
     async stopAgent(agent) {
         const state = this.requireAgent(agent);
         this.clearRestartTimer(state);
+        this.clearCooldownTimer(state);
         state.stopRequested = true;
+        for (const [dispatchId] of state.pendingDispatches) {
+            this.rejectPendingDispatch(state, dispatchId, new Error(`Managed agent '${agent}' stopped before acknowledged dispatch completed.`));
+        }
+        state.pendingIpcMessages = [];
+        // NOTE: do not touch state.respawnLoopTripped / orchestratedRestartTimestamps
+        // here. restartAgent calls stopAgent internally; clearing the guard here
+        // would reset the window every cycle and defeat the loop-detection. The
+        // guard self-clears when timestamps age out of the window (handled inside
+        // restartAgent at the prune step).
         if (!state.process) {
             state.snapshot.status = "stopped";
             state.snapshot.pid = null;
+            this.notifySnapshotChange(state.snapshot);
             return;
         }
         const child = state.process;
-        state.process = null;
-        state.snapshot.status = "stopped";
-        state.snapshot.pid = null;
-        try {
-            child.kill("SIGTERM");
+        const childPid = child.pid ?? null;
+        const childPidLabel = childPid === null ? "unknown" : String(childPid);
+        await new Promise((resolve, reject) => {
+            let settled = false;
+            let timeout = null;
+            const finish = (error) => {
+                /* v8 ignore next -- defensive idempotence if an injected timer fires after cancellation @preserve */
+                if (settled)
+                    return;
+                settled = true;
+                child.removeListener("exit", onExit);
+                if (timeout !== null)
+                    this.clearTimeoutFn(timeout);
+                if (error)
+                    reject(error);
+                else
+                    resolve();
+            };
+            const onExit = () => finish();
+            child.once("exit", onExit);
+            let signalAccepted = false;
+            try {
+                signalAccepted = child.kill("SIGTERM");
+            }
+            catch {
+                const errorReason = "failed to send SIGTERM to managed agent; worker remains tracked";
+                state.stopFailureActive = true;
+                state.snapshot.errorReason = errorReason;
+                state.snapshot.fixHint = `Inspect managed worker PID ${childPidLabel} before retrying the stop or restart.`;
+                (0, runtime_1.emitNervesEvent)({
+                    level: "error",
+                    component: "daemon",
+                    event: "daemon.agent_stop_error",
+                    message: errorReason,
+                    meta: { agent, pid: childPid },
+                });
+                this.notifySnapshotChange(state.snapshot);
+                finish(new Error(errorReason));
+                return;
+            }
+            if (!signalAccepted) {
+                const errorReason = "managed agent rejected SIGTERM; worker remains tracked";
+                state.stopFailureActive = true;
+                state.snapshot.errorReason = errorReason;
+                state.snapshot.fixHint = `Inspect managed worker PID ${childPidLabel} before retrying the stop or restart.`;
+                (0, runtime_1.emitNervesEvent)({
+                    level: "error",
+                    component: "daemon",
+                    event: "daemon.agent_stop_error",
+                    message: errorReason,
+                    meta: { agent, pid: childPid },
+                });
+                this.notifySnapshotChange(state.snapshot);
+                finish(new Error(errorReason));
+                return;
+            }
+            if (settled)
+                return;
+            timeout = this.setTimeoutFn(() => {
+                if (settled)
+                    return;
+                const errorReason = `managed agent did not exit within ${this.stopTimeoutMs}ms; replacement was not started`;
+                state.stopFailureActive = true;
+                state.snapshot.errorReason = errorReason;
+                state.snapshot.fixHint = `Inspect managed worker PID ${childPidLabel}; retry only after that worker has exited.`;
+                (0, runtime_1.emitNervesEvent)({
+                    level: "error",
+                    component: "daemon",
+                    event: "daemon.agent_stop_timeout",
+                    message: errorReason,
+                    meta: { agent, pid: childPid, timeoutMs: this.stopTimeoutMs },
+                });
+                this.notifySnapshotChange(state.snapshot);
+                finish(new Error(errorReason));
+            }, this.stopTimeoutMs);
+        });
+    }
+    /**
+     * Stop an agent whose home moved to another machine and keep it stopped:
+     * it is parked (health checks no longer restart it) and no later start,
+     * wake or restart runs it again in this daemon. `ouro up` replaces the
+     * daemon with one that does not manage it at all.
+     */
+    async displaceAgent(agent, reason) {
+        const state = this.requireAgent(agent);
+        if (state.displacedReason !== null)
+            return;
+        state.displacedReason = reason;
+        state.snapshot.autoStart = false;
+        state.snapshot.errorReason = reason;
+        state.snapshot.fixHint = "Run 'ouro up' on this machine to stop managing it, or 'ouro move <agent> here' to take it back.";
+        (0, runtime_1.emitNervesEvent)({
+            level: "warn",
+            component: "daemon",
+            event: "daemon.agent_displaced",
+            message: "stopping agent homed on another machine",
+            meta: { agent, reason },
+        });
+        this.notifySnapshotChange(state.snapshot);
+        await this.stopAgent(agent);
+    }
+    async restartAgent(agent, options = {}) {
+        const state = this.requireAgent(agent);
+        if (this.shutdownRequested)
+            return;
+        // Respawn-loop guard: prune timestamps outside the window, then check
+        // whether we've already restarted this agent too many times in it.
+        const now = this.now();
+        const windowStart = now - exports.RESPAWN_GUARD_WINDOW_MS;
+        state.orchestratedRestartTimestamps = state.orchestratedRestartTimestamps.filter((ts) => ts >= windowStart);
+        // If the window is now empty, the trip naturally self-clears. That means
+        // after RESPAWN_GUARD_WINDOW_MS of no restart attempts, the daemon is
+        // willing to try again (e.g. for a fresh health probe failure that has
+        // nothing to do with the original loop).
+        if (state.respawnLoopTripped && state.orchestratedRestartTimestamps.length === 0) {
+            state.respawnLoopTripped = false;
+            state.snapshot.errorReason = null;
+            state.snapshot.fixHint = null;
+            (0, runtime_1.emitNervesEvent)({
+                component: "daemon",
+                event: "daemon.agent_respawn_loop_cleared",
+                message: "respawn-loop guard cleared by window-aging",
+                meta: { agent, windowMs: exports.RESPAWN_GUARD_WINDOW_MS },
+            });
+            this.notifySnapshotChange(state.snapshot);
         }
-        catch {
+        if (state.respawnLoopTripped) {
+            (0, runtime_1.emitNervesEvent)({
+                level: "error",
+                component: "daemon",
+                event: "daemon.agent_respawn_loop_blocked",
+                message: "refused agent restart — respawn-loop guard tripped; manual intervention required",
+                meta: {
+                    agent,
+                    recentRestartCount: state.orchestratedRestartTimestamps.length,
+                    windowMs: exports.RESPAWN_GUARD_WINDOW_MS,
+                },
+            });
+            return;
+        }
+        if (state.orchestratedRestartTimestamps.length >= exports.RESPAWN_GUARD_MAX_RESTARTS) {
+            state.respawnLoopTripped = true;
+            state.snapshot.errorReason = `respawn loop detected: ${exports.RESPAWN_GUARD_MAX_RESTARTS}+ restarts in ${Math.round(exports.RESPAWN_GUARD_WINDOW_MS / 60_000)}min — refusing further restarts`;
+            state.snapshot.fixHint = "investigate the root cause then run `ouro up` to resume";
+            (0, runtime_1.emitNervesEvent)({
+                level: "error",
+                component: "daemon",
+                event: "daemon.agent_respawn_loop_tripped",
+                message: "respawn-loop guard tripped; further restarts blocked",
+                meta: {
+                    agent,
+                    restartCount: state.orchestratedRestartTimestamps.length,
+                    windowMs: exports.RESPAWN_GUARD_WINDOW_MS,
+                    maxRestarts: exports.RESPAWN_GUARD_MAX_RESTARTS,
+                },
+            });
+            this.notifySnapshotChange(state.snapshot);
+            return;
+        }
+        state.orchestratedRestartTimestamps.push(now);
+        if (state.startInFlight && !state.process) {
+            const startedAt = state.startAttemptedAtMs;
+            /* v8 ignore next -- defensive: startInFlight always records a start timestamp @preserve */
+            const elapsedMs = startedAt === null ? 0 : this.currentTimeMs() - startedAt;
+            if (elapsedMs < this.startupStaleAfterMs) {
+                (0, runtime_1.emitNervesEvent)({
+                    component: "daemon",
+                    event: "daemon.agent_restart_deferred",
+                    message: "managed agent restart skipped while startup is already in flight",
+                    meta: { agent, elapsedMs, startupStaleAfterMs: this.startupStaleAfterMs },
+                });
+                return;
+            }
             (0, runtime_1.emitNervesEvent)({
                 level: "warn",
                 component: "daemon",
-                event: "daemon.agent_stop_error",
-                message: "failed to send SIGTERM to managed agent",
-                meta: { agent },
+                event: "daemon.agent_startup_stale_recovered",
+                message: "managed agent startup was stale; clearing the pending attempt before restart",
+                meta: { agent, elapsedMs, startupStaleAfterMs: this.startupStaleAfterMs },
             });
+            state.startAttemptId += 1;
+            state.startInFlight = false;
+            state.startAttemptedAtMs = null;
         }
-    }
-    async restartAgent(agent) {
         await this.stopAgent(agent);
-        await this.startAgent(agent);
+        await this.startAgent(agent, options);
     }
     async stopAll() {
-        for (const state of this.agents.values()) {
-            await this.stopAgent(state.config.name);
+        this.shutdownRequested = true;
+        const states = [...this.agents.values()];
+        const results = await Promise.allSettled(states.map((state) => this.stopAgent(state.config.name)));
+        const failures = results.flatMap((result, index) => result.status === "rejected"
+            ? [{ agent: states[index].config.name, reason: result.reason }]
+            : []);
+        if (failures.length > 0) {
+            throw new AggregateError(failures.map((failure) => failure.reason), `failed to stop ${failures.length} managed worker(s): ${failures.map((failure) => failure.agent).join(", ")}`);
         }
     }
+    resetAgentFailureState(agent) {
+        const state = this.requireAgent(agent);
+        this.clearRestartTimer(state);
+        this.clearCooldownTimer(state);
+        state.cooldownRetryCount = 0;
+        state.crashTimestamps = [];
+        state.fastCrashCount = 0;
+        state.respawnLoopTripped = false;
+        state.orchestratedRestartTimestamps = [];
+        state.snapshot.errorReason = null;
+        state.snapshot.fixHint = null;
+        this.notifySnapshotChange(state.snapshot);
+    }
     sendToAgent(agent, message) {
         const state = this.requireAgent(agent);
-        if (!state.process)
+        if (!state.process) {
+            if (state.startInFlight) {
+                if (state.pendingIpcMessages.length >= MAX_PENDING_IPC_MESSAGES) {
+                    const evicted = state.pendingIpcMessages.shift();
+                    if (evicted?.dispatchId)
+                        this.rejectPendingDispatch(state, evicted.dispatchId, new Error(`Managed agent '${agent}' startup IPC queue evicted acknowledged dispatch.`));
+                }
+                state.pendingIpcMessages.push({ message });
+                (0, runtime_1.emitNervesEvent)({
+                    component: "daemon",
+                    event: "daemon.agent_ipc_queued_during_startup",
+                    message: "queued IPC message while managed agent startup is in flight",
+                    meta: { agent, queuedCount: state.pendingIpcMessages.length },
+                });
+            }
             return;
+        }
         try {
             state.process.send(message);
         }
@@ -181,47 +719,168 @@ class DaemonProcessManager {
             });
         }
     }
+    dispatchToAgent(agent, message, options = {}) {
+        const state = this.requireAgent(agent);
+        const dispatchId = (0, crypto_1.randomUUID)();
+        const timeoutMs = options.timeoutMs ?? exports.PRIVATE_RUNTIME_DISPATCH_ACK_TIMEOUT_MS;
+        return new Promise((resolve, reject) => {
+            const timer = this.setTimeoutFn(() => {
+                const pending = state.pendingDispatches.get(dispatchId);
+                if (!pending)
+                    return;
+                state.pendingDispatches.delete(dispatchId);
+                if (pending.child && state.process === pending.child) {
+                    try {
+                        pending.child.send?.({ type: "ouro.privateRuntimeDispatchCancel", dispatchId });
+                    }
+                    catch { /* cancellation is best effort */ }
+                }
+                reject(new Error(`Private-runtime dispatch timed out after ${timeoutMs}ms.`));
+            }, timeoutMs);
+            const child = state.process?.connected === false ? null : state.process;
+            state.pendingDispatches.set(dispatchId, { child, timer, resolve, reject });
+            if (child) {
+                this.sendAcknowledgedMessage(state, child, dispatchId, message);
+                return;
+            }
+            if (state.startInFlight) {
+                if (state.pendingIpcMessages.length >= MAX_PENDING_IPC_MESSAGES) {
+                    const evicted = state.pendingIpcMessages.shift();
+                    if (evicted?.dispatchId)
+                        this.rejectPendingDispatch(state, evicted.dispatchId, new Error(`Managed agent '${agent}' startup IPC queue evicted acknowledged dispatch.`));
+                }
+                state.pendingIpcMessages.push({ message, dispatchId });
+                return;
+            }
+            this.rejectPendingDispatch(state, dispatchId, new Error(`Managed agent '${agent}' is unavailable for acknowledged dispatch.`));
+        });
+    }
+    rejectPendingDispatch(state, dispatchId, error) {
+        const pending = state.pendingDispatches.get(dispatchId);
+        if (!pending)
+            return;
+        state.pendingDispatches.delete(dispatchId);
+        this.clearTimeoutFn(pending.timer);
+        pending.reject(error);
+    }
+    rejectPendingDispatchesForChild(state, child, error) {
+        for (const [dispatchId, pending] of state.pendingDispatches) {
+            if (pending.child === child)
+                this.rejectPendingDispatch(state, dispatchId, error);
+        }
+        if (child === null)
+            state.pendingIpcMessages = state.pendingIpcMessages.filter((pending) => pending.dispatchId === undefined);
+    }
+    sendAcknowledgedMessage(state, child, dispatchId, message) {
+        try {
+            child.send?.({ ...message, dispatchId }, (error) => {
+                if (!error)
+                    return;
+                const pending = state.pendingDispatches.get(dispatchId);
+                if (!pending || pending.child !== child)
+                    return;
+                this.rejectPendingDispatch(state, dispatchId, error);
+            });
+        }
+        catch (error) {
+            this.rejectPendingDispatch(state, dispatchId, error instanceof Error ? error : new Error(String(error)));
+        }
+    }
     getAgentSnapshot(agent) {
         return this.agents.get(agent)?.snapshot;
     }
     listAgentSnapshots() {
         return [...this.agents.values()].map((state) => state.snapshot);
     }
-    onExit(state, code, signal) {
-        if (!state.process)
+    onExit(state, child, code, signal) {
+        /* v8 ignore next -- defensive: replacement cannot start before this child's one-shot exit listener drains @preserve */
+        if (state.process !== child)
             return;
+        this.rejectPendingDispatchesForChild(state, child, new Error(`Managed agent '${state.config.name}' exited before acknowledged dispatch completed.`));
         state.process = null;
+        state.startInFlight = false;
+        state.startAttemptedAtMs = null;
         state.snapshot.pid = null;
-        const crashed = !state.stopRequested && code !== 0;
-        const now = this.now();
+        state.snapshot.lastExitCode = code;
+        state.snapshot.lastSignal = signal;
+        if (state.stopFailureActive) {
+            state.stopFailureActive = false;
+            state.snapshot.errorReason = null;
+            state.snapshot.fixHint = null;
+        }
+        const unexpectedExit = !state.stopRequested;
+        const crashed = unexpectedExit && code !== 0;
+        const now = this.currentTimeMs();
         const startedAt = state.snapshot.startedAt ? Date.parse(state.snapshot.startedAt) : now;
         const runDuration = Math.max(0, now - startedAt);
         (0, runtime_1.emitNervesEvent)({
-            level: crashed ? "warn" : "info",
+            level: unexpectedExit ? "warn" : "info",
             component: "daemon",
             event: "daemon.agent_exit",
             message: "managed agent process exited",
-            meta: { agent: state.config.name, code, signal, crashed, runDurationMs: runDuration },
+            meta: { agent: state.config.name, code, signal, crashed, unexpectedExit, runDurationMs: runDuration },
         });
-        if (!crashed) {
+        if (runDuration >= this.stabilityThresholdMs) {
+            state.snapshot.backoffMs = this.initialBackoffMs;
+        }
+        if (!unexpectedExit) {
             state.snapshot.status = "stopped";
-            if (runDuration >= this.stabilityThresholdMs) {
-                state.snapshot.backoffMs = this.initialBackoffMs;
-            }
+            this.notifySnapshotChange(state.snapshot);
             return;
         }
-        state.snapshot.lastCrashAt = new Date(now).toISOString();
+        if (crashed) {
+            state.snapshot.lastCrashAt = new Date(now).toISOString();
+        }
+        if (state.config.autoStart === false) {
+            state.snapshot.status = "stopped";
+            state.snapshot.errorReason = "non-autostart worker exited unexpectedly; not respawning";
+            state.snapshot.fixHint = "Start it explicitly after checking the exit cause.";
+            this.notifySnapshotChange(state.snapshot);
+            return;
+        }
+        // Fast-exit detection: repeated early exits, including clean exit-code-0
+        // terminations, indicate a startup/configuration or lifecycle fault rather
+        // than a stable long-lived worker. Bound them without claiming a crash.
+        const FAST_CRASH_THRESHOLD_MS = 5000;
+        const FAST_CRASH_MAX = 3;
+        if (runDuration < FAST_CRASH_THRESHOLD_MS) {
+            state.fastCrashCount = state.fastCrashCount + 1;
+            if (state.fastCrashCount >= FAST_CRASH_MAX) {
+                state.snapshot.status = "crashed";
+                // Capture the fast-exit diagnosis on the snapshot so it surfaces via
+                // the pulse with a prescriptive, cause-neutral repair path.
+                state.snapshot.errorReason = `agent exited unexpectedly ${FAST_CRASH_MAX} times within ${FAST_CRASH_THRESHOLD_MS}ms of startup — likely a startup/configuration or lifecycle issue.`;
+                state.snapshot.fixHint = `Inspect daemon logs and lifecycle/configuration, then run \`ouro up\` to restart.`;
+                (0, runtime_1.emitNervesEvent)({
+                    level: "error",
+                    component: "daemon",
+                    event: "daemon.agent_config_failure",
+                    message: `agent exited unexpectedly ${FAST_CRASH_MAX} times within ${FAST_CRASH_THRESHOLD_MS}ms of startup. Not retrying. Inspect lifecycle/configuration and run \`ouro up\` to restart.`,
+                    meta: { agent: state.config.name, fastUnexpectedExitCount: state.fastCrashCount, avgRunDurationMs: runDuration },
+                });
+                this.notifySnapshotChange(state.snapshot);
+                return; // Don't schedule cooldown recovery — this needs human/agent intervention
+            }
+        }
+        else {
+            // Reset the fast unexpected-exit counter on a stable run.
+            state.fastCrashCount = 0;
+        }
         state.crashTimestamps = state.crashTimestamps.filter((crashTs) => crashTs >= startOfHour(now));
         state.crashTimestamps.push(now);
         if (state.crashTimestamps.length > this.maxRestartsPerHour) {
             state.snapshot.status = "crashed";
+            state.snapshot.errorReason = `agent exceeded unexpected-exit restart limit (${this.maxRestartsPerHour}/hr) — entering cooldown`;
+            state.snapshot.fixHint = "investigate why the agent keeps exiting unexpectedly; cooldown will retry shortly";
             (0, runtime_1.emitNervesEvent)({
                 level: "error",
                 component: "daemon",
                 event: "daemon.agent_restart_exhausted",
-                message: "managed agent exceeded restart limit and is marked crashed",
+                message: "managed agent exceeded unexpected-exit restart limit and is marked crashed",
                 meta: { agent: state.config.name, maxRestartsPerHour: this.maxRestartsPerHour },
             });
+            this.notifySnapshotChange(state.snapshot);
+            this.scheduleCooldownRecovery(state);
             return;
         }
         state.snapshot.status = "starting";
@@ -232,6 +891,7 @@ class DaemonProcessManager {
         state.restartTimer = this.setTimeoutFn(() => {
             void this.startAgent(state.config.name);
         }, waitMs);
+        this.notifySnapshotChange(state.snapshot);
     }
     clearRestartTimer(state) {
         if (state.restartTimer === null)
@@ -239,6 +899,45 @@ class DaemonProcessManager {
         this.clearTimeoutFn(state.restartTimer);
         state.restartTimer = null;
     }
+    scheduleCooldownRecovery(state) {
+        if (state.cooldownRetryCount >= this.maxCooldownRetries) {
+            (0, runtime_1.emitNervesEvent)({
+                level: "error",
+                component: "daemon",
+                event: "daemon.agent_permanent_failure",
+                message: "managed agent exhausted all cooldown retries — permanently stopped",
+                meta: { agent: state.config.name, cooldownRetryCount: state.cooldownRetryCount, maxCooldownRetries: this.maxCooldownRetries },
+            });
+            return;
+        }
+        this.clearCooldownTimer(state);
+        state.cooldownTimer = this.setTimeoutFn(() => {
+            state.cooldownRetryCount += 1;
+            state.crashTimestamps = [];
+            state.snapshot.backoffMs = this.initialBackoffMs;
+            state.snapshot.status = "starting";
+            state.snapshot.restartCount += 1;
+            (0, runtime_1.emitNervesEvent)({
+                component: "daemon",
+                event: "daemon.agent_cooldown_recovery",
+                message: "attempting cooldown recovery for managed agent",
+                meta: { agent: state.config.name, cooldownRetryCount: state.cooldownRetryCount },
+            });
+            void this.startAgent(state.config.name);
+        }, this.cooldownRecoveryMs);
+        (0, runtime_1.emitNervesEvent)({
+            component: "daemon",
+            event: "daemon.agent_cooldown_scheduled",
+            message: `scheduled cooldown recovery in ${this.cooldownRecoveryMs}ms`,
+            meta: { agent: state.config.name, cooldownRecoveryMs: this.cooldownRecoveryMs, cooldownRetryCount: state.cooldownRetryCount },
+        });
+    }
+    clearCooldownTimer(state) {
+        if (state.cooldownTimer === null)
+            return;
+        this.clearTimeoutFn(state.cooldownTimer);
+        state.cooldownTimer = null;
+    }
     requireAgent(agent) {
         const state = this.agents.get(agent);
         if (!state) {