@ouro.bot/cli 0.1.0-alpha.56 → 0.1.0-alpha.561

package/dist/heart/daemon/daemon-runtime-sync.js

@@ -2,6 +2,22 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ensureCurrentDaemonRuntime = ensureCurrentDaemonRuntime;
 const runtime_1 = require("../../nerves/runtime");
+/* v8 ignore start -- daemon liveness poll: real socket timing untestable in vitest @preserve */
+async function verifyDaemonStarted(deps) {
+    if (!deps.checkSocketAlive)
+        return true;
+    const maxWaitMs = 10_000;
+    const pollIntervalMs = 500;
+    const deadline = Date.now() + maxWaitMs;
+    deps.onProgress?.("waiting for the replacement background service to answer");
+    while (Date.now() < deadline) {
+        await new Promise((r) => setTimeout(r, pollIntervalMs));
+        if (await deps.checkSocketAlive(deps.socketPath))
+            return true;
+    }
+    return false;
+}
+/* v8 ignore stop */
 function isKnownVersion(version) {
     return version !== "unknown" && version.trim().length > 0;
 }
@@ -17,6 +33,7 @@ function normalizeRuntimeIdentity(value) {
         lastUpdated: typeof value.lastUpdated === "string" ? value.lastUpdated : "unknown",
         repoRoot: typeof value.repoRoot === "string" ? value.repoRoot : "unknown",
         configFingerprint: typeof value.configFingerprint === "string" ? value.configFingerprint : "unknown",
+        managedAgents: typeof value.managedAgents === "string" ? value.managedAgents : "unknown",
     };
 }
 async function readRunningRuntimeIdentity(deps) {
@@ -55,23 +72,29 @@ function collectRuntimeDriftReasons(local, running) {
             running: running.configFingerprint,
         });
     }
-    return reasons;
-}
-function formatRuntimeValue(reason) {
-    if (reason.key === "configFingerprint") {
-        return `${reason.running.slice(0, 12)} -> ${reason.local.slice(0, 12)}`;
+    if (isKnownRuntimeValue(local.managedAgents)
+        && isKnownRuntimeValue(running.managedAgents)
+        && local.managedAgents !== running.managedAgents) {
+        reasons.push({
+            key: "managedAgents",
+            label: "managed agents",
+            local: local.managedAgents,
+            running: running.managedAgents,
+        });
     }
-    return `${reason.running} -> ${reason.local}`;
+    return reasons;
 }
-function formatRuntimeDriftSummary(reasons) {
-    return reasons.map((reason) => `${reason.label} ${formatRuntimeValue(reason)}`).join("; ");
+function formatRuntimeDriftPublicSummary(reasons) {
+    return reasons.map((reason) => reason.label).join(", ");
 }
 async function ensureCurrentDaemonRuntime(deps) {
+    deps.onProgress?.("checking whether an older background service is already running");
     const localRuntime = normalizeRuntimeIdentity({
         version: deps.localVersion,
         lastUpdated: deps.localLastUpdated,
         repoRoot: deps.localRepoRoot,
         configFingerprint: deps.localConfigFingerprint,
+        managedAgents: deps.localManagedAgents,
     });
     try {
         const runningRuntime = await readRunningRuntimeIdentity(deps);
@@ -80,17 +103,40 @@ async function ensureCurrentDaemonRuntime(deps) {
         let result;
         if (driftReasons.length > 0) {
             const includesVersionDrift = driftReasons.some((entry) => entry.key === "version");
-            const driftSummary = formatRuntimeDriftSummary(driftReasons);
+            const publicDriftSummary = formatRuntimeDriftPublicSummary(driftReasons);
             try {
+                deps.onProgress?.("stopping the older background service");
                 await deps.stopDaemon();
+                if (deps.prepareDaemonRuntimeReplacement) {
+                    deps.onProgress?.("disabling daemon auto-restart during replacement");
+                    try {
+                        await Promise.resolve(deps.prepareDaemonRuntimeReplacement());
+                    }
+                    catch (error) {
+                        (0, runtime_1.emitNervesEvent)({
+                            level: "warn",
+                            component: "daemon",
+                            event: "daemon.runtime_sync_replacement_prepare_error",
+                            message: "daemon runtime replacement preparation failed",
+                            meta: {
+                                socketPath: deps.socketPath,
+                                reason: formatErrorReason(error),
+                            },
+                        });
+                    }
+                }
             }
             catch (error) {
                 const reason = formatErrorReason(error);
                 result = {
+                    ok: false,
                     alreadyRunning: true,
                     message: includesVersionDrift
-                        ? `daemon already running (${deps.socketPath}; could not replace stale daemon ${runningVersion} -> ${deps.localVersion}: ${reason})`
-                        : `daemon already running (${deps.socketPath}; could not replace drifted daemon ${driftSummary}: ${reason})`,
+                        ? `daemon already running (${deps.socketPath}; could not replace the older background service ${runningVersion} -> ${deps.localVersion}: ${reason})`
+                        : `daemon already running (${deps.socketPath}; could not replace the older background service after runtime drift ${publicDriftSummary}: ${reason})`,
+                    startupFailureReason: includesVersionDrift
+                        ? "could not replace the older background service"
+                        : "could not replace the older background service after runtime drift",
                 };
                 (0, runtime_1.emitNervesEvent)({
                     level: "warn",
@@ -103,10 +149,12 @@ async function ensureCurrentDaemonRuntime(deps) {
                         localLastUpdated: localRuntime.lastUpdated,
                         localRepoRoot: localRuntime.repoRoot,
                         localConfigFingerprint: localRuntime.configFingerprint,
+                        localManagedAgents: localRuntime.managedAgents,
                         runningVersion,
                         runningLastUpdated: runningRuntime.lastUpdated,
                         runningRepoRoot: runningRuntime.repoRoot,
                         runningConfigFingerprint: runningRuntime.configFingerprint,
+                        runningManagedAgents: runningRuntime.managedAgents,
                         action: "stale_replace_failed",
                         driftKeys: driftReasons.map((entry) => entry.key),
                         reason,
@@ -115,12 +163,27 @@ async function ensureCurrentDaemonRuntime(deps) {
                 return result;
             }
             deps.cleanupStaleSocket(deps.socketPath);
+            deps.onProgress?.("starting the replacement background service");
+            const bootStartedAtMs = (deps.now ?? Date.now)();
             const started = await deps.startDaemonProcess(deps.socketPath);
+            const pid = started.pid ?? "unknown";
+            const startupCheck = deps.waitForDaemonStartup
+                ? await deps.waitForDaemonStartup({ pid: started.pid ?? null, bootStartedAtMs })
+                : { ok: await verifyDaemonStarted(deps) };
+            const verified = startupCheck.ok;
+            /* v8 ignore next -- daemon liveness failure: requires real daemon crash timing @preserve */
+            const suffix = verified
+                ? ""
+                : `\n${startupCheck.reason ?? "replacement background service did not answer in time"}; check logs with \`ouro logs\` or run \`ouro doctor\`.`;
             result = {
+                ok: verified,
                 alreadyRunning: false,
                 message: includesVersionDrift
-                    ? `restarted stale daemon from ${runningVersion} to ${deps.localVersion} (pid ${started.pid ?? "unknown"})`
-                    : `restarted drifted daemon (${driftSummary}) (pid ${started.pid ?? "unknown"})`,
+                    ? `replaced an older background service ${runningVersion} -> ${deps.localVersion} (pid ${pid})${suffix}`
+                    : `replaced an older background service after runtime drift: ${publicDriftSummary} (pid ${pid})${suffix}`,
+                verifyStartupStatus: verified,
+                startedPid: started.pid ?? null,
+                startupFailureReason: verified ? null : (startupCheck.reason ?? "replacement background service did not answer in time"),
             };
             (0, runtime_1.emitNervesEvent)({
                 component: "daemon",
@@ -132,10 +195,12 @@ async function ensureCurrentDaemonRuntime(deps) {
                     localLastUpdated: localRuntime.lastUpdated,
                     localRepoRoot: localRuntime.repoRoot,
                     localConfigFingerprint: localRuntime.configFingerprint,
+                    localManagedAgents: localRuntime.managedAgents,
                     runningVersion,
                     runningLastUpdated: runningRuntime.lastUpdated,
                     runningRepoRoot: runningRuntime.repoRoot,
                     runningConfigFingerprint: runningRuntime.configFingerprint,
+                    runningManagedAgents: runningRuntime.managedAgents,
                     action: "stale_restarted",
                     driftKeys: driftReasons.map((entry) => entry.key),
                     pid: started.pid ?? null,
@@ -145,6 +210,7 @@ async function ensureCurrentDaemonRuntime(deps) {
         }
         if (!isKnownVersion(localRuntime.version) || !isKnownVersion(runningVersion)) {
             result = {
+                ok: true,
                 alreadyRunning: true,
                 message: `daemon already running (${deps.socketPath}; unable to verify version)`,
             };
@@ -158,10 +224,12 @@ async function ensureCurrentDaemonRuntime(deps) {
                     localLastUpdated: localRuntime.lastUpdated,
                     localRepoRoot: localRuntime.repoRoot,
                     localConfigFingerprint: localRuntime.configFingerprint,
+                    localManagedAgents: localRuntime.managedAgents,
                     runningVersion,
                     runningLastUpdated: runningRuntime.lastUpdated,
                     runningRepoRoot: runningRuntime.repoRoot,
                     runningConfigFingerprint: runningRuntime.configFingerprint,
+                    runningManagedAgents: runningRuntime.managedAgents,
                     action: "unknown_version",
                 },
             });
@@ -171,6 +239,7 @@ async function ensureCurrentDaemonRuntime(deps) {
     catch (error) {
         const reason = formatErrorReason(error);
         const result = {
+            ok: true,
             alreadyRunning: true,
             message: `daemon already running (${deps.socketPath}; unable to verify version: ${reason})`,
         };
@@ -185,6 +254,7 @@ async function ensureCurrentDaemonRuntime(deps) {
                 localLastUpdated: localRuntime.lastUpdated,
                 localRepoRoot: localRuntime.repoRoot,
                 localConfigFingerprint: localRuntime.configFingerprint,
+                localManagedAgents: localRuntime.managedAgents,
                 action: "status_lookup_failed",
                 reason,
             },
@@ -192,8 +262,12 @@ async function ensureCurrentDaemonRuntime(deps) {
         return result;
     }
     const result = {
+        ok: true,
         alreadyRunning: true,
         message: `daemon already running (${deps.socketPath})`,
+        verifyStartupStatus: true,
+        startedPid: null,
+        startupFailureReason: null,
     };
     (0, runtime_1.emitNervesEvent)({
         component: "daemon",
@@ -205,6 +279,7 @@ async function ensureCurrentDaemonRuntime(deps) {
             localLastUpdated: localRuntime.lastUpdated,
             localRepoRoot: localRuntime.repoRoot,
             localConfigFingerprint: localRuntime.configFingerprint,
+            localManagedAgents: localRuntime.managedAgents,
             action: "already_current",
         },
     });

package/dist/heart/daemon/daemon-tombstone.js

@@ -0,0 +1,236 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RECENT_CRASHES_MAX = void 0;
+exports.getTombstonePath = getTombstonePath;
+exports.setTombstonePath = setTombstonePath;
+exports.captureDeathForensics = captureDeathForensics;
+exports.writeDaemonTombstone = writeDaemonTombstone;
+exports.readDaemonTombstone = readDaemonTombstone;
+const child_process_1 = require("child_process");
+const fs = __importStar(require("fs"));
+const os = __importStar(require("os"));
+const path = __importStar(require("path"));
+const runtime_1 = require("../../nerves/runtime");
+/** Maximum number of historical crash timestamps to retain in the tombstone. */
+exports.RECENT_CRASHES_MAX = 100;
+let _tombstonePath = null;
+function getTombstonePath() {
+    if (!_tombstonePath) {
+        _tombstonePath = path.join(os.homedir(), ".ouro-cli", "daemon-death.json");
+    }
+    return _tombstonePath;
+}
+/** Overrides the tombstone path for testing. Pass null to reset. */
+function setTombstonePath(p) {
+    _tombstonePath = p;
+}
+/* v8 ignore start -- shells out to ps; covered via injected runPs in unit tests @preserve */
+function defaultRunPs(args) {
+    try {
+        return (0, child_process_1.execSync)(`ps ${args.join(" ")}`, { encoding: "utf-8", timeout: 2000 });
+    }
+    catch {
+        return null;
+    }
+}
+/* v8 ignore stop */
+const DEFAULT_FORENSICS_DEPS = {
+    /* v8 ignore next -- defensive: process.ppid always exists in node @preserve */
+    ppid: () => (typeof process !== "undefined" && typeof process.ppid === "number" ? process.ppid : 0),
+    runPs: defaultRunPs,
+};
+/**
+ * Capture forensic data about who is likely to have killed the daemon.
+ * Synchronous and best-effort: never throws, always returns a structured
+ * record (with nulls if anything failed). Called from inside SIGTERM/SIGINT
+ * handlers, so it must be fast and have no async dependencies.
+ */
+function captureDeathForensics(deps = DEFAULT_FORENSICS_DEPS) {
+    const parentPid = (() => {
+        try {
+            const p = deps.ppid();
+            return typeof p === "number" && p > 0 ? p : null;
+        }
+        catch {
+            return null;
+        }
+    })();
+    let parentCommand = null;
+    if (parentPid !== null) {
+        const psParent = deps.runPs(["-p", String(parentPid), "-o", "command="]);
+        if (psParent !== null) {
+            const trimmed = psParent.trim();
+            parentCommand = trimmed.length > 0 ? trimmed : null;
+        }
+    }
+    const processSnapshot = (() => {
+        const psAll = deps.runPs(["-eo", "pid,ppid,command"]);
+        if (psAll === null)
+            return null;
+        const lines = psAll.split("\n");
+        // Filter for relevant processes only — node, vitest, ouro, kill commands.
+        // Keeps the snapshot small and human-scannable.
+        const relevant = lines.filter((line) => {
+            const lower = line.toLowerCase();
+            return (lower.includes("node")
+                || lower.includes("vitest")
+                || lower.includes("ouro")
+                || lower.includes("/kill ")
+                || lower.includes("pkill")
+                || lower.includes("killall"));
+        });
+        if (relevant.length === 0)
+            return null;
+        return relevant.join("\n");
+    })();
+    const killerHint = deriveKillerHint(parentCommand, processSnapshot);
+    return { parentPid, parentCommand, processSnapshot, killerHint };
+}
+function deriveKillerHint(parentCommand, snapshot) {
+    if (parentCommand !== null && parentCommand.toLowerCase().includes("launchd")) {
+        return "process was reparented to launchd — likely killed by launchctl bootout, KeepAlive thrash, or RSS pressure";
+    }
+    if (snapshot !== null) {
+        const lower = snapshot.toLowerCase();
+        if (lower.includes("vitest")) {
+            return "vitest worker is running — possible test cleanup killing detached processes";
+        }
+        if (lower.includes("pkill") || lower.includes("killall")) {
+            return "saw pkill/killall in process list — explicit kill command";
+        }
+    }
+    return null;
+}
+function writeDaemonTombstone(reason, error, forensicsDeps) {
+    const now = new Date().toISOString();
+    const filePath = getTombstonePath();
+    // Read existing recentCrashes from previous tombstone (best-effort)
+    let existingCrashes = [];
+    try {
+        const raw = fs.readFileSync(filePath, "utf-8");
+        const existing = JSON.parse(raw);
+        if (Array.isArray(existing.recentCrashes)) {
+            existingCrashes = existing.recentCrashes.filter((entry) => typeof entry === "string");
+        }
+    }
+    catch {
+        // No existing tombstone or unreadable — start fresh
+    }
+    // Append the new crash and cap at the most recent RECENT_CRASHES_MAX entries
+    // so the tombstone doesn't grow without bound (we saw it hit 12,265 entries
+    // after a March 31 crash-restart thrash loop).
+    const recentCrashes = [...existingCrashes, now].slice(-exports.RECENT_CRASHES_MAX);
+    // Forensics: only meaningful for signal-driven deaths (sigterm/sigint).
+    // For uncaughtException etc. we skip the snapshot to keep tombstone writes
+    // fast on the unhappy path.
+    const shouldCaptureForensics = reason === "sigterm" || reason === "sigint";
+    const forensics = shouldCaptureForensics
+        ? captureDeathForensics(forensicsDeps)
+        : undefined;
+    const tombstone = {
+        reason,
+        message: error?.message ?? reason,
+        stack: error?.stack ?? null,
+        timestamp: now,
+        pid: process.pid,
+        uptimeSeconds: Math.floor(process.uptime()),
+        recentCrashes,
+        ...(forensics ? { forensics } : {}),
+    };
+    try {
+        fs.mkdirSync(path.dirname(filePath), { recursive: true });
+        fs.writeFileSync(filePath, JSON.stringify(tombstone, null, 2) + "\n", "utf-8");
+    }
+    catch {
+        // Best-effort: if we can't write, we're already dying.
+    }
+    (0, runtime_1.emitNervesEvent)({
+        level: "error",
+        component: "daemon",
+        event: "daemon.tombstone_written",
+        message: `daemon tombstone written: ${reason}`,
+        meta: {
+            reason,
+            errorMessage: error?.message ?? null,
+            filePath,
+            parentPid: forensics?.parentPid ?? null,
+            killerHint: forensics?.killerHint ?? null,
+        },
+    });
+}
+function readDaemonTombstone() {
+    const filePath = getTombstonePath();
+    try {
+        const raw = fs.readFileSync(filePath, "utf-8");
+        const parsed = JSON.parse(raw);
+        if (typeof parsed.reason !== "string" || typeof parsed.timestamp !== "string") {
+            return null;
+        }
+        (0, runtime_1.emitNervesEvent)({
+            component: "daemon",
+            event: "daemon.tombstone_read",
+            message: "read daemon tombstone",
+            meta: { filePath },
+        });
+        const forensics = parseForensics(parsed.forensics);
+        return {
+            reason: parsed.reason,
+            message: typeof parsed.message === "string" ? parsed.message : String(parsed.reason),
+            stack: typeof parsed.stack === "string" ? parsed.stack : null,
+            timestamp: parsed.timestamp,
+            pid: typeof parsed.pid === "number" ? parsed.pid : 0,
+            uptimeSeconds: typeof parsed.uptimeSeconds === "number" ? parsed.uptimeSeconds : 0,
+            recentCrashes: Array.isArray(parsed.recentCrashes)
+                ? parsed.recentCrashes.filter((e) => typeof e === "string")
+                : [],
+            ...(forensics ? { forensics } : {}),
+        };
+    }
+    catch {
+        return null;
+    }
+}
+function parseForensics(value) {
+    if (value === null || typeof value !== "object")
+        return null;
+    const v = value;
+    return {
+        parentPid: typeof v.parentPid === "number" ? v.parentPid : null,
+        parentCommand: typeof v.parentCommand === "string" ? v.parentCommand : null,
+        processSnapshot: typeof v.processSnapshot === "string" ? v.processSnapshot : null,
+        killerHint: typeof v.killerHint === "string" ? v.killerHint : null,
+    };
+}