@ouro.bot/cli 0.1.0-alpha.44 → 0.1.0-alpha.441

package/dist/heart/daemon/daemon-runtime-sync.js

@@ -2,52 +2,183 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ensureCurrentDaemonRuntime = ensureCurrentDaemonRuntime;
 const runtime_1 = require("../../nerves/runtime");
+/* v8 ignore start -- daemon liveness poll: real socket timing untestable in vitest @preserve */
+async function verifyDaemonStarted(deps) {
+    if (!deps.checkSocketAlive)
+        return true;
+    const maxWaitMs = 10_000;
+    const pollIntervalMs = 500;
+    const deadline = Date.now() + maxWaitMs;
+    deps.onProgress?.("waiting for the replacement background service to answer");
+    while (Date.now() < deadline) {
+        await new Promise((r) => setTimeout(r, pollIntervalMs));
+        if (await deps.checkSocketAlive(deps.socketPath))
+            return true;
+    }
+    return false;
+}
+/* v8 ignore stop */
 function isKnownVersion(version) {
     return version !== "unknown" && version.trim().length > 0;
 }
+function isKnownRuntimeValue(value) {
+    return typeof value === "string" && value.trim().length > 0 && value !== "unknown";
+}
 function formatErrorReason(error) {
     return error instanceof Error ? error.message : String(error);
 }
+function normalizeRuntimeIdentity(value) {
+    return {
+        version: typeof value.version === "string" ? value.version : "unknown",
+        lastUpdated: typeof value.lastUpdated === "string" ? value.lastUpdated : "unknown",
+        repoRoot: typeof value.repoRoot === "string" ? value.repoRoot : "unknown",
+        configFingerprint: typeof value.configFingerprint === "string" ? value.configFingerprint : "unknown",
+        managedAgents: typeof value.managedAgents === "string" ? value.managedAgents : "unknown",
+    };
+}
+async function readRunningRuntimeIdentity(deps) {
+    if (!deps.fetchRunningRuntimeMetadata) {
+        return normalizeRuntimeIdentity({
+            version: await deps.fetchRunningVersion(),
+        });
+    }
+    const metadata = normalizeRuntimeIdentity(await deps.fetchRunningRuntimeMetadata());
+    if (isKnownVersion(metadata.version))
+        return metadata;
+    return normalizeRuntimeIdentity({
+        ...metadata,
+        version: await deps.fetchRunningVersion(),
+    });
+}
+function collectRuntimeDriftReasons(local, running) {
+    const reasons = [];
+    const comparableVersions = isKnownVersion(local.version) && isKnownVersion(running.version);
+    if (comparableVersions && local.version !== running.version) {
+        reasons.push({ key: "version", label: "version", local: local.version, running: running.version });
+    }
+    if (comparableVersions && isKnownRuntimeValue(local.lastUpdated) && isKnownRuntimeValue(running.lastUpdated) && local.lastUpdated !== running.lastUpdated) {
+        reasons.push({ key: "lastUpdated", label: "last updated", local: local.lastUpdated, running: running.lastUpdated });
+    }
+    if (isKnownRuntimeValue(local.repoRoot) && isKnownRuntimeValue(running.repoRoot) && local.repoRoot !== running.repoRoot) {
+        reasons.push({ key: "repoRoot", label: "code path", local: local.repoRoot, running: running.repoRoot });
+    }
+    if (isKnownRuntimeValue(local.configFingerprint)
+        && isKnownRuntimeValue(running.configFingerprint)
+        && local.configFingerprint !== running.configFingerprint) {
+        reasons.push({
+            key: "configFingerprint",
+            label: "config fingerprint",
+            local: local.configFingerprint,
+            running: running.configFingerprint,
+        });
+    }
+    if (isKnownRuntimeValue(local.managedAgents)
+        && isKnownRuntimeValue(running.managedAgents)
+        && local.managedAgents !== running.managedAgents) {
+        reasons.push({
+            key: "managedAgents",
+            label: "managed agents",
+            local: local.managedAgents,
+            running: running.managedAgents,
+        });
+    }
+    return reasons;
+}
+function formatRuntimeDriftPublicSummary(reasons) {
+    return reasons.map((reason) => reason.label).join(", ");
+}
 async function ensureCurrentDaemonRuntime(deps) {
+    deps.onProgress?.("checking whether an older background service is already running");
+    const localRuntime = normalizeRuntimeIdentity({
+        version: deps.localVersion,
+        lastUpdated: deps.localLastUpdated,
+        repoRoot: deps.localRepoRoot,
+        configFingerprint: deps.localConfigFingerprint,
+        managedAgents: deps.localManagedAgents,
+    });
     try {
-        const runningVersion = await deps.fetchRunningVersion();
+        const runningRuntime = await readRunningRuntimeIdentity(deps);
+        const runningVersion = runningRuntime.version;
+        const driftReasons = collectRuntimeDriftReasons(localRuntime, runningRuntime);
         let result;
-        if (isKnownVersion(deps.localVersion) &&
-            isKnownVersion(runningVersion) &&
-            runningVersion !== deps.localVersion) {
+        if (driftReasons.length > 0) {
+            const includesVersionDrift = driftReasons.some((entry) => entry.key === "version");
+            const publicDriftSummary = formatRuntimeDriftPublicSummary(driftReasons);
             try {
+                deps.onProgress?.("stopping the older background service");
                 await deps.stopDaemon();
             }
             catch (error) {
                 const reason = formatErrorReason(error);
                 result = {
                     alreadyRunning: true,
-                    message: `daemon already running (${deps.socketPath}; could not replace stale daemon ${runningVersion} -> ${deps.localVersion}: ${reason})`,
+                    message: includesVersionDrift
+                        ? `daemon already running (${deps.socketPath}; could not replace the older background service ${runningVersion} -> ${deps.localVersion}: ${reason})`
+                        : `daemon already running (${deps.socketPath}; could not replace the older background service after runtime drift ${publicDriftSummary}: ${reason})`,
                 };
                 (0, runtime_1.emitNervesEvent)({
                     level: "warn",
                     component: "daemon",
                     event: "daemon.runtime_sync_decision",
                     message: "evaluated daemon runtime sync outcome",
-                    meta: { socketPath: deps.socketPath, localVersion: deps.localVersion, runningVersion, action: "stale_replace_failed", reason },
+                    meta: {
+                        socketPath: deps.socketPath,
+                        localVersion: deps.localVersion,
+                        localLastUpdated: localRuntime.lastUpdated,
+                        localRepoRoot: localRuntime.repoRoot,
+                        localConfigFingerprint: localRuntime.configFingerprint,
+                        localManagedAgents: localRuntime.managedAgents,
+                        runningVersion,
+                        runningLastUpdated: runningRuntime.lastUpdated,
+                        runningRepoRoot: runningRuntime.repoRoot,
+                        runningConfigFingerprint: runningRuntime.configFingerprint,
+                        runningManagedAgents: runningRuntime.managedAgents,
+                        action: "stale_replace_failed",
+                        driftKeys: driftReasons.map((entry) => entry.key),
+                        reason,
+                    },
                 });
                 return result;
             }
             deps.cleanupStaleSocket(deps.socketPath);
+            deps.onProgress?.("starting the replacement background service");
             const started = await deps.startDaemonProcess(deps.socketPath);
+            const pid = started.pid ?? "unknown";
+            const verified = await verifyDaemonStarted(deps);
+            /* v8 ignore next -- daemon liveness failure: requires real daemon crash timing @preserve */
+            const suffix = verified ? "" : "\nreplacement daemon did not answer in time; check logs with `ouro logs` or run `ouro doctor`.";
             result = {
                 alreadyRunning: false,
-                message: `restarted stale daemon from ${runningVersion} to ${deps.localVersion} (pid ${started.pid ?? "unknown"})`,
+                message: includesVersionDrift
+                    ? `replaced an older background service ${runningVersion} -> ${deps.localVersion} (pid ${pid})${suffix}`
+                    : `replaced an older background service after runtime drift: ${publicDriftSummary} (pid ${pid})${suffix}`,
+                verifyStartupStatus: verified,
+                startedPid: started.pid ?? null,
             };
             (0, runtime_1.emitNervesEvent)({
                 component: "daemon",
                 event: "daemon.runtime_sync_decision",
                 message: "evaluated daemon runtime sync outcome",
-                meta: { socketPath: deps.socketPath, localVersion: deps.localVersion, runningVersion, action: "stale_restarted", pid: started.pid ?? null },
+                meta: {
+                    socketPath: deps.socketPath,
+                    localVersion: deps.localVersion,
+                    localLastUpdated: localRuntime.lastUpdated,
+                    localRepoRoot: localRuntime.repoRoot,
+                    localConfigFingerprint: localRuntime.configFingerprint,
+                    localManagedAgents: localRuntime.managedAgents,
+                    runningVersion,
+                    runningLastUpdated: runningRuntime.lastUpdated,
+                    runningRepoRoot: runningRuntime.repoRoot,
+                    runningConfigFingerprint: runningRuntime.configFingerprint,
+                    runningManagedAgents: runningRuntime.managedAgents,
+                    action: "stale_restarted",
+                    driftKeys: driftReasons.map((entry) => entry.key),
+                    pid: started.pid ?? null,
+                },
             });
             return result;
         }
-        if (!isKnownVersion(deps.localVersion) || !isKnownVersion(runningVersion)) {
+        if (!isKnownVersion(localRuntime.version) || !isKnownVersion(runningVersion)) {
             result = {
                 alreadyRunning: true,
                 message: `daemon already running (${deps.socketPath}; unable to verify version)`,
@@ -56,7 +187,20 @@ async function ensureCurrentDaemonRuntime(deps) {
                 component: "daemon",
                 event: "daemon.runtime_sync_decision",
                 message: "evaluated daemon runtime sync outcome",
-                meta: { socketPath: deps.socketPath, localVersion: deps.localVersion, runningVersion, action: "unknown_version" },
+                meta: {
+                    socketPath: deps.socketPath,
+                    localVersion: deps.localVersion,
+                    localLastUpdated: localRuntime.lastUpdated,
+                    localRepoRoot: localRuntime.repoRoot,
+                    localConfigFingerprint: localRuntime.configFingerprint,
+                    localManagedAgents: localRuntime.managedAgents,
+                    runningVersion,
+                    runningLastUpdated: runningRuntime.lastUpdated,
+                    runningRepoRoot: runningRuntime.repoRoot,
+                    runningConfigFingerprint: runningRuntime.configFingerprint,
+                    runningManagedAgents: runningRuntime.managedAgents,
+                    action: "unknown_version",
+                },
             });
             return result;
         }
@@ -72,19 +216,38 @@ async function ensureCurrentDaemonRuntime(deps) {
             component: "daemon",
             event: "daemon.runtime_sync_decision",
             message: "evaluated daemon runtime sync outcome",
-            meta: { socketPath: deps.socketPath, localVersion: deps.localVersion, action: "status_lookup_failed", reason },
+            meta: {
+                socketPath: deps.socketPath,
+                localVersion: deps.localVersion,
+                localLastUpdated: localRuntime.lastUpdated,
+                localRepoRoot: localRuntime.repoRoot,
+                localConfigFingerprint: localRuntime.configFingerprint,
+                localManagedAgents: localRuntime.managedAgents,
+                action: "status_lookup_failed",
+                reason,
+            },
         });
         return result;
     }
     const result = {
         alreadyRunning: true,
         message: `daemon already running (${deps.socketPath})`,
+        verifyStartupStatus: true,
+        startedPid: null,
     };
     (0, runtime_1.emitNervesEvent)({
         component: "daemon",
         event: "daemon.runtime_sync_decision",
         message: "evaluated daemon runtime sync outcome",
-        meta: { socketPath: deps.socketPath, localVersion: deps.localVersion, action: "already_current" },
+        meta: {
+            socketPath: deps.socketPath,
+            localVersion: deps.localVersion,
+            localLastUpdated: localRuntime.lastUpdated,
+            localRepoRoot: localRuntime.repoRoot,
+            localConfigFingerprint: localRuntime.configFingerprint,
+            localManagedAgents: localRuntime.managedAgents,
+            action: "already_current",
+        },
     });
     return result;
 }

package/dist/heart/daemon/daemon-tombstone.js

@@ -0,0 +1,236 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RECENT_CRASHES_MAX = void 0;
+exports.getTombstonePath = getTombstonePath;
+exports.setTombstonePath = setTombstonePath;
+exports.captureDeathForensics = captureDeathForensics;
+exports.writeDaemonTombstone = writeDaemonTombstone;
+exports.readDaemonTombstone = readDaemonTombstone;
+const child_process_1 = require("child_process");
+const fs = __importStar(require("fs"));
+const os = __importStar(require("os"));
+const path = __importStar(require("path"));
+const runtime_1 = require("../../nerves/runtime");
+/** Maximum number of historical crash timestamps to retain in the tombstone. */
+exports.RECENT_CRASHES_MAX = 100;
+let _tombstonePath = null;
+function getTombstonePath() {
+    if (!_tombstonePath) {
+        _tombstonePath = path.join(os.homedir(), ".ouro-cli", "daemon-death.json");
+    }
+    return _tombstonePath;
+}
+/** Overrides the tombstone path for testing. Pass null to reset. */
+function setTombstonePath(p) {
+    _tombstonePath = p;
+}
+/* v8 ignore start -- shells out to ps; covered via injected runPs in unit tests @preserve */
+function defaultRunPs(args) {
+    try {
+        return (0, child_process_1.execSync)(`ps ${args.join(" ")}`, { encoding: "utf-8", timeout: 2000 });
+    }
+    catch {
+        return null;
+    }
+}
+/* v8 ignore stop */
+const DEFAULT_FORENSICS_DEPS = {
+    /* v8 ignore next -- defensive: process.ppid always exists in node @preserve */
+    ppid: () => (typeof process !== "undefined" && typeof process.ppid === "number" ? process.ppid : 0),
+    runPs: defaultRunPs,
+};
+/**
+ * Capture forensic data about who is likely to have killed the daemon.
+ * Synchronous and best-effort: never throws, always returns a structured
+ * record (with nulls if anything failed). Called from inside SIGTERM/SIGINT
+ * handlers, so it must be fast and have no async dependencies.
+ */
+function captureDeathForensics(deps = DEFAULT_FORENSICS_DEPS) {
+    const parentPid = (() => {
+        try {
+            const p = deps.ppid();
+            return typeof p === "number" && p > 0 ? p : null;
+        }
+        catch {
+            return null;
+        }
+    })();
+    let parentCommand = null;
+    if (parentPid !== null) {
+        const psParent = deps.runPs(["-p", String(parentPid), "-o", "command="]);
+        if (psParent !== null) {
+            const trimmed = psParent.trim();
+            parentCommand = trimmed.length > 0 ? trimmed : null;
+        }
+    }
+    const processSnapshot = (() => {
+        const psAll = deps.runPs(["-eo", "pid,ppid,command"]);
+        if (psAll === null)
+            return null;
+        const lines = psAll.split("\n");
+        // Filter for relevant processes only — node, vitest, ouro, kill commands.
+        // Keeps the snapshot small and human-scannable.
+        const relevant = lines.filter((line) => {
+            const lower = line.toLowerCase();
+            return (lower.includes("node")
+                || lower.includes("vitest")
+                || lower.includes("ouro")
+                || lower.includes("/kill ")
+                || lower.includes("pkill")
+                || lower.includes("killall"));
+        });
+        if (relevant.length === 0)
+            return null;
+        return relevant.join("\n");
+    })();
+    const killerHint = deriveKillerHint(parentCommand, processSnapshot);
+    return { parentPid, parentCommand, processSnapshot, killerHint };
+}
+function deriveKillerHint(parentCommand, snapshot) {
+    if (parentCommand !== null && parentCommand.toLowerCase().includes("launchd")) {
+        return "process was reparented to launchd — likely killed by launchctl bootout, KeepAlive thrash, or RSS pressure";
+    }
+    if (snapshot !== null) {
+        const lower = snapshot.toLowerCase();
+        if (lower.includes("vitest")) {
+            return "vitest worker is running — possible test cleanup killing detached processes";
+        }
+        if (lower.includes("pkill") || lower.includes("killall")) {
+            return "saw pkill/killall in process list — explicit kill command";
+        }
+    }
+    return null;
+}
+function writeDaemonTombstone(reason, error, forensicsDeps) {
+    const now = new Date().toISOString();
+    const filePath = getTombstonePath();
+    // Read existing recentCrashes from previous tombstone (best-effort)
+    let existingCrashes = [];
+    try {
+        const raw = fs.readFileSync(filePath, "utf-8");
+        const existing = JSON.parse(raw);
+        if (Array.isArray(existing.recentCrashes)) {
+            existingCrashes = existing.recentCrashes.filter((entry) => typeof entry === "string");
+        }
+    }
+    catch {
+        // No existing tombstone or unreadable — start fresh
+    }
+    // Append the new crash and cap at the most recent RECENT_CRASHES_MAX entries
+    // so the tombstone doesn't grow without bound (we saw it hit 12,265 entries
+    // after a March 31 crash-restart thrash loop).
+    const recentCrashes = [...existingCrashes, now].slice(-exports.RECENT_CRASHES_MAX);
+    // Forensics: only meaningful for signal-driven deaths (sigterm/sigint).
+    // For uncaughtException etc. we skip the snapshot to keep tombstone writes
+    // fast on the unhappy path.
+    const shouldCaptureForensics = reason === "sigterm" || reason === "sigint";
+    const forensics = shouldCaptureForensics
+        ? captureDeathForensics(forensicsDeps)
+        : undefined;
+    const tombstone = {
+        reason,
+        message: error?.message ?? reason,
+        stack: error?.stack ?? null,
+        timestamp: now,
+        pid: process.pid,
+        uptimeSeconds: Math.floor(process.uptime()),
+        recentCrashes,
+        ...(forensics ? { forensics } : {}),
+    };
+    try {
+        fs.mkdirSync(path.dirname(filePath), { recursive: true });
+        fs.writeFileSync(filePath, JSON.stringify(tombstone, null, 2) + "\n", "utf-8");
+    }
+    catch {
+        // Best-effort: if we can't write, we're already dying.
+    }
+    (0, runtime_1.emitNervesEvent)({
+        level: "error",
+        component: "daemon",
+        event: "daemon.tombstone_written",
+        message: `daemon tombstone written: ${reason}`,
+        meta: {
+            reason,
+            errorMessage: error?.message ?? null,
+            filePath,
+            parentPid: forensics?.parentPid ?? null,
+            killerHint: forensics?.killerHint ?? null,
+        },
+    });
+}
+function readDaemonTombstone() {
+    const filePath = getTombstonePath();
+    try {
+        const raw = fs.readFileSync(filePath, "utf-8");
+        const parsed = JSON.parse(raw);
+        if (typeof parsed.reason !== "string" || typeof parsed.timestamp !== "string") {
+            return null;
+        }
+        (0, runtime_1.emitNervesEvent)({
+            component: "daemon",
+            event: "daemon.tombstone_read",
+            message: "read daemon tombstone",
+            meta: { filePath },
+        });
+        const forensics = parseForensics(parsed.forensics);
+        return {
+            reason: parsed.reason,
+            message: typeof parsed.message === "string" ? parsed.message : String(parsed.reason),
+            stack: typeof parsed.stack === "string" ? parsed.stack : null,
+            timestamp: parsed.timestamp,
+            pid: typeof parsed.pid === "number" ? parsed.pid : 0,
+            uptimeSeconds: typeof parsed.uptimeSeconds === "number" ? parsed.uptimeSeconds : 0,
+            recentCrashes: Array.isArray(parsed.recentCrashes)
+                ? parsed.recentCrashes.filter((e) => typeof e === "string")
+                : [],
+            ...(forensics ? { forensics } : {}),
+        };
+    }
+    catch {
+        return null;
+    }
+}
+function parseForensics(value) {
+    if (value === null || typeof value !== "object")
+        return null;
+    const v = value;
+    return {
+        parentPid: typeof v.parentPid === "number" ? v.parentPid : null,
+        parentCommand: typeof v.parentCommand === "string" ? v.parentCommand : null,
+        processSnapshot: typeof v.processSnapshot === "string" ? v.processSnapshot : null,
+        killerHint: typeof v.killerHint === "string" ? v.killerHint : null,
+    };
+}