@ouro.bot/cli 0.1.0-alpha.60 → 0.1.0-alpha.600

package/dist/heart/daemon/daemon-entry.js

@@ -39,14 +39,29 @@ const path = __importStar(require("path"));
 const process_manager_1 = require("./process-manager");
 const daemon_1 = require("./daemon");
 const runtime_1 = require("../../nerves/runtime");
+const index_1 = require("../../nerves/index");
 const message_router_1 = require("./message-router");
 const health_monitor_1 = require("./health-monitor");
+const daemon_health_1 = require("./daemon-health");
+const daemon_rollup_1 = require("./daemon-rollup");
 const task_scheduler_1 = require("./task-scheduler");
 const runtime_logging_1 = require("./runtime-logging");
 const sense_manager_1 = require("./sense-manager");
 const agent_discovery_1 = require("./agent-discovery");
 const identity_1 = require("../identity");
 const runtime_mode_1 = require("./runtime-mode");
+const habit_scheduler_1 = require("../habits/habit-scheduler");
+const habit_migration_1 = require("../habits/habit-migration");
+const await_scheduler_1 = require("../awaiting/await-scheduler");
+const await_expiry_1 = require("../awaiting/await-expiry");
+const os_cron_deps_1 = require("./os-cron-deps");
+const os_cron_1 = require("./os-cron");
+const daemon_tombstone_1 = require("./daemon-tombstone");
+const agent_config_check_1 = require("./agent-config-check");
+const pulse_1 = require("./pulse");
+const socket_client_1 = require("./socket-client");
+const bundle_manifest_1 = require("../../mind/bundle-manifest");
+const mcp_canary_1 = require("./mcp-canary");
 function parseSocketPath(argv) {
     const socketIndex = argv.indexOf("--socket");
     if (socketIndex >= 0) {
@@ -66,6 +81,16 @@ const mode = (0, runtime_mode_1.detectRuntimeMode)((0, identity_1.getRepoRoot)()
     message: "starting daemon entrypoint",
     meta: { socketPath, entryPath, mode },
 });
+/* v8 ignore next -- dev-mode indicator: false branch (production) tested in daemon-boot-updates.test.ts @preserve */
+if (mode === "dev") {
+    const repoRoot = (0, identity_1.getRepoRoot)();
+    (0, runtime_1.emitNervesEvent)({
+        component: "daemon",
+        event: "daemon.dev_mode_indicator",
+        message: `[dev] running from ${repoRoot}`,
+        meta: { repoRoot },
+    });
+}
 const managedAgents = (0, agent_discovery_1.listEnabledBundleAgents)();
 const processManager = new process_manager_1.DaemonProcessManager({
     agents: managedAgents.map((agent) => ({
@@ -75,6 +100,28 @@ const processManager = new process_manager_1.DaemonProcessManager({
         autoStart: true,
     })),
     existsSync: fs.existsSync,
+    /* v8 ignore next 4 -- wiring: delegates to checkAgentConfigWithProviderHealth which has full unit tests @preserve */
+    configCheck: async (agent) => {
+        const bundlesRoot = (0, identity_1.getAgentBundlesRoot)();
+        return (0, agent_config_check_1.checkAgentConfigWithProviderHealth)(agent, bundlesRoot);
+    },
+    /* v8 ignore start -- pulse flush wiring: integration code; flushPulse itself has full unit tests @preserve */
+    onSnapshotChange: () => {
+        (0, pulse_1.flushPulse)({
+            snapshots: processManager.listAgentSnapshots(),
+            bundlesRoot: (0, identity_1.getAgentBundlesRoot)(),
+            daemonVersion: (0, bundle_manifest_1.getPackageVersion)(),
+            now: new Date(),
+            // Default I/O wired into pulse.ts (writePulse, readPulse, etc.)
+            // Wake recipient: send inner.wake over the daemon's own socket so
+            // the recipient agent runs an inner-dialog turn that picks up the
+            // pulse alert. Catch errors silently — pulse is best-effort.
+            fireInnerWake: (agent) => {
+                (0, socket_client_1.sendDaemonCommand)(socketPath, { kind: "inner.wake", agent }).catch(() => { });
+            },
+        });
+    },
+    /* v8 ignore stop */
 });
 const scheduler = new task_scheduler_1.TaskDrivenScheduler({
     agents: [...managedAgents],
@@ -86,6 +133,23 @@ const senseManager = new sense_manager_1.DaemonSenseManager({
 const healthMonitor = new health_monitor_1.HealthMonitor({
     processManager,
     scheduler,
+    senseProbeProvider: () => [
+        ...senseManager.listHealthProbes(),
+        ...managedAgents.map((agent) => (0, mcp_canary_1.createMcpStatusCanaryProbe)({
+            agent,
+            socketPath,
+            command: process.execPath,
+            commandArgs: [
+                path.join(__dirname, "ouro-bot-entry.js"),
+                "mcp-serve",
+                "--agent",
+                agent,
+                "--socket",
+                socketPath,
+            ],
+            ignoreOverviewHealth: true,
+        })),
+    ],
     alertSink: (message) => {
         (0, runtime_1.emitNervesEvent)({
             level: "error",
@@ -95,7 +159,49 @@ const healthMonitor = new health_monitor_1.HealthMonitor({
             meta: { message },
         });
     },
+    /* v8 ignore next 3 -- wiring: delegates to processManager.restartAgent which has full unit tests @preserve */
+    onCriticalAgent: (agentName) => {
+        try {
+            processManager.restartAgent(agentName);
+        }
+        catch { /* recovery is best-effort */ }
+    },
+    /* v8 ignore next 3 -- wiring: delegates to senseManager.restartSense which has focused tests @preserve */
+    onCriticalSense: (managedName) => {
+        try {
+            void senseManager.restartSense(managedName);
+        }
+        catch { /* recovery is best-effort */ }
+    },
 });
+const habitSchedulers = [];
+const awaitSchedulers = [];
+let entryRuntimeStopping = false;
+let stopCommandExitScheduled = false;
+function stopEntryRuntime() {
+    if (entryRuntimeStopping)
+        return;
+    entryRuntimeStopping = true;
+    for (const s of habitSchedulers) {
+        s.stopWatching();
+        s.stop();
+    }
+    for (const s of awaitSchedulers) {
+        s.stopWatching();
+        s.stop();
+    }
+    healthMonitor.stopPeriodicChecks();
+}
+function scheduleCleanProcessExitAfterStopCommand() {
+    if (stopCommandExitScheduled)
+        return;
+    stopCommandExitScheduled = true;
+    // Account for the explicit daemon.stop path so the process exit catch-all
+    // does not mislabel an operator-requested stop as an unexpected clean exit.
+    _tombstoneWritten = true;
+    writeStopCommandHealthState();
+    setTimeout(() => process.exit(0), 100);
+}
 const daemon = new daemon_1.OuroDaemon({
     socketPath,
     processManager,
@@ -103,21 +209,398 @@ const daemon = new daemon_1.OuroDaemon({
     scheduler,
     healthMonitor,
     router,
+    mode,
+    onStopCommandComplete: () => {
+        stopEntryRuntime();
+        scheduleCleanProcessExitAfterStopCommand();
+    },
 });
-void daemon.start().catch(async () => {
+const daemonStartedAt = new Date().toISOString();
+const degradedComponents = [];
+function buildDaemonHealthState() {
+    const snapshots = processManager.listAgentSnapshots();
+    const agentDegradedComponents = snapshots
+        .filter((snapshot) => snapshot.status !== "running")
+        .map((snapshot) => {
+        const reasonParts = [
+            snapshot.errorReason ?? `${snapshot.channel} is ${snapshot.status}`,
+            snapshot.fixHint ? `Fix: ${snapshot.fixHint}` : null,
+        ].filter((part) => part !== null);
+        return {
+            component: `agent:${snapshot.name}`,
+            reason: reasonParts.join(" "),
+            since: snapshot.lastCrashAt ?? daemonStartedAt,
+        };
+    });
+    // Preserved for backwards-compatible inspection: callers (status
+    // command, mailbox surface, etc.) may still read this combined list
+    // for per-component reasons. The rollup status field above is what
+    // changed meaning — the array is still the union of bootstrap +
+    // agent-derived degradation entries.
+    const degraded = [
+        ...degradedComponents.map((entry) => ({ ...entry })),
+        ...agentDegradedComponents,
+    ];
+    // Layer 1 rollup: project per-agent snapshots into the minimal
+    // AgentRollupInput shape and let computeDaemonRollup decide. The
+    // input is "every enabled agent" — managedAgents was filtered via
+    // listEnabledBundleAgents at module init, and snapshots only covers
+    // agents the process manager was told to manage, so by construction
+    // these entries are all enabled. The rollup function is a pure
+    // declarative function on the data we hand it.
+    //
+    // Note: safe-mode is wired as `false` here. Existing crash-loop
+    // detection (safe-mode.ts) already runs at the daemon-up boot path
+    // (cli-exec.ts), not from inside the daemon process itself. Once
+    // the daemon is up and reaching this rollup, safe mode no longer
+    // applies — the daemon is by definition past the crash-loop gate.
+    // If a future PR moves safe-mode signal into the running daemon,
+    // wire it through this third argument.
+    const rollupStatus = (0, daemon_rollup_1.computeDaemonRollup)({
+        enabledAgents: snapshots.map((snapshot) => ({
+            name: snapshot.name,
+            status: snapshot.status,
+        })),
+        bootstrapDegraded: degradedComponents,
+        safeMode: false,
+    });
+    return {
+        status: rollupStatus,
+        mode,
+        pid: process.pid,
+        startedAt: daemonStartedAt,
+        uptimeSeconds: Math.floor(process.uptime()),
+        safeMode: null,
+        degraded,
+        agents: Object.fromEntries(snapshots.map((snapshot) => [
+            snapshot.name,
+            {
+                status: snapshot.status,
+                pid: snapshot.pid,
+                crashes: snapshot.restartCount,
+            },
+        ])),
+        habits: {},
+    };
+}
+function recordRecoverableBootstrapFailure(options) {
+    const errorMessage = options.error instanceof Error ? options.error.message : String(options.error);
+    const existing = degradedComponents.find((entry) => entry.component === options.component);
+    const reason = `${errorMessage}. ${options.guidance}`;
+    if (existing) {
+        existing.reason = reason;
+    }
+    else {
+        degradedComponents.push({
+            component: options.component,
+            reason,
+            since: new Date().toISOString(),
+        });
+    }
+    (0, runtime_1.emitNervesEvent)({
+        level: "warn",
+        component: "daemon",
+        event: "daemon.bootstrap_degraded",
+        message: "recoverable daemon bootstrap failure; daemon remains available in degraded mode",
+        meta: {
+            agent: options.agent,
+            component: options.component,
+            habitsDir: options.habitsDir,
+            error: errorMessage,
+            guidance: options.guidance,
+        },
+    });
+}
+function emitHabitSetupError(agent, error) {
+    const normalized = error instanceof Error ? error : new Error(String(error));
+    (0, runtime_1.emitNervesEvent)({
+        level: "error",
+        component: "daemon",
+        event: "daemon.habit_setup_error",
+        message: `habit setup failed for agent ${agent}`,
+        meta: { agent, error: normalized.message },
+    });
+}
+function emitAwaitSetupError(agent, error) {
+    const normalized = error instanceof Error ? error : new Error(String(error));
+    (0, runtime_1.emitNervesEvent)({
+        level: "error",
+        component: "daemon",
+        event: "daemon.await_setup_error",
+        message: `await setup failed for agent ${agent}`,
+        meta: { agent, error: normalized.message },
+    });
+}
+/* v8 ignore start — daemon health writer wiring, tested via daemon-health.test.ts @preserve */
+const healthWriter = new daemon_health_1.DaemonHealthWriter((0, daemon_health_1.getDefaultHealthPath)());
+const healthSink = (0, daemon_health_1.createHealthNervesSink)(healthWriter, buildDaemonHealthState);
+(0, index_1.registerGlobalLogSink)(healthSink);
+/* v8 ignore stop */
+function writeStopCommandHealthState() {
+    try {
+        healthWriter.writeHealth({
+            ...buildDaemonHealthState(),
+            status: "down",
+            uptimeSeconds: Math.floor(process.uptime()),
+        });
+    }
+    catch {
+        // Health writes are best-effort during shutdown.
+    }
+}
+/* v8 ignore start -- habit wiring: lambdas delegate to processManager/fs; tested via HabitScheduler unit tests @preserve */
+void daemon.start().then(() => {
+    const bundlesRoot = (0, identity_1.getAgentBundlesRoot)();
+    const ouroPath = (0, os_cron_deps_1.resolveOuroBinaryPath)();
+    const osCronDeps = (0, os_cron_deps_1.createRealOsCronDeps)();
+    for (const agent of managedAgents) {
+        const bundleRoot = path.join(bundlesRoot, `${agent}.ouro`);
+        const habitsDir = path.join(bundleRoot, "habits");
+        const degradedComponent = `habits:${agent}`;
+        try {
+            // Migrate old tasks/habits/ to habits/ at bundle root
+            (0, habit_migration_1.migrateHabitsFromTaskSystem)(bundleRoot);
+            const osCronManager = new os_cron_1.LaunchdCronManager(osCronDeps);
+            const scheduler = new habit_scheduler_1.HabitScheduler({
+                agent,
+                habitsDir,
+                osCronManager,
+                onHabitFire: (habitName) => {
+                    processManager.sendToAgent(agent, { type: "habit", habitName });
+                },
+                deps: {
+                    readdir: (dir) => fs.readdirSync(dir),
+                    readFile: (p, enc) => fs.readFileSync(p, enc),
+                    writeFile: (p, c, enc) => fs.writeFileSync(p, c, enc),
+                    existsSync: (p) => fs.existsSync(p),
+                    now: () => Date.now(),
+                    ouroPath,
+                    watch: (dir, cb) => fs.watch(dir, cb),
+                },
+            });
+            try {
+                scheduler.start();
+                scheduler.startPeriodicReconciliation();
+                scheduler.watchForChanges();
+                habitSchedulers.push(scheduler);
+            }
+            catch (error) {
+                try {
+                    scheduler.stopWatching();
+                    scheduler.stop();
+                }
+                catch {
+                    // Cleanup is best-effort for partially initialized schedulers.
+                }
+                emitHabitSetupError(agent, error);
+                recordRecoverableBootstrapFailure({
+                    agent,
+                    component: degradedComponent,
+                    habitsDir,
+                    error,
+                    guidance: `fix ${agent} habits or cron setup and rerun ouro up to restore habit automation`,
+                });
+            }
+        }
+        catch (err) {
+            const error = err instanceof Error ? err : new Error(String(err));
+            emitHabitSetupError(agent, error);
+            recordRecoverableBootstrapFailure({
+                agent,
+                component: degradedComponent,
+                habitsDir,
+                error,
+                guidance: `fix ${agent} habits or cron setup and rerun ouro up to restore habit automation`,
+            });
+        }
+        // Parallel await-condition scheduler. Uses its own OS cron manager so
+        // habits and awaits don't share label namespace and stale removals can't
+        // collide.
+        const awaitsDir = path.join(bundleRoot, "awaiting");
+        const awaitDegradedComponent = `awaits:${agent}`;
+        try {
+            const awaitOsCronManager = new os_cron_1.LaunchdCronManager(osCronDeps);
+            const awaitScheduler = new await_scheduler_1.AwaitScheduler({
+                agent,
+                awaitsDir,
+                osCronManager: awaitOsCronManager,
+                onAwaitFire: (awaitName) => {
+                    processManager.sendToAgent(agent, { type: "await", awaitName });
+                },
+                onAwaitExpire: (awaitName) => {
+                    void (0, await_expiry_1.archiveAndAlertExpiredAwait)({
+                        agentRoot: bundleRoot,
+                        agentName: agent,
+                        awaitName,
+                        deliveryDeps: {
+                            agentName: agent,
+                            queuePending: () => {
+                                // Best-effort: queue inner-dialog wake so the agent processes the alert path
+                                (0, socket_client_1.sendDaemonCommand)(socketPath, { kind: "inner.wake", agent }).catch(() => { });
+                            },
+                        },
+                    }).catch((err) => {
+                        (0, runtime_1.emitNervesEvent)({
+                            level: "error",
+                            component: "daemon",
+                            event: "daemon.await_expire_error",
+                            message: "await expiry handler threw",
+                            meta: { agent, awaitName, error: err instanceof Error ? err.message : String(err) },
+                        });
+                    });
+                },
+                deps: {
+                    readdir: (dir) => fs.readdirSync(dir),
+                    readFile: (p, enc) => fs.readFileSync(p, enc),
+                    existsSync: (p) => fs.existsSync(p),
+                    mkdir: (dir) => { fs.mkdirSync(dir, { recursive: true }); },
+                    now: () => Date.now(),
+                    ouroPath,
+                    watch: (dir, cb) => fs.watch(dir, cb),
+                },
+            });
+            try {
+                awaitScheduler.start();
+                awaitScheduler.startPeriodicReconciliation();
+                awaitScheduler.watchForChanges();
+                awaitSchedulers.push(awaitScheduler);
+            }
+            catch (error) {
+                try {
+                    awaitScheduler.stopWatching();
+                    awaitScheduler.stop();
+                }
+                catch {
+                    // best-effort cleanup
+                }
+                emitAwaitSetupError(agent, error);
+                recordRecoverableBootstrapFailure({
+                    agent,
+                    component: awaitDegradedComponent,
+                    habitsDir: awaitsDir,
+                    error,
+                    guidance: `fix ${agent} awaits or cron setup and rerun ouro up to restore await automation`,
+                });
+            }
+        }
+        catch (err) {
+            const error = err instanceof Error ? err : new Error(String(err));
+            emitAwaitSetupError(agent, error);
+            recordRecoverableBootstrapFailure({
+                agent,
+                component: awaitDegradedComponent,
+                habitsDir: awaitsDir,
+                error,
+                guidance: `fix ${agent} awaits or cron setup and rerun ouro up to restore await automation`,
+            });
+        }
+    }
+    healthMonitor.startPeriodicChecks(60_000);
+    /* v8 ignore start -- startup failure + signal handlers: call process.exit, untestable in vitest @preserve */
+}).catch(async (err) => {
+    const error = err instanceof Error ? err : new Error(String(err));
+    _tombstoneWritten = true;
+    (0, daemon_tombstone_1.writeDaemonTombstone)("startupFailure", error);
     (0, runtime_1.emitNervesEvent)({
         level: "error",
         component: "daemon",
         event: "daemon.entry_error",
         message: "daemon entrypoint failed",
-        meta: {},
+        meta: { error: error.message },
     });
+    setTimeout(() => process.exit(1), 5_000).unref();
     await daemon.stop();
     process.exit(1);
 });
 process.on("SIGINT", () => {
+    // ALWAYS write a tombstone, even on signal-driven shutdown. The previous
+    // behavior was to set _gracefulShutdown=true and skip the tombstone, which
+    // meant ANY external SIGINT/SIGTERM (launchd policy, OOM killer, manual
+    // kill, killOrphanProcesses from a sibling daemon) silently disappeared
+    // from the death log. The user lost weeks of visibility into why their
+    // daemon kept dying. Tombstones are informational — having a "sigint"
+    // tombstone is strictly better than silence.
+    _tombstoneWritten = true;
+    (0, daemon_tombstone_1.writeDaemonTombstone)("sigint", new Error("daemon received SIGINT"));
+    stopEntryRuntime();
+    setTimeout(() => process.exit(1), 5_000).unref();
     void daemon.stop().then(() => process.exit(0));
 });
 process.on("SIGTERM", () => {
+    _tombstoneWritten = true;
+    (0, daemon_tombstone_1.writeDaemonTombstone)("sigterm", new Error("daemon received SIGTERM"));
+    stopEntryRuntime();
+    setTimeout(() => process.exit(1), 5_000).unref();
     void daemon.stop().then(() => process.exit(0));
 });
+/* v8 ignore stop */
+// Suppress EPIPE on stdout/stderr — normal when detached daemon's parent exits
+/* v8 ignore start -- EPIPE suppression: only fires when parent process exits @preserve */
+process.stdout?.on?.("error", () => { });
+process.stderr?.on?.("error", () => { });
+/* v8 ignore stop */
+/* v8 ignore start -- global exception handlers: genuinely untestable in vitest; exercised by real daemon crashes @preserve */
+let _uncaughtCount = 0;
+let _tombstoneWritten = false;
+let _lastKnownCause = null;
+const CIRCUIT_BREAKER_WINDOW_MS = 60_000;
+const CIRCUIT_BREAKER_MAX = 10;
+process.on("uncaughtException", (error) => {
+    // EPIPE is normal for detached daemon processes — parent closed the pipe
+    if (error.code === "EPIPE")
+        return;
+    _uncaughtCount++;
+    _lastKnownCause = error;
+    setTimeout(() => { _uncaughtCount--; }, CIRCUIT_BREAKER_WINDOW_MS).unref();
+    _tombstoneWritten = true;
+    (0, daemon_tombstone_1.writeDaemonTombstone)("uncaughtException", error);
+    (0, runtime_1.emitNervesEvent)({
+        level: "error",
+        component: "daemon",
+        event: "daemon.uncaught_exception",
+        message: "uncaught exception in daemon process (continuing)",
+        meta: { error: error.message, stack: error.stack ?? null, uncaughtCount: _uncaughtCount },
+    });
+    // Circuit breaker: if too many exceptions in a short window, the process
+    // is in a bad state — exit so launchd/self-spawn can restart fresh.
+    if (_uncaughtCount >= CIRCUIT_BREAKER_MAX) {
+        (0, runtime_1.emitNervesEvent)({
+            level: "error",
+            component: "daemon",
+            event: "daemon.circuit_breaker_exit",
+            message: `daemon exiting: ${_uncaughtCount} uncaught exceptions in ${CIRCUIT_BREAKER_WINDOW_MS / 1000}s`,
+            meta: { uncaughtCount: _uncaughtCount },
+        });
+        setTimeout(() => process.exit(1), 5_000).unref();
+        void daemon.stop().then(() => process.exit(1));
+    }
+});
+process.on("unhandledRejection", (reason) => {
+    const error = reason instanceof Error ? reason : new Error(String(reason));
+    _lastKnownCause = error;
+    _tombstoneWritten = true;
+    (0, daemon_tombstone_1.writeDaemonTombstone)("unhandledRejection", error);
+    (0, runtime_1.emitNervesEvent)({
+        level: "error",
+        component: "daemon",
+        event: "daemon.unhandled_rejection",
+        message: "unhandled promise rejection in daemon process",
+        meta: { reason: error.message, stack: error.stack ?? null },
+    });
+});
+// Catch-all: write tombstone on any exit where we didn't already record the cause.
+// process.on('exit') is synchronous-only — writeDaemonTombstone uses writeFileSync, so it works.
+//
+// Previously this skipped writing if `_gracefulShutdown` was true, which made
+// SIGINT/SIGTERM-driven exits invisible in the death log. The signal handlers
+// above now always write their own tombstone before exiting, so this catch-all
+// only runs for exits the signal handlers didn't reach (e.g. process.exit
+// called from somewhere unexpected).
+process.on("exit", (code) => {
+    if (_tombstoneWritten)
+        return;
+    const reason = code === 0 ? "unexpectedCleanExit" : "unexpectedExit";
+    const error = _lastKnownCause ?? new Error(`daemon exited with code ${code} (no specific cause captured)`);
+    (0, daemon_tombstone_1.writeDaemonTombstone)(reason, error);
+});
+/* v8 ignore stop */