@ouro.bot/cli 0.1.0-alpha.45 → 0.1.0-alpha.450

package/dist/heart/daemon/daemon-entry.js

@@ -1,15 +1,65 @@
 #!/usr/bin/env node
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
 const process_manager_1 = require("./process-manager");
 const daemon_1 = require("./daemon");
 const runtime_1 = require("../../nerves/runtime");
+const index_1 = require("../../nerves/index");
 const message_router_1 = require("./message-router");
 const health_monitor_1 = require("./health-monitor");
+const daemon_health_1 = require("./daemon-health");
 const task_scheduler_1 = require("./task-scheduler");
 const runtime_logging_1 = require("./runtime-logging");
 const sense_manager_1 = require("./sense-manager");
 const agent_discovery_1 = require("./agent-discovery");
+const identity_1 = require("../identity");
+const runtime_mode_1 = require("./runtime-mode");
+const habit_scheduler_1 = require("../habits/habit-scheduler");
+const habit_migration_1 = require("../habits/habit-migration");
+const os_cron_deps_1 = require("./os-cron-deps");
+const os_cron_1 = require("./os-cron");
+const daemon_tombstone_1 = require("./daemon-tombstone");
+const agent_config_check_1 = require("./agent-config-check");
+const pulse_1 = require("./pulse");
+const socket_client_1 = require("./socket-client");
+const bundle_manifest_1 = require("../../mind/bundle-manifest");
+const http_health_probe_1 = require("./http-health-probe");
+const config_1 = require("../config");
 function parseSocketPath(argv) {
     const socketIndex = argv.indexOf("--socket");
     if (socketIndex >= 0) {
@@ -21,12 +71,24 @@ function parseSocketPath(argv) {
 }
 const socketPath = parseSocketPath(process.argv);
 (0, runtime_logging_1.configureDaemonRuntimeLogger)("daemon");
+const entryPath = path.resolve(__dirname, "daemon-entry.js");
+const mode = (0, runtime_mode_1.detectRuntimeMode)((0, identity_1.getRepoRoot)());
 (0, runtime_1.emitNervesEvent)({
     component: "daemon",
     event: "daemon.entry_start",
     message: "starting daemon entrypoint",
-    meta: { socketPath },
+    meta: { socketPath, entryPath, mode },
 });
+/* v8 ignore next -- dev-mode indicator: false branch (production) tested in daemon-boot-updates.test.ts @preserve */
+if (mode === "dev") {
+    const repoRoot = (0, identity_1.getRepoRoot)();
+    (0, runtime_1.emitNervesEvent)({
+        component: "daemon",
+        event: "daemon.dev_mode_indicator",
+        message: `[dev] running from ${repoRoot}`,
+        meta: { repoRoot },
+    });
+}
 const managedAgents = (0, agent_discovery_1.listEnabledBundleAgents)();
 const processManager = new process_manager_1.DaemonProcessManager({
     agents: managedAgents.map((agent) => ({
@@ -35,6 +97,29 @@ const processManager = new process_manager_1.DaemonProcessManager({
         channel: "inner-dialog",
         autoStart: true,
     })),
+    existsSync: fs.existsSync,
+    /* v8 ignore next 4 -- wiring: delegates to checkAgentConfigWithProviderHealth which has full unit tests @preserve */
+    configCheck: async (agent) => {
+        const bundlesRoot = (0, identity_1.getAgentBundlesRoot)();
+        return (0, agent_config_check_1.checkAgentConfigWithProviderHealth)(agent, bundlesRoot);
+    },
+    /* v8 ignore start -- pulse flush wiring: integration code; flushPulse itself has full unit tests @preserve */
+    onSnapshotChange: () => {
+        (0, pulse_1.flushPulse)({
+            snapshots: processManager.listAgentSnapshots(),
+            bundlesRoot: (0, identity_1.getAgentBundlesRoot)(),
+            daemonVersion: (0, bundle_manifest_1.getPackageVersion)(),
+            now: new Date(),
+            // Default I/O wired into pulse.ts (writePulse, readPulse, etc.)
+            // Wake recipient: send inner.wake over the daemon's own socket so
+            // the recipient agent runs an inner-dialog turn that picks up the
+            // pulse alert. Catch errors silently — pulse is best-effort.
+            fireInnerWake: (agent) => {
+                (0, socket_client_1.sendDaemonCommand)(socketPath, { kind: "inner.wake", agent }).catch(() => { });
+            },
+        });
+    },
+    /* v8 ignore stop */
 });
 const scheduler = new task_scheduler_1.TaskDrivenScheduler({
     agents: [...managedAgents],
@@ -43,9 +128,19 @@ const router = new message_router_1.FileMessageRouter();
 const senseManager = new sense_manager_1.DaemonSenseManager({
     agents: [...managedAgents],
 });
+/* v8 ignore next 5 -- entry-point wiring: probe factory and HealthMonitor both have full unit tests @preserve */
+let bbPort = 18790;
+try {
+    bbPort = (0, config_1.getBlueBubblesChannelConfig)().port;
+}
+catch {
+    // Daemon runs without --agent; agent-scoped config may not be available.
+}
+const bbProbe = (0, http_health_probe_1.createHttpHealthProbe)("bluebubbles", bbPort);
 const healthMonitor = new health_monitor_1.HealthMonitor({
     processManager,
     scheduler,
+    senseProbes: [bbProbe],
     alertSink: (message) => {
         (0, runtime_1.emitNervesEvent)({
             level: "error",
@@ -55,6 +150,13 @@ const healthMonitor = new health_monitor_1.HealthMonitor({
             meta: { message },
         });
     },
+    /* v8 ignore next 3 -- wiring: delegates to processManager.restartAgent which has full unit tests @preserve */
+    onCriticalAgent: (agentName) => {
+        try {
+            processManager.restartAgent(agentName);
+        }
+        catch { /* recovery is best-effort */ }
+    },
 });
 const daemon = new daemon_1.OuroDaemon({
     socketPath,
@@ -63,21 +165,272 @@ const daemon = new daemon_1.OuroDaemon({
     scheduler,
     healthMonitor,
     router,
+    mode,
 });
-void daemon.start().catch(async () => {
+const daemonStartedAt = new Date().toISOString();
+const degradedComponents = [];
+function buildDaemonHealthState() {
+    const snapshots = processManager.listAgentSnapshots();
+    const agentDegradedComponents = snapshots
+        .filter((snapshot) => snapshot.status !== "running")
+        .map((snapshot) => {
+        const reasonParts = [
+            snapshot.errorReason ?? `${snapshot.channel} is ${snapshot.status}`,
+            snapshot.fixHint ? `Fix: ${snapshot.fixHint}` : null,
+        ].filter((part) => part !== null);
+        return {
+            component: `agent:${snapshot.name}`,
+            reason: reasonParts.join(" "),
+            since: snapshot.lastCrashAt ?? daemonStartedAt,
+        };
+    });
+    const degraded = [
+        ...degradedComponents.map((entry) => ({ ...entry })),
+        ...agentDegradedComponents,
+    ];
+    return {
+        status: degraded.length > 0 ? "degraded" : "ok",
+        mode,
+        pid: process.pid,
+        startedAt: daemonStartedAt,
+        uptimeSeconds: Math.floor(process.uptime()),
+        safeMode: null,
+        degraded,
+        agents: Object.fromEntries(snapshots.map((snapshot) => [
+            snapshot.name,
+            {
+                status: snapshot.status,
+                pid: snapshot.pid,
+                crashes: snapshot.restartCount,
+            },
+        ])),
+        habits: {},
+    };
+}
+function recordRecoverableBootstrapFailure(options) {
+    const errorMessage = options.error instanceof Error ? options.error.message : String(options.error);
+    const existing = degradedComponents.find((entry) => entry.component === options.component);
+    const reason = `${errorMessage}. ${options.guidance}`;
+    if (existing) {
+        existing.reason = reason;
+    }
+    else {
+        degradedComponents.push({
+            component: options.component,
+            reason,
+            since: new Date().toISOString(),
+        });
+    }
+    (0, runtime_1.emitNervesEvent)({
+        level: "warn",
+        component: "daemon",
+        event: "daemon.bootstrap_degraded",
+        message: "recoverable daemon bootstrap failure; daemon remains available in degraded mode",
+        meta: {
+            agent: options.agent,
+            component: options.component,
+            habitsDir: options.habitsDir,
+            error: errorMessage,
+            guidance: options.guidance,
+        },
+    });
+}
+function emitHabitSetupError(agent, error) {
+    const normalized = error instanceof Error ? error : new Error(String(error));
+    (0, runtime_1.emitNervesEvent)({
+        level: "error",
+        component: "daemon",
+        event: "daemon.habit_setup_error",
+        message: `habit setup failed for agent ${agent}`,
+        meta: { agent, error: normalized.message },
+    });
+}
+/* v8 ignore start — daemon health writer wiring, tested via daemon-health.test.ts @preserve */
+const healthWriter = new daemon_health_1.DaemonHealthWriter((0, daemon_health_1.getDefaultHealthPath)());
+const healthSink = (0, daemon_health_1.createHealthNervesSink)(healthWriter, buildDaemonHealthState);
+(0, index_1.registerGlobalLogSink)(healthSink);
+/* v8 ignore stop */
+const habitSchedulers = [];
+/* v8 ignore start -- habit wiring: lambdas delegate to processManager/fs; tested via HabitScheduler unit tests @preserve */
+void daemon.start().then(() => {
+    const bundlesRoot = (0, identity_1.getAgentBundlesRoot)();
+    const ouroPath = (0, os_cron_deps_1.resolveOuroBinaryPath)();
+    const osCronDeps = (0, os_cron_deps_1.createRealOsCronDeps)();
+    for (const agent of managedAgents) {
+        const bundleRoot = path.join(bundlesRoot, `${agent}.ouro`);
+        const habitsDir = path.join(bundleRoot, "habits");
+        const degradedComponent = `habits:${agent}`;
+        try {
+            // Migrate old tasks/habits/ to habits/ at bundle root
+            (0, habit_migration_1.migrateHabitsFromTaskSystem)(bundleRoot);
+            const osCronManager = new os_cron_1.LaunchdCronManager(osCronDeps);
+            const scheduler = new habit_scheduler_1.HabitScheduler({
+                agent,
+                habitsDir,
+                osCronManager,
+                onHabitFire: (habitName) => {
+                    processManager.sendToAgent(agent, { type: "habit", habitName });
+                },
+                deps: {
+                    readdir: (dir) => fs.readdirSync(dir),
+                    readFile: (p, enc) => fs.readFileSync(p, enc),
+                    writeFile: (p, c, enc) => fs.writeFileSync(p, c, enc),
+                    existsSync: (p) => fs.existsSync(p),
+                    now: () => Date.now(),
+                    ouroPath,
+                    watch: (dir, cb) => fs.watch(dir, cb),
+                },
+            });
+            try {
+                scheduler.start();
+                scheduler.startPeriodicReconciliation();
+                scheduler.watchForChanges();
+                habitSchedulers.push(scheduler);
+            }
+            catch (error) {
+                try {
+                    scheduler.stopWatching();
+                    scheduler.stop();
+                }
+                catch {
+                    // Cleanup is best-effort for partially initialized schedulers.
+                }
+                emitHabitSetupError(agent, error);
+                recordRecoverableBootstrapFailure({
+                    agent,
+                    component: degradedComponent,
+                    habitsDir,
+                    error,
+                    guidance: `fix ${agent} habits or cron setup and rerun ouro up to restore habit automation`,
+                });
+            }
+        }
+        catch (err) {
+            const error = err instanceof Error ? err : new Error(String(err));
+            emitHabitSetupError(agent, error);
+            recordRecoverableBootstrapFailure({
+                agent,
+                component: degradedComponent,
+                habitsDir,
+                error,
+                guidance: `fix ${agent} habits or cron setup and rerun ouro up to restore habit automation`,
+            });
+        }
+    }
+    healthMonitor.startPeriodicChecks(60_000);
+    /* v8 ignore start -- startup failure + signal handlers: call process.exit, untestable in vitest @preserve */
+}).catch(async (err) => {
+    const error = err instanceof Error ? err : new Error(String(err));
+    _tombstoneWritten = true;
+    (0, daemon_tombstone_1.writeDaemonTombstone)("startupFailure", error);
     (0, runtime_1.emitNervesEvent)({
         level: "error",
         component: "daemon",
         event: "daemon.entry_error",
         message: "daemon entrypoint failed",
-        meta: {},
+        meta: { error: error.message },
     });
+    setTimeout(() => process.exit(1), 5_000).unref();
     await daemon.stop();
     process.exit(1);
 });
 process.on("SIGINT", () => {
+    // ALWAYS write a tombstone, even on signal-driven shutdown. The previous
+    // behavior was to set _gracefulShutdown=true and skip the tombstone, which
+    // meant ANY external SIGINT/SIGTERM (launchd policy, OOM killer, manual
+    // kill, killOrphanProcesses from a sibling daemon) silently disappeared
+    // from the death log. The user lost weeks of visibility into why their
+    // daemon kept dying. Tombstones are informational — having a "sigint"
+    // tombstone is strictly better than silence.
+    _tombstoneWritten = true;
+    (0, daemon_tombstone_1.writeDaemonTombstone)("sigint", new Error("daemon received SIGINT"));
+    for (const s of habitSchedulers) {
+        s.stopWatching();
+        s.stop();
+    }
+    healthMonitor.stopPeriodicChecks();
+    setTimeout(() => process.exit(1), 5_000).unref();
     void daemon.stop().then(() => process.exit(0));
 });
 process.on("SIGTERM", () => {
+    _tombstoneWritten = true;
+    (0, daemon_tombstone_1.writeDaemonTombstone)("sigterm", new Error("daemon received SIGTERM"));
+    for (const s of habitSchedulers) {
+        s.stopWatching();
+        s.stop();
+    }
+    healthMonitor.stopPeriodicChecks();
+    setTimeout(() => process.exit(1), 5_000).unref();
     void daemon.stop().then(() => process.exit(0));
 });
+/* v8 ignore stop */
+// Suppress EPIPE on stdout/stderr — normal when detached daemon's parent exits
+/* v8 ignore start -- EPIPE suppression: only fires when parent process exits @preserve */
+process.stdout?.on?.("error", () => { });
+process.stderr?.on?.("error", () => { });
+/* v8 ignore stop */
+/* v8 ignore start -- global exception handlers: genuinely untestable in vitest; exercised by real daemon crashes @preserve */
+let _uncaughtCount = 0;
+let _tombstoneWritten = false;
+let _lastKnownCause = null;
+const CIRCUIT_BREAKER_WINDOW_MS = 60_000;
+const CIRCUIT_BREAKER_MAX = 10;
+process.on("uncaughtException", (error) => {
+    // EPIPE is normal for detached daemon processes — parent closed the pipe
+    if (error.code === "EPIPE")
+        return;
+    _uncaughtCount++;
+    _lastKnownCause = error;
+    setTimeout(() => { _uncaughtCount--; }, CIRCUIT_BREAKER_WINDOW_MS).unref();
+    _tombstoneWritten = true;
+    (0, daemon_tombstone_1.writeDaemonTombstone)("uncaughtException", error);
+    (0, runtime_1.emitNervesEvent)({
+        level: "error",
+        component: "daemon",
+        event: "daemon.uncaught_exception",
+        message: "uncaught exception in daemon process (continuing)",
+        meta: { error: error.message, stack: error.stack ?? null, uncaughtCount: _uncaughtCount },
+    });
+    // Circuit breaker: if too many exceptions in a short window, the process
+    // is in a bad state — exit so launchd/self-spawn can restart fresh.
+    if (_uncaughtCount >= CIRCUIT_BREAKER_MAX) {
+        (0, runtime_1.emitNervesEvent)({
+            level: "error",
+            component: "daemon",
+            event: "daemon.circuit_breaker_exit",
+            message: `daemon exiting: ${_uncaughtCount} uncaught exceptions in ${CIRCUIT_BREAKER_WINDOW_MS / 1000}s`,
+            meta: { uncaughtCount: _uncaughtCount },
+        });
+        setTimeout(() => process.exit(1), 5_000).unref();
+        void daemon.stop().then(() => process.exit(1));
+    }
+});
+process.on("unhandledRejection", (reason) => {
+    const error = reason instanceof Error ? reason : new Error(String(reason));
+    _lastKnownCause = error;
+    _tombstoneWritten = true;
+    (0, daemon_tombstone_1.writeDaemonTombstone)("unhandledRejection", error);
+    (0, runtime_1.emitNervesEvent)({
+        level: "error",
+        component: "daemon",
+        event: "daemon.unhandled_rejection",
+        message: "unhandled promise rejection in daemon process",
+        meta: { reason: error.message, stack: error.stack ?? null },
+    });
+});
+// Catch-all: write tombstone on any exit where we didn't already record the cause.
+// process.on('exit') is synchronous-only — writeDaemonTombstone uses writeFileSync, so it works.
+//
+// Previously this skipped writing if `_gracefulShutdown` was true, which made
+// SIGINT/SIGTERM-driven exits invisible in the death log. The signal handlers
+// above now always write their own tombstone before exiting, so this catch-all
+// only runs for exits the signal handlers didn't reach (e.g. process.exit
+// called from somewhere unexpected).
+process.on("exit", (code) => {
+    if (_tombstoneWritten)
+        return;
+    const reason = code === 0 ? "unexpectedCleanExit" : "unexpectedExit";
+    const error = _lastKnownCause ?? new Error(`daemon exited with code ${code} (no specific cause captured)`);
+    (0, daemon_tombstone_1.writeDaemonTombstone)(reason, error);
+});
+/* v8 ignore stop */

package/dist/heart/daemon/daemon-health.js

@@ -0,0 +1,141 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HEALTH_TRACKED_EVENTS = exports.DaemonHealthWriter = void 0;
+exports.getDefaultHealthPath = getDefaultHealthPath;
+exports.createHealthNervesSink = createHealthNervesSink;
+exports.readHealth = readHealth;
+const fs = __importStar(require("fs"));
+const os = __importStar(require("os"));
+const path = __importStar(require("path"));
+const runtime_1 = require("../../nerves/runtime");
+class DaemonHealthWriter {
+    healthPath;
+    constructor(healthPath) {
+        this.healthPath = healthPath;
+    }
+    writeHealth(state) {
+        try {
+            const dir = path.dirname(this.healthPath);
+            fs.mkdirSync(dir, { recursive: true });
+            const tmpPath = `${this.healthPath}.tmp.${process.pid}`;
+            fs.writeFileSync(tmpPath, JSON.stringify(state, null, 2) + "\n", "utf-8");
+            fs.renameSync(tmpPath, this.healthPath);
+            (0, runtime_1.emitNervesEvent)({
+                component: "daemon",
+                event: "daemon.health_written",
+                message: "daemon health file written",
+                meta: { path: this.healthPath, status: state.status },
+            });
+        }
+        catch {
+            // Best-effort: if we can't write, don't crash the daemon.
+        }
+    }
+}
+exports.DaemonHealthWriter = DaemonHealthWriter;
+function getDefaultHealthPath() {
+    return path.join(os.homedir(), ".ouro-cli", "daemon-health.json");
+}
+/** Events that trigger a debounced health file write */
+exports.HEALTH_TRACKED_EVENTS = new Set([
+    "daemon.habit_cron_verification_failed",
+    "daemon.habit_fire",
+    "daemon.agent_exit",
+    "daemon.agent_started",
+    "daemon.agent_config_invalid",
+    "daemon.agent_config_failure",
+    "daemon.agent_entry_missing",
+    "daemon.agent_spawn_failed",
+    "daemon.agent_restart_exhausted",
+    "daemon.agent_permanent_failure",
+    "daemon.agent_cooldown_recovery",
+    "daemon.bootstrap_degraded",
+    "daemon.safe_mode_entered",
+    "daemon.habit_scheduler_start",
+]);
+/**
+ * Creates a nerves LogSink that triggers debounced health writes on relevant events.
+ * Components don't know about the health writer — they just emit events.
+ */
+function createHealthNervesSink(writer, getState) {
+    let debounceTimer = null;
+    return (entry) => {
+        if (!exports.HEALTH_TRACKED_EVENTS.has(entry.event)) {
+            return;
+        }
+        // Debounce: max once per second
+        if (debounceTimer !== null) {
+            clearTimeout(debounceTimer);
+        }
+        debounceTimer = setTimeout(() => {
+            debounceTimer = null;
+            const state = getState();
+            writer.writeHealth(state);
+        }, 1000);
+    };
+}
+function readHealth(healthPath) {
+    try {
+        const raw = fs.readFileSync(healthPath, "utf-8");
+        const parsed = JSON.parse(raw);
+        if (typeof parsed.status !== "string" ||
+            typeof parsed.mode !== "string" ||
+            typeof parsed.pid !== "number" ||
+            typeof parsed.startedAt !== "string" ||
+            typeof parsed.uptimeSeconds !== "number" ||
+            !Array.isArray(parsed.degraded) ||
+            typeof parsed.agents !== "object" ||
+            parsed.agents === null ||
+            typeof parsed.habits !== "object" ||
+            parsed.habits === null) {
+            return null;
+        }
+        return {
+            status: parsed.status,
+            mode: parsed.mode,
+            pid: parsed.pid,
+            startedAt: parsed.startedAt,
+            uptimeSeconds: parsed.uptimeSeconds,
+            safeMode: parsed.safeMode,
+            degraded: parsed.degraded,
+            agents: parsed.agents,
+            habits: parsed.habits,
+        };
+    }
+    catch {
+        return null;
+    }
+}