@ouro.bot/cli 0.1.0-alpha.148 → 0.1.0-alpha.149

package/changelog.json

@@ -1,6 +1,18 @@
 {
   "_note": "This changelog is maintained as part of the PR/version-bump workflow. Agent-curated, not auto-generated. Agents read this file directly via read_file to understand what changed between versions.",
   "versions": [
+    {
+      "version": "0.1.0-alpha.149",
+      "changes": [
+        "Daemon tombstone: writeDaemonTombstone persists crash context (reason, stack, uptime) to ~/.ouro-cli/daemon-death.json on uncaught exceptions and startup failures.",
+        "Global exception handlers: daemon-entry installs uncaughtException (tombstone + graceful 5s shutdown) and unhandledRejection (warn, no exit) handlers.",
+        "Log rotation: ndjson file sink rotates logs at 50MB (configurable), keeping .1 and .2 backups. Throttled stat checks via byte counter.",
+        "Socket connection error handling: connection.on('error') prevents server crash on client disconnect, connection.end() wrapped for EPIPE safety, persistent server.on('error') after listen.",
+        "Crash context in status: DaemonAgentSnapshot includes lastExitCode/lastSignal, Workers table shows last exit info, stopped-daemon status reads tombstone for 'Last death' line.",
+        "Cooldown recovery: after restart exhaustion, agents schedule recovery after cooldownRecoveryMs (default 5min). Reset crash history on recovery. maxCooldownRetries (default 3) before permanent give-up.",
+        "Defensive handleCommand: extracted handleCommandInner, outer handleCommand wraps with try/catch logging daemon.command_error with kind/error/stack before re-throw."
+      ]
+    },
     {
       "version": "0.1.0-alpha.148",
       "changes": [

package/dist/heart/daemon/daemon-cli.js

@@ -71,6 +71,7 @@ const thoughts_1 = require("./thoughts");
 const ouro_bot_global_installer_1 = require("./ouro-bot-global-installer");
 const launchd_1 = require("./launchd");
 const socket_client_1 = require("./socket-client");
+const daemon_tombstone_1 = require("./daemon-tombstone");
 const session_activity_1 = require("../session-activity");
 const auth_flow_1 = require("./auth-flow");
 function stringField(value) {
@@ -145,6 +146,8 @@ function parseStatusPayload(data) {
             status,
             pid,
             restartCount,
+            lastExitCode: numberField(row.lastExitCode) ?? null,
+            lastSignal: stringField(row.lastSignal) ?? null,
         };
     });
     if (parsedSenses.some((row) => row === null) || parsedWorkers.some((row) => row === null))
@@ -198,13 +201,23 @@ function formatDaemonStatusOutput(response, fallback) {
         row.status,
         row.detail,
     ]);
-    const workerRows = payload.workers.map((row) => [
-        row.agent,
-        row.worker,
-        row.status,
-        row.pid === null ? "n/a" : String(row.pid),
-        String(row.restartCount),
-    ]);
+    const workerRows = payload.workers.map((row) => {
+        /* v8 ignore start — exit info branches tested via daemon-crash-context; v8 misreports conditional chains @preserve */
+        let exitInfo = "n/a";
+        if (row.lastExitCode !== null)
+            exitInfo = `code=${row.lastExitCode}`;
+        if (row.lastSignal !== null)
+            exitInfo = row.lastExitCode !== null ? `code=${row.lastExitCode} sig=${row.lastSignal}` : `sig=${row.lastSignal}`;
+        /* v8 ignore stop */
+        return [
+            row.agent,
+            row.worker,
+            row.status,
+            row.pid === null ? "n/a" : String(row.pid),
+            String(row.restartCount),
+            exitInfo,
+        ];
+    });
     return [
         "Overview",
         formatTable(["Item", "Value"], overviewRows),
@@ -213,7 +226,7 @@ function formatDaemonStatusOutput(response, fallback) {
         formatTable(["Agent", "Sense", "Enabled", "State", "Detail"], senseRows),
         "",
         "Workers",
-        formatTable(["Agent", "Worker", "State", "PID", "Restarts"], workerRows),
+        formatTable(["Agent", "Worker", "State", "PID", "Restarts", "Last Exit"], workerRows),
     ].join("\n");
 }
 async function ensureDaemonRunning(deps) {
@@ -338,15 +351,28 @@ function buildStoppedStatusPayload(socketPath) {
     };
 }
 function daemonUnavailableStatusOutput(socketPath) {
-    return [
+    /* v8 ignore start — tombstone read tested in daemon-status-tombstone.test; branch misreported @preserve */
+    const tombstone = (0, daemon_tombstone_1.readDaemonTombstone)();
+    const deathLine = tombstone
+        ? `Last death: ${tombstone.timestamp} -- ${tombstone.reason}: ${tombstone.message}`
+        : null;
+    /* v8 ignore stop */
+    const lines = [
         formatDaemonStatusOutput({
             ok: true,
             summary: "daemon not running",
             data: buildStoppedStatusPayload(socketPath),
         }, "daemon not running"),
         "",
-        "daemon not running; run `ouro up`",
-    ].join("\n");
+    ];
+    /* v8 ignore start — tombstone presence requires real daemon crash @preserve */
+    if (deathLine) {
+        lines.push(deathLine);
+        lines.push("");
+        /* v8 ignore stop */
+    }
+    lines.push("daemon not running; run `ouro up`");
+    return lines.join("\n");
 }
 function isDaemonUnavailableError(error) {
     const code = typeof error === "object" && error !== null && "code" in error

package/dist/heart/daemon/daemon-entry.js

@@ -51,6 +51,7 @@ const habit_scheduler_1 = require("./habit-scheduler");
 const habit_migration_1 = require("./habit-migration");
 const os_cron_deps_1 = require("./os-cron-deps");
 const os_cron_1 = require("./os-cron");
+const daemon_tombstone_1 = require("./daemon-tombstone");
 function parseSocketPath(argv) {
     const socketIndex = argv.indexOf("--socket");
     if (socketIndex >= 0) {
@@ -152,14 +153,18 @@ void daemon.start().then(() => {
         scheduler.watchForChanges();
         habitSchedulers.push(scheduler);
     }
-}).catch(async () => {
+}).catch(async (err) => {
     /* v8 ignore stop */
+    /* v8 ignore start — instanceof branch defensive; catch always receives Error in practice @preserve */
+    const error = err instanceof Error ? err : new Error(String(err));
+    /* v8 ignore stop */
+    (0, daemon_tombstone_1.writeDaemonTombstone)("startupFailure", error);
     (0, runtime_1.emitNervesEvent)({
         level: "error",
         component: "daemon",
         event: "daemon.entry_error",
         message: "daemon entrypoint failed",
-        meta: {},
+        meta: { error: error.message },
     });
     await daemon.stop();
     process.exit(1);
@@ -178,3 +183,27 @@ process.on("SIGTERM", () => {
     }
     void daemon.stop().then(() => process.exit(0));
 });
+/* v8 ignore start -- global exception handlers: genuinely untestable in vitest; exercised by real daemon crashes @preserve */
+process.on("uncaughtException", (error) => {
+    (0, daemon_tombstone_1.writeDaemonTombstone)("uncaughtException", error);
+    (0, runtime_1.emitNervesEvent)({
+        level: "error",
+        component: "daemon",
+        event: "daemon.uncaught_exception",
+        message: "uncaught exception in daemon process",
+        meta: { error: error.message, stack: error.stack ?? null },
+    });
+    // Graceful 5-second shutdown window
+    setTimeout(() => process.exit(1), 5_000).unref();
+    void daemon.stop().then(() => process.exit(1));
+});
+process.on("unhandledRejection", (reason) => {
+    (0, runtime_1.emitNervesEvent)({
+        level: "warn",
+        component: "daemon",
+        event: "daemon.unhandled_rejection",
+        message: "unhandled promise rejection in daemon process",
+        meta: { reason: reason instanceof Error ? reason.message : String(reason) },
+    });
+});
+/* v8 ignore stop */

package/dist/heart/daemon/daemon-tombstone.js

@@ -0,0 +1,106 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getTombstonePath = getTombstonePath;
+exports.setTombstonePath = setTombstonePath;
+exports.writeDaemonTombstone = writeDaemonTombstone;
+exports.readDaemonTombstone = readDaemonTombstone;
+const fs = __importStar(require("fs"));
+const os = __importStar(require("os"));
+const path = __importStar(require("path"));
+const runtime_1 = require("../../nerves/runtime");
+let _tombstonePath = null;
+function getTombstonePath() {
+    if (!_tombstonePath) {
+        _tombstonePath = path.join(os.homedir(), ".ouro-cli", "daemon-death.json");
+    }
+    return _tombstonePath;
+}
+/** Overrides the tombstone path for testing. Pass null to reset. */
+function setTombstonePath(p) {
+    _tombstonePath = p;
+}
+function writeDaemonTombstone(reason, error) {
+    const tombstone = {
+        reason,
+        message: error?.message ?? reason,
+        stack: error?.stack ?? null,
+        timestamp: new Date().toISOString(),
+        pid: process.pid,
+        uptimeSeconds: Math.floor(process.uptime()),
+    };
+    const filePath = getTombstonePath();
+    try {
+        fs.mkdirSync(path.dirname(filePath), { recursive: true });
+        fs.writeFileSync(filePath, JSON.stringify(tombstone, null, 2) + "\n", "utf-8");
+    }
+    catch {
+        // Best-effort: if we can't write, we're already dying.
+    }
+    (0, runtime_1.emitNervesEvent)({
+        level: "error",
+        component: "daemon",
+        event: "daemon.tombstone_written",
+        message: `daemon tombstone written: ${reason}`,
+        meta: { reason, errorMessage: error?.message ?? null, filePath },
+    });
+}
+function readDaemonTombstone() {
+    const filePath = getTombstonePath();
+    try {
+        const raw = fs.readFileSync(filePath, "utf-8");
+        const parsed = JSON.parse(raw);
+        if (typeof parsed.reason !== "string" || typeof parsed.timestamp !== "string") {
+            return null;
+        }
+        (0, runtime_1.emitNervesEvent)({
+            component: "daemon",
+            event: "daemon.tombstone_read",
+            message: "read daemon tombstone",
+            meta: { filePath },
+        });
+        return {
+            reason: parsed.reason,
+            message: typeof parsed.message === "string" ? parsed.message : String(parsed.reason),
+            stack: typeof parsed.stack === "string" ? parsed.stack : null,
+            timestamp: parsed.timestamp,
+            pid: typeof parsed.pid === "number" ? parsed.pid : 0,
+            uptimeSeconds: typeof parsed.uptimeSeconds === "number" ? parsed.uptimeSeconds : 0,
+        };
+    }
+    catch {
+        return null;
+    }
+}

package/dist/heart/daemon/daemon.js

@@ -135,6 +135,8 @@ function buildWorkerRows(snapshots) {
         pid: snapshot.pid,
         restartCount: snapshot.restartCount,
         startedAt: snapshot.startedAt,
+        lastExitCode: snapshot.lastExitCode ?? null,
+        lastSignal: snapshot.lastSignal ?? null,
     }));
 }
 function formatStatusSummary(payload) {
@@ -269,12 +271,36 @@ class OuroDaemon {
         this.server = net.createServer((connection) => {
             let raw = "";
             let responded = false;
+            /* v8 ignore start — connection error handler requires real socket error @preserve */
+            connection.on("error", (err) => {
+                (0, runtime_1.emitNervesEvent)({
+                    level: "warn",
+                    component: "daemon",
+                    event: "daemon.connection_error",
+                    message: "socket connection error",
+                    meta: { error: err.message, code: err.code ?? null },
+                });
+            });
+            /* v8 ignore stop */
             const flushResponse = async () => {
                 if (responded)
                     return;
                 responded = true;
                 const response = await this.handleRawPayload(raw);
-                connection.end(response);
+                try {
+                    connection.end(response);
+                    /* v8 ignore start — EPIPE catch requires real socket disconnect @preserve */
+                }
+                catch (err) {
+                    (0, runtime_1.emitNervesEvent)({
+                        level: "warn",
+                        component: "daemon",
+                        event: "daemon.connection_end_error",
+                        message: "failed to send response to client (EPIPE)",
+                        meta: { error: err instanceof Error ? err.message : String(err) },
+                    });
+                }
+                /* v8 ignore stop */
             };
             connection.on("data", (chunk) => {
                 raw += chunk.toString("utf-8");
@@ -287,7 +313,22 @@ class OuroDaemon {
         const server = this.server;
         await new Promise((resolve, reject) => {
             server.once("error", reject);
-            server.listen(this.socketPath, () => resolve());
+            server.listen(this.socketPath, () => {
+                // Replace the one-time error listener with a persistent one after successful listen
+                server.removeAllListeners("error");
+                /* v8 ignore start — server error after listen requires real socket race condition @preserve */
+                server.on("error", (err) => {
+                    (0, runtime_1.emitNervesEvent)({
+                        level: "error",
+                        component: "daemon",
+                        event: "daemon.server_error",
+                        message: "daemon server error after listen",
+                        meta: { error: err.message, code: err.code ?? null },
+                    });
+                });
+                /* v8 ignore stop */
+                resolve();
+            });
         });
     }
     async drainPendingBundleMessages() {
@@ -467,6 +508,27 @@ class OuroDaemon {
             message: "handling daemon command",
             meta: { kind: command.kind },
         });
+        try {
+            return await this.handleCommandInner(command);
+            /* v8 ignore start — command error catch tested in daemon-command-error.test; instanceof branches defensive @preserve */
+        }
+        catch (error) {
+            (0, runtime_1.emitNervesEvent)({
+                level: "error",
+                component: "daemon",
+                event: "daemon.command_error",
+                message: "unexpected error handling daemon command",
+                meta: {
+                    kind: command.kind,
+                    error: error instanceof Error ? error.message : String(error),
+                    stack: error instanceof Error ? error.stack ?? null : null,
+                },
+            });
+            throw error;
+        }
+        /* v8 ignore stop */
+    }
+    async handleCommandInner(command) {
         switch (command.kind) {
             case "daemon.start":
                 await this.start();

package/dist/heart/daemon/process-manager.js

@@ -51,12 +51,16 @@ class DaemonProcessManager {
     now;
     setTimeoutFn;
     clearTimeoutFn;
+    cooldownRecoveryMs;
+    maxCooldownRetries;
     existsSyncFn;
     constructor(options) {
         this.maxRestartsPerHour = options.maxRestartsPerHour ?? 10;
         this.stabilityThresholdMs = options.stabilityThresholdMs ?? 60_000;
         this.initialBackoffMs = options.initialBackoffMs ?? 1_000;
         this.maxBackoffMs = options.maxBackoffMs ?? 60_000;
+        this.cooldownRecoveryMs = options.cooldownRecoveryMs ?? 5 * 60 * 1_000;
+        this.maxCooldownRetries = options.maxCooldownRetries ?? 3;
         this.spawnFn = options.spawn ?? ((command, args, spawnOptions) => (0, child_process_1.spawn)(command, args, spawnOptions));
         this.now = options.now ?? (() => Date.now());
         this.setTimeoutFn = options.setTimeoutFn ?? ((cb, delay) => setTimeout(cb, delay));
@@ -69,6 +73,8 @@ class DaemonProcessManager {
                 restartTimer: null,
                 crashTimestamps: [],
                 stopRequested: false,
+                cooldownTimer: null,
+                cooldownRetryCount: 0,
                 snapshot: {
                     name: agent.name,
                     channel: agent.channel,
@@ -78,6 +84,8 @@ class DaemonProcessManager {
                     startedAt: null,
                     lastCrashAt: null,
                     backoffMs: this.initialBackoffMs,
+                    lastExitCode: null,
+                    lastSignal: null,
                 },
             });
         }
@@ -143,6 +151,7 @@ class DaemonProcessManager {
     async stopAgent(agent) {
         const state = this.requireAgent(agent);
         this.clearRestartTimer(state);
+        this.clearCooldownTimer(state);
         state.stopRequested = true;
         if (!state.process) {
             state.snapshot.status = "stopped";
@@ -203,6 +212,8 @@ class DaemonProcessManager {
             return;
         state.process = null;
         state.snapshot.pid = null;
+        state.snapshot.lastExitCode = code;
+        state.snapshot.lastSignal = signal;
         const crashed = !state.stopRequested && code !== 0;
         const now = this.now();
         const startedAt = state.snapshot.startedAt ? Date.parse(state.snapshot.startedAt) : now;
@@ -233,6 +244,7 @@ class DaemonProcessManager {
                 message: "managed agent exceeded restart limit and is marked crashed",
                 meta: { agent: state.config.name, maxRestartsPerHour: this.maxRestartsPerHour },
             });
+            this.scheduleCooldownRecovery(state);
             return;
         }
         state.snapshot.status = "starting";
@@ -250,6 +262,45 @@ class DaemonProcessManager {
         this.clearTimeoutFn(state.restartTimer);
         state.restartTimer = null;
     }
+    scheduleCooldownRecovery(state) {
+        if (state.cooldownRetryCount >= this.maxCooldownRetries) {
+            (0, runtime_1.emitNervesEvent)({
+                level: "error",
+                component: "daemon",
+                event: "daemon.agent_permanent_failure",
+                message: "managed agent exhausted all cooldown retries — permanently stopped",
+                meta: { agent: state.config.name, cooldownRetryCount: state.cooldownRetryCount, maxCooldownRetries: this.maxCooldownRetries },
+            });
+            return;
+        }
+        this.clearCooldownTimer(state);
+        state.cooldownTimer = this.setTimeoutFn(() => {
+            state.cooldownRetryCount += 1;
+            state.crashTimestamps = [];
+            state.snapshot.backoffMs = this.initialBackoffMs;
+            state.snapshot.status = "starting";
+            state.snapshot.restartCount += 1;
+            (0, runtime_1.emitNervesEvent)({
+                component: "daemon",
+                event: "daemon.agent_cooldown_recovery",
+                message: "attempting cooldown recovery for managed agent",
+                meta: { agent: state.config.name, cooldownRetryCount: state.cooldownRetryCount },
+            });
+            void this.startAgent(state.config.name);
+        }, this.cooldownRecoveryMs);
+        (0, runtime_1.emitNervesEvent)({
+            component: "daemon",
+            event: "daemon.agent_cooldown_scheduled",
+            message: `scheduled cooldown recovery in ${this.cooldownRecoveryMs}ms`,
+            meta: { agent: state.config.name, cooldownRecoveryMs: this.cooldownRecoveryMs, cooldownRetryCount: state.cooldownRetryCount },
+        });
+    }
+    clearCooldownTimer(state) {
+        if (state.cooldownTimer === null)
+            return;
+        this.clearTimeoutFn(state.cooldownTimer);
+        state.cooldownTimer = null;
+    }
     requireAgent(agent) {
         const state = this.agents.get(agent);
         if (!state) {

package/dist/nerves/index.js

@@ -7,6 +7,7 @@ exports.formatTerminalEntry = formatTerminalEntry;
 exports.registerSpinnerHooks = registerSpinnerHooks;
 exports.createTerminalSink = createTerminalSink;
 exports.createStderrSink = createStderrSink;
+exports.rotateIfNeeded = rotateIfNeeded;
 exports.createNdjsonFileSink = createNdjsonFileSink;
 exports.registerGlobalLogSink = registerGlobalLogSink;
 exports.createLogger = createLogger;
@@ -99,22 +100,66 @@ function createTerminalSink(write = (chunk) => process.stderr.write(chunk), colo
 function createStderrSink(write = (chunk) => process.stderr.write(chunk)) {
     return createTerminalSink(write);
 }
-function createNdjsonFileSink(filePath) {
+const DEFAULT_MAX_LOG_SIZE_BYTES = 50 * 1024 * 1024; // 50MB
+const ROTATION_CHECK_INTERVAL_BYTES = 1024 * 1024; // ~1MB between stat checks
+/**
+ * Rotate a log file: shift .1 -> .2, delete old .2, rename current -> .1.
+ * Uses the ndjson extension pattern: file.ndjson -> file.1.ndjson -> file.2.ndjson
+ */
+function rotateIfNeeded(filePath, maxSizeBytes) {
+    let size;
+    try {
+        size = (0, fs_1.statSync)(filePath).size;
+    }
+    catch {
+        return false;
+    }
+    if (size < maxSizeBytes)
+        return false;
+    const ext = filePath.endsWith(".ndjson") ? ".ndjson" : "";
+    const base = ext ? filePath.slice(0, -ext.length) : filePath;
+    const file1 = `${base}.1${ext}`;
+    const file2 = `${base}.2${ext}`;
+    try {
+        (0, fs_1.unlinkSync)(file2);
+    }
+    catch { /* may not exist */ }
+    try {
+        (0, fs_1.renameSync)(file1, file2);
+    }
+    catch { /* may not exist */ }
+    try {
+        (0, fs_1.renameSync)(filePath, file1);
+    }
+    catch { /* best effort */ }
+    return true;
+}
+function createNdjsonFileSink(filePath, maxSizeBytes) {
     (0, fs_1.mkdirSync)((0, path_1.dirname)(filePath), { recursive: true });
     const queue = [];
     let flushing = false;
+    let bytesSinceCheck = 0;
+    const maxSize = maxSizeBytes ?? DEFAULT_MAX_LOG_SIZE_BYTES;
     function flush() {
         if (flushing || queue.length === 0)
             return;
         flushing = true;
         const line = queue.shift();
+        /* v8 ignore start — rotation triggers only after ~1MB of writes @preserve */
+        if (bytesSinceCheck >= ROTATION_CHECK_INTERVAL_BYTES) {
+            bytesSinceCheck = 0;
+            rotateIfNeeded(filePath, maxSize);
+        }
+        /* v8 ignore stop */
         (0, fs_1.appendFile)(filePath, line, "utf8", () => {
             flushing = false;
             flush();
         });
     }
     return (entry) => {
-        queue.push(`${JSON.stringify(entry)}\n`);
+        const line = `${JSON.stringify(entry)}\n`;
+        bytesSinceCheck += line.length;
+        queue.push(line);
         flush();
     };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ouro.bot/cli",
-  "version": "0.1.0-alpha.148",
+  "version": "0.1.0-alpha.149",
   "main": "dist/heart/daemon/ouro-entry.js",
   "bin": {
     "cli": "dist/heart/daemon/ouro-bot-entry.js",