@songsid/agend 2.1.1-beta.13 → 2.1.1-beta.15

package/dist/fleet-manager.js

@@ -177,6 +177,9 @@ export class FleetManager {
     healthServer = null;
     healthPortRetried = false;
     updateCheckTimer = null;
+    eventLogPruneTimer = null;
+    /** Days of event/activity history to keep. */
+    static EVENT_LOG_RETENTION_DAYS = 30;
     watchdogTimer = null;
     startedAt = 0;
     // Mirror topic: buffer cross-instance messages, flush every 3s
@@ -898,7 +901,7 @@ export class FleetManager {
         }
         const pidPath = join(this.dataDir, "fleet.pid");
         writeFileSync(pidPath, String(process.pid), "utf-8");
-        this.eventLog = new EventLog(join(this.dataDir, "events.db"));
+        this.eventLog = this.openEventLog();
         // Initialize classic channel manager. The primary adapter (channels[0])
         // migrates legacy single-bot entries and names without a suffix. Classic
         // routing does NOT go through the routing engine (single-key, can't hold two
@@ -1134,6 +1137,12 @@ export class FleetManager {
         // Signal systemd: generals ready
         sdNotify("READY=1");
         this.watchdogTimer = setInterval(() => sdNotify("WATCHDOG=1"), 30_000);
+        // EventLog.prune() existed but was never called, so `events` and `activity`
+        // grew without bound for the life of the install. Prune once at startup and
+        // daily after that; the timer is unref'd so it never holds the loop open.
+        this.pruneEventLog();
+        this.eventLogPruneTimer = setInterval(() => this.pruneEventLog(), 24 * 60 * 60_000);
+        this.eventLogPruneTimer.unref?.();
         // Phase 2: Start remaining instances with staggered concurrency
         if (others.length > 0) {
             await this.startInstancesWithConcurrency(others, topicMode);
@@ -1585,17 +1594,7 @@ export class FleetManager {
                     await data.respond(t("not_authorized"));
                     return;
                 }
-                try {
-                    const { execSync } = await import("node:child_process");
-                    const backend = this.fleetConfig?.defaults?.backend || "claude-code";
-                    const result = execSync(`agend backend doctor ${backend}`, { timeout: 30_000, encoding: "utf-8" });
-                    const clean = result.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
-                    await data.respond(clean || "No output");
-                }
-                catch (err) {
-                    const output = (err.stdout ?? err.message ?? "Doctor failed").replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
-                    await data.respond(output);
-                }
+                await data.respond(await this.runBackendDoctor());
             }
             else if (data.command === "status") {
                 const text = await this.topicCommands.getStatusText();
@@ -1867,17 +1866,7 @@ export class FleetManager {
                     await data.respond(t("not_authorized"));
                     return;
                 }
-                try {
-                    const { execSync } = await import("node:child_process");
-                    const backend = this.fleetConfig?.defaults?.backend || "claude-code";
-                    const result = execSync(`agend backend doctor ${backend}`, { timeout: 30_000, encoding: "utf-8" });
-                    const clean = result.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
-                    await data.respond(clean || "No output");
-                }
-                catch (err) {
-                    const output = (err.stdout ?? err.message ?? "Doctor failed").replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
-                    await data.respond(output);
-                }
+                await data.respond(await this.runBackendDoctor());
             }
             else if (data.command === "status") {
                 const text = await this.topicCommands.getStatusText();
@@ -2105,9 +2094,22 @@ export class FleetManager {
                 if (existsSync(windowIdPath)) {
                     const windowId = readFileSync(windowIdPath, "utf-8").trim();
                     if (windowId) {
+                        // Async with an explicit timeout: this was execSync with NO timeout at
+                        // all, so a wedged tmux server blocked the whole fleet event loop
+                        // indefinitely — while we were here to diagnose a lost connection.
+                        // A timeout is also the correct signal: an unresponsive tmux server
+                        // means we cannot verify the pane, which is treated as dead (the same
+                        // conclusion the old code reached only by throwing).
                         try {
-                            const { execSync } = await import("node:child_process");
-                            execSync(`tmux list-panes -t "${windowId}"`, { stdio: "ignore" });
+                            const { execFile } = await import("node:child_process");
+                            const { promisify } = await import("node:util");
+                            const { getTmuxSocketName } = await import("./paths.js");
+                            // Honour socket isolation: without -L this queried the user's default
+                            // tmux server instead of the fleet's, so under a custom AGEND_HOME the
+                            // check was meaningless (it reported every pane dead).
+                            const socket = getTmuxSocketName();
+                            const args = socket ? ["-L", socket, "list-panes", "-t", windowId] : ["list-panes", "-t", windowId];
+                            await promisify(execFile)("tmux", args, { timeout: 5_000 });
                         }
                         catch {
                             // Pane dead — respawn
@@ -3454,6 +3456,136 @@ export class FleetManager {
         this.collabInstances.add(instanceName);
         return true;
     }
+    /**
+     * Open the event log, tolerating a corrupt file.
+     *
+     * `events.db` holds history only — event rows and the activity feed. Nothing the
+     * fleet needs to run depends on it, and every consumer already uses
+     * `this.eventLog?.`. An unguarded `new EventLog(...)` here meant a corrupt or
+     * unreadable history file (a truncated WAL after a hard kill, a full disk)
+     * threw during startAll and the WHOLE FLEET FAILED TO BOOT — trading every
+     * running agent for a file whose only job is reporting.
+     *
+     * So: try, move a bad file aside and retry once with a fresh one, and if even
+     * that fails carry on without an event log.
+     */
+    /**
+     * Run `agend backend doctor` for the fleet's default backend and return its
+     * cleaned output.
+     *
+     * Async on purpose: this was `execSync` with a 30s timeout, reachable by any
+     * allowlisted user through `/doctor`. While it ran, the entire fleet event loop
+     * was frozen — no IPC, no adapter, no message delivery, no health responses,
+     * and critically no WATCHDOG ping, so a slow doctor could push past
+     * WatchdogSec and have systemd SIGABRT the fleet.
+     */
+    async runBackendDoctor() {
+        const stripAnsi = (s) => s.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
+        const backend = this.fleetConfig?.defaults?.backend || "claude-code";
+        try {
+            const { execFile } = await import("node:child_process");
+            const { promisify } = await import("node:util");
+            // execFile with an argv array — no shell, so the backend name cannot be
+            // interpreted as a command even if config is malformed.
+            const { stdout } = await promisify(execFile)("agend", ["backend", "doctor", backend], {
+                timeout: 30_000,
+                encoding: "utf-8",
+            });
+            return stripAnsi(stdout) || "No output";
+        }
+        catch (err) {
+            const e = err;
+            return stripAnsi(e.stdout ?? e.message ?? "Doctor failed");
+        }
+    }
+    /** Drop event/activity rows older than the retention window. Best-effort. */
+    pruneEventLog() {
+        try {
+            this.eventLog?.prune(FleetManager.EVENT_LOG_RETENTION_DAYS);
+        }
+        catch (err) {
+            this.logger.warn({ err }, "Event log prune failed");
+        }
+    }
+    openEventLog() {
+        const dbPath = join(this.dataDir, "events.db");
+        try {
+            return new EventLog(dbPath);
+        }
+        catch (err) {
+            this.logger.error({ err, dbPath }, "events.db unusable — moving it aside and starting a fresh one");
+            const stamp = new Date().toISOString().replace(/[:.]/g, "-");
+            for (const suffix of ["", "-wal", "-shm"]) {
+                try {
+                    renameSync(`${dbPath}${suffix}`, `${dbPath}${suffix}.corrupt-${stamp}`);
+                }
+                catch { /* may not exist */ }
+            }
+            try {
+                return new EventLog(dbPath);
+            }
+            catch (retryErr) {
+                // History is worth losing; a fleet that won't start is not.
+                this.logger.error({ err: retryErr, dbPath }, "Could not open a fresh events.db — continuing without event logging");
+                return null;
+            }
+        }
+    }
+    /**
+     * Report a fleet-level fault (not attributable to one instance) to the General
+     * topic, so the operator learns about it without reading daemon.log.
+     *
+     * Throttled per distinct message: an unhandled rejection typically comes from a
+     * loop (a poller, a repeating timer), and one channel message per occurrence
+     * would bury the topic — which is worse than silence. First occurrence goes out
+     * immediately, repeats are suppressed for THROTTLE_MS and then re-sent with a
+     * count.
+     *
+     * The log line is written by the caller regardless: if every adapter is down,
+     * the only notification path is the one that is broken.
+     */
+    notifyFleetError(text) {
+        const now = Date.now();
+        const key = text.slice(0, 200);
+        const seen = this.fleetErrorNotices.get(key);
+        if (seen && now - seen.at < FleetManager.FLEET_ERROR_THROTTLE_MS) {
+            seen.suppressed++;
+            return;
+        }
+        const suppressed = seen?.suppressed ?? 0;
+        this.fleetErrorNotices.set(key, { at: now, suppressed: 0 });
+        // Bound the map: it is keyed by message text, and a message with a varying
+        // suffix (a path, an id) would otherwise grow it without limit.
+        if (this.fleetErrorNotices.size > 100) {
+            const oldest = this.fleetErrorNotices.keys().next().value;
+            if (oldest !== undefined)
+                this.fleetErrorNotices.delete(oldest);
+        }
+        const body = suppressed > 0
+            ? `${text}\n(plus ${suppressed} more in the last ${Math.round(FleetManager.FLEET_ERROR_THROTTLE_MS / 60_000)}m)`
+            : text;
+        // Resolved from config, NOT findGeneralInstance(): that requires a live daemon,
+        // and a fleet-level fault is exactly when the General may be down. The topic
+        // itself still exists, and notifyInstanceTopic only needs adapter + group +
+        // topic_id to post into it.
+        const general = Object.entries(this.fleetConfig?.instances ?? {})
+            .find(([, config]) => config.general_topic === true)?.[0];
+        if (general) {
+            this.notifyInstanceTopic(general, body);
+            return;
+        }
+        // No General instance — fall back to the primary channel's group.
+        const channelCfg = this.getChannelConfig();
+        const groupId = channelCfg?.group_id;
+        if (this.adapter && groupId) {
+            this.adapter.sendText(String(groupId), body)
+                .catch(err => this.logger.warn({ err }, "Failed to send fleet error notification"));
+            return;
+        }
+        this.logger.warn({ text: body }, "Fleet error had no notification target (no General instance, no adapter)");
+    }
+    static FLEET_ERROR_THROTTLE_MS = 10 * 60_000;
+    fleetErrorNotices = new Map();
     notifyInstanceTopic(instanceName, text, extraOpts) {
         const adapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
         if (!adapter)
@@ -4581,9 +4713,19 @@ When users create specialized instances, suggest these configurations:
             // DC path: respond immediately with progress text
             await pending.respond(progressText).catch(() => { });
         }
-        // Apply model in background — don't await here (keeps callback handler fast)
+        // Apply model in background — don't await here (keeps callback handler fast).
+        // Guarded: applyModel() restarts the instance, and an unguarded rejection here
+        // meant a user picking from the /model menu could take the whole fleet down.
+        // On failure the user gets told, rather than the click silently doing nothing.
         void (async () => {
-            const result = await this.applyModel(pending.instanceName, model);
+            let result;
+            try {
+                result = await this.applyModel(pending.instanceName, model);
+            }
+            catch (err) {
+                this.logger.error({ err, instance: pending.instanceName, model }, "Model switch failed");
+                result = `Model switch to \`${model}\` failed: ${err instanceof Error ? err.message : String(err)}`;
+            }
             if (pending.adapter && pending.adapterChatId) {
                 if (progressMsgId) {
                     pending.adapter.editMessage(pending.adapterChatId, progressMsgId, result, pending.adapterThreadId).catch(() => {
@@ -4863,7 +5005,20 @@ When users create specialized instances, suggest these configurations:
         this.logger.info({ channelId, adapterId, instanceName: ch.instanceName }, "Classic channel stopped");
         return t("classic.stopped");
     }
-    async stopAll() {
+    /**
+     * Idempotent while in flight: SIGINT and SIGTERM share one handler and the
+     * uncaughtException path calls this too, so overlapping runs were possible —
+     * each snapshotting the daemon map and calling stop() on the same daemons
+     * concurrently. Deliberately NOT `async`, so callers receive the same promise
+     * object rather than a fresh wrapper around it. The latch clears when the run
+     * settles, so a later genuine stop (after a restart) still does the work.
+     */
+    stopAll() {
+        this.stopAllInFlight ??= this.doStopAll().finally(() => { this.stopAllInFlight = null; });
+        return this.stopAllInFlight;
+    }
+    stopAllInFlight = null;
+    async doStopAll() {
         this.startupComplete = false;
         this.reloadPending = false;
         this.ipcStoppingInstances.add("__fleet_stopping__");
@@ -4887,6 +5042,10 @@ When users create specialized instances, suggest these configurations:
             clearInterval(this.updateCheckTimer);
             this.updateCheckTimer = null;
         }
+        if (this.eventLogPruneTimer) {
+            clearInterval(this.eventLogPruneTimer);
+            this.eventLogPruneTimer = null;
+        }
         if (this.topicCleanupTimer) {
             clearInterval(this.topicCleanupTimer);
             this.topicCleanupTimer = null;
@@ -5304,10 +5463,20 @@ When users create specialized instances, suggest these configurations:
     // ── Update check ────────────────────────────────────────────────────
     async checkForUpdates() {
         try {
-            const { execSync } = await import("node:child_process");
+            // Both npm lookups are async: as execSync they froze the fleet event loop for
+            // up to 15s each, and on a beta build BOTH ran — 30s with no WATCHDOG ping,
+            // past WatchdogSec's half-interval and enough for systemd to SIGABRT the fleet
+            // for a background version check.
+            const { execFile } = await import("node:child_process");
+            const { promisify } = await import("node:util");
+            const execFileP = promisify(execFile);
+            const npmVersion = async (spec) => {
+                const { stdout } = await execFileP("npm", ["view", spec, "version"], { timeout: 15_000 });
+                return stdout.toString().trim();
+            };
             const pkgPath = join(dirname(fileURLToPath(import.meta.url)), "..", "package.json");
             const currentVersion = JSON.parse(readFileSync(pkgPath, "utf-8")).version ?? "0.0.0";
-            const latest = execSync("npm view @songsid/agend version", { stdio: "pipe", timeout: 15_000 }).toString().trim();
+            const latest = await npmVersion("@songsid/agend");
             let target = latest;
             if (currentVersion.includes("-beta")) {
                 // Beta users track the @beta channel (never fall back to @latest, which is
@@ -5315,7 +5484,7 @@ When users create specialized instances, suggest these configurations:
                 // of beta/latest is the newest.
                 let beta = "";
                 try {
-                    beta = execSync("npm view @songsid/agend@beta version", { stdio: "pipe", timeout: 15_000 }).toString().trim();
+                    beta = await npmVersion("@songsid/agend@beta");
                 }
                 catch { /* no beta tag */ }
                 target = beta || latest;
@@ -5584,7 +5753,10 @@ When users create specialized instances, suggest these configurations:
                         res.writeHead(500);
                         res.end(JSON.stringify({ error: `Start failed: ${err.message}` }));
                     }
-                })();
+                    // The inner catch can itself throw (writeHead after a successful
+                    // writeHead is ERR_HTTP_HEADERS_SENT), and that rejection escapes the
+                    // IIFE. Same for the two handlers below.
+                })().catch(err => this.logger.error({ err, name }, "HTTP start handler failed"));
                 return;
             }
             // Instance restart (immediate, no idle wait)
@@ -5605,7 +5777,7 @@ When users create specialized instances, suggest these configurations:
                         res.writeHead(status);
                         res.end(JSON.stringify({ error: `Restart failed: ${err.message}` }));
                     }
-                })();
+                })().catch(err => this.logger.error({ err, name }, "HTTP restart handler failed"));
                 return;
             }
             if (req.method === "POST" && req.url?.startsWith("/stop/")) {
@@ -5628,7 +5800,7 @@ When users create specialized instances, suggest these configurations:
                         res.writeHead(500);
                         res.end(JSON.stringify({ error: `Stop failed: ${err.message}` }));
                     }
-                })();
+                })().catch(err => this.logger.error({ err, name }, "HTTP stop handler failed"));
                 return;
             }
             // ── Agent CLI endpoint ─────