@songsid/agend 2.1.1-beta.13 → 2.1.1-beta.14

package/dist/fleet-manager.js

@@ -177,6 +177,9 @@ export class FleetManager {
     healthServer = null;
     healthPortRetried = false;
     updateCheckTimer = null;
+    eventLogPruneTimer = null;
+    /** Days of event/activity history to keep. */
+    static EVENT_LOG_RETENTION_DAYS = 30;
     watchdogTimer = null;
     startedAt = 0;
     // Mirror topic: buffer cross-instance messages, flush every 3s
@@ -898,7 +901,7 @@ export class FleetManager {
         }
         const pidPath = join(this.dataDir, "fleet.pid");
         writeFileSync(pidPath, String(process.pid), "utf-8");
-        this.eventLog = new EventLog(join(this.dataDir, "events.db"));
+        this.eventLog = this.openEventLog();
         // Initialize classic channel manager. The primary adapter (channels[0])
         // migrates legacy single-bot entries and names without a suffix. Classic
         // routing does NOT go through the routing engine (single-key, can't hold two
@@ -1134,6 +1137,12 @@ export class FleetManager {
         // Signal systemd: generals ready
         sdNotify("READY=1");
         this.watchdogTimer = setInterval(() => sdNotify("WATCHDOG=1"), 30_000);
+        // EventLog.prune() existed but was never called, so `events` and `activity`
+        // grew without bound for the life of the install. Prune once at startup and
+        // daily after that; the timer is unref'd so it never holds the loop open.
+        this.pruneEventLog();
+        this.eventLogPruneTimer = setInterval(() => this.pruneEventLog(), 24 * 60 * 60_000);
+        this.eventLogPruneTimer.unref?.();
         // Phase 2: Start remaining instances with staggered concurrency
         if (others.length > 0) {
             await this.startInstancesWithConcurrency(others, topicMode);
@@ -3454,6 +3463,107 @@ export class FleetManager {
         this.collabInstances.add(instanceName);
         return true;
     }
+    /**
+     * Open the event log, tolerating a corrupt file.
+     *
+     * `events.db` holds history only — event rows and the activity feed. Nothing the
+     * fleet needs to run depends on it, and every consumer already uses
+     * `this.eventLog?.`. An unguarded `new EventLog(...)` here meant a corrupt or
+     * unreadable history file (a truncated WAL after a hard kill, a full disk)
+     * threw during startAll and the WHOLE FLEET FAILED TO BOOT — trading every
+     * running agent for a file whose only job is reporting.
+     *
+     * So: try, move a bad file aside and retry once with a fresh one, and if even
+     * that fails carry on without an event log.
+     */
+    /** Drop event/activity rows older than the retention window. Best-effort. */
+    pruneEventLog() {
+        try {
+            this.eventLog?.prune(FleetManager.EVENT_LOG_RETENTION_DAYS);
+        }
+        catch (err) {
+            this.logger.warn({ err }, "Event log prune failed");
+        }
+    }
+    openEventLog() {
+        const dbPath = join(this.dataDir, "events.db");
+        try {
+            return new EventLog(dbPath);
+        }
+        catch (err) {
+            this.logger.error({ err, dbPath }, "events.db unusable — moving it aside and starting a fresh one");
+            const stamp = new Date().toISOString().replace(/[:.]/g, "-");
+            for (const suffix of ["", "-wal", "-shm"]) {
+                try {
+                    renameSync(`${dbPath}${suffix}`, `${dbPath}${suffix}.corrupt-${stamp}`);
+                }
+                catch { /* may not exist */ }
+            }
+            try {
+                return new EventLog(dbPath);
+            }
+            catch (retryErr) {
+                // History is worth losing; a fleet that won't start is not.
+                this.logger.error({ err: retryErr, dbPath }, "Could not open a fresh events.db — continuing without event logging");
+                return null;
+            }
+        }
+    }
+    /**
+     * Report a fleet-level fault (not attributable to one instance) to the General
+     * topic, so the operator learns about it without reading daemon.log.
+     *
+     * Throttled per distinct message: an unhandled rejection typically comes from a
+     * loop (a poller, a repeating timer), and one channel message per occurrence
+     * would bury the topic — which is worse than silence. First occurrence goes out
+     * immediately, repeats are suppressed for THROTTLE_MS and then re-sent with a
+     * count.
+     *
+     * The log line is written by the caller regardless: if every adapter is down,
+     * the only notification path is the one that is broken.
+     */
+    notifyFleetError(text) {
+        const now = Date.now();
+        const key = text.slice(0, 200);
+        const seen = this.fleetErrorNotices.get(key);
+        if (seen && now - seen.at < FleetManager.FLEET_ERROR_THROTTLE_MS) {
+            seen.suppressed++;
+            return;
+        }
+        const suppressed = seen?.suppressed ?? 0;
+        this.fleetErrorNotices.set(key, { at: now, suppressed: 0 });
+        // Bound the map: it is keyed by message text, and a message with a varying
+        // suffix (a path, an id) would otherwise grow it without limit.
+        if (this.fleetErrorNotices.size > 100) {
+            const oldest = this.fleetErrorNotices.keys().next().value;
+            if (oldest !== undefined)
+                this.fleetErrorNotices.delete(oldest);
+        }
+        const body = suppressed > 0
+            ? `${text}\n(plus ${suppressed} more in the last ${Math.round(FleetManager.FLEET_ERROR_THROTTLE_MS / 60_000)}m)`
+            : text;
+        // Resolved from config, NOT findGeneralInstance(): that requires a live daemon,
+        // and a fleet-level fault is exactly when the General may be down. The topic
+        // itself still exists, and notifyInstanceTopic only needs adapter + group +
+        // topic_id to post into it.
+        const general = Object.entries(this.fleetConfig?.instances ?? {})
+            .find(([, config]) => config.general_topic === true)?.[0];
+        if (general) {
+            this.notifyInstanceTopic(general, body);
+            return;
+        }
+        // No General instance — fall back to the primary channel's group.
+        const channelCfg = this.getChannelConfig();
+        const groupId = channelCfg?.group_id;
+        if (this.adapter && groupId) {
+            this.adapter.sendText(String(groupId), body)
+                .catch(err => this.logger.warn({ err }, "Failed to send fleet error notification"));
+            return;
+        }
+        this.logger.warn({ text: body }, "Fleet error had no notification target (no General instance, no adapter)");
+    }
+    static FLEET_ERROR_THROTTLE_MS = 10 * 60_000;
+    fleetErrorNotices = new Map();
     notifyInstanceTopic(instanceName, text, extraOpts) {
         const adapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
         if (!adapter)
@@ -4581,9 +4691,19 @@ When users create specialized instances, suggest these configurations:
             // DC path: respond immediately with progress text
             await pending.respond(progressText).catch(() => { });
         }
-        // Apply model in background — don't await here (keeps callback handler fast)
+        // Apply model in background — don't await here (keeps callback handler fast).
+        // Guarded: applyModel() restarts the instance, and an unguarded rejection here
+        // meant a user picking from the /model menu could take the whole fleet down.
+        // On failure the user gets told, rather than the click silently doing nothing.
         void (async () => {
-            const result = await this.applyModel(pending.instanceName, model);
+            let result;
+            try {
+                result = await this.applyModel(pending.instanceName, model);
+            }
+            catch (err) {
+                this.logger.error({ err, instance: pending.instanceName, model }, "Model switch failed");
+                result = `Model switch to \`${model}\` failed: ${err instanceof Error ? err.message : String(err)}`;
+            }
             if (pending.adapter && pending.adapterChatId) {
                 if (progressMsgId) {
                     pending.adapter.editMessage(pending.adapterChatId, progressMsgId, result, pending.adapterThreadId).catch(() => {
@@ -4863,7 +4983,20 @@ When users create specialized instances, suggest these configurations:
         this.logger.info({ channelId, adapterId, instanceName: ch.instanceName }, "Classic channel stopped");
         return t("classic.stopped");
     }
-    async stopAll() {
+    /**
+     * Idempotent while in flight: SIGINT and SIGTERM share one handler and the
+     * uncaughtException path calls this too, so overlapping runs were possible —
+     * each snapshotting the daemon map and calling stop() on the same daemons
+     * concurrently. Deliberately NOT `async`, so callers receive the same promise
+     * object rather than a fresh wrapper around it. The latch clears when the run
+     * settles, so a later genuine stop (after a restart) still does the work.
+     */
+    stopAll() {
+        this.stopAllInFlight ??= this.doStopAll().finally(() => { this.stopAllInFlight = null; });
+        return this.stopAllInFlight;
+    }
+    stopAllInFlight = null;
+    async doStopAll() {
         this.startupComplete = false;
         this.reloadPending = false;
         this.ipcStoppingInstances.add("__fleet_stopping__");
@@ -4887,6 +5020,10 @@ When users create specialized instances, suggest these configurations:
             clearInterval(this.updateCheckTimer);
             this.updateCheckTimer = null;
         }
+        if (this.eventLogPruneTimer) {
+            clearInterval(this.eventLogPruneTimer);
+            this.eventLogPruneTimer = null;
+        }
         if (this.topicCleanupTimer) {
             clearInterval(this.topicCleanupTimer);
             this.topicCleanupTimer = null;
@@ -5584,7 +5721,10 @@ When users create specialized instances, suggest these configurations:
                         res.writeHead(500);
                         res.end(JSON.stringify({ error: `Start failed: ${err.message}` }));
                     }
-                })();
+                    // The inner catch can itself throw (writeHead after a successful
+                    // writeHead is ERR_HTTP_HEADERS_SENT), and that rejection escapes the
+                    // IIFE. Same for the two handlers below.
+                })().catch(err => this.logger.error({ err, name }, "HTTP start handler failed"));
                 return;
             }
             // Instance restart (immediate, no idle wait)
@@ -5605,7 +5745,7 @@ When users create specialized instances, suggest these configurations:
                         res.writeHead(status);
                         res.end(JSON.stringify({ error: `Restart failed: ${err.message}` }));
                     }
-                })();
+                })().catch(err => this.logger.error({ err, name }, "HTTP restart handler failed"));
                 return;
             }
             if (req.method === "POST" && req.url?.startsWith("/stop/")) {
@@ -5628,7 +5768,7 @@ When users create specialized instances, suggest these configurations:
                         res.writeHead(500);
                         res.end(JSON.stringify({ error: `Stop failed: ${err.message}` }));
                     }
-                })();
+                })().catch(err => this.logger.error({ err, name }, "HTTP stop handler failed"));
                 return;
             }
             // ── Agent CLI endpoint ─────