@songsid/agend 2.1.4-beta.52 → 2.1.4-beta.54

package/dist/fleet-manager.js

@@ -1,7 +1,7 @@
 import { existsSync, readFileSync, mkdirSync, writeFileSync, unlinkSync, rmSync, readdirSync, renameSync, copyFileSync, chmodSync, statSync } from "node:fs";
 import { randomBytes } from "node:crypto";
 import { spawnSync } from "node:child_process";
-import { freemem } from "node:os";
+import { freemem, totalmem } from "node:os";
 import { createServer } from "node:http";
 import { join, dirname, basename } from "node:path";
 import { fileURLToPath } from "node:url";
@@ -567,7 +567,8 @@ export class FleetManager {
     getSysInfo() {
         const mem = process.memoryUsage();
         const toMB = (b) => Math.round(b / 1024 / 1024 * 10) / 10;
-        const instances = Object.keys(this.fleetConfig?.instances ?? {}).map(name => ({
+        // Fleet instances (fleet.yaml)
+        const fleetInstances = Object.keys(this.fleetConfig?.instances ?? {}).map(name => ({
             name,
             status: this.getInstanceStatus(name),
             state: this.getInstanceExecutionState(name),
@@ -575,12 +576,46 @@ export class FleetManager {
             costCents: this.costGuard?.getDailyCostCents(name) ?? 0,
             rateLimits: this.statuslineWatcher.getRateLimits(name) ?? null,
         }));
+        // Classic instances (classicBot.yaml) — dedupe against fleet
+        const fleetNames = new Set(fleetInstances.map(i => i.name));
+        const classicInstances = (this.classicChannels?.getAll() ?? [])
+            .filter(ch => !fleetNames.has(ch.instanceName))
+            .map(ch => ({
+            name: ch.instanceName,
+            status: this.getInstanceStatus(ch.instanceName),
+            state: this.getInstanceExecutionState(ch.instanceName),
+            ipc: this.instanceIpcClients.has(ch.instanceName),
+            costCents: this.costGuard?.getDailyCostCents(ch.instanceName) ?? 0,
+            rateLimits: this.statuslineWatcher.getRateLimits(ch.instanceName) ?? null,
+        }));
+        // Combined roster (matches /api/fleet and agend ls)
+        const allInstances = [...fleetInstances, ...classicInstances];
+        // Fleet summary counts (fleet + Classic combined)
+        const running_count = allInstances.filter(i => i.status === "running").length;
+        const paused_count = allInstances.filter(i => i.status === "paused").length;
+        // System memory (GB, 1 decimal)
+        const totalGB = totalmem() / (1024 ** 3);
+        const usedGB = (totalmem() - freemem()) / (1024 ** 3);
+        const system_mem_gb = {
+            used: Math.round(usedGB * 10) / 10,
+            total: Math.round(totalGB * 10) / 10,
+        };
+        // Fleet memory: O(1) cgroup read (includes entire service tree: fleet + CLIs + MCP servers)
+        // This avoids blocking the event loop with per-instance tree scans.
+        const fleetMem = readFleetMemory();
+        const fleet_mem_mb = fleetMem.cgroupAnonBytes != null
+            ? Math.round(fleetMem.cgroupAnonBytes / (1024 * 1024) * 10) / 10
+            : null;
         return {
             uptime_seconds: Math.floor((Date.now() - this.startedAt) / 1000),
             memory_mb: { rss: toMB(mem.rss), heapUsed: toMB(mem.heapUsed), heapTotal: toMB(mem.heapTotal) },
-            instances,
+            instances: fleetInstances, // SysInfo.instances is fleet-only; /api/fleet enriches with Classic
             fleet_cost_cents: this.costGuard?.getFleetTotalCents() ?? 0,
             fleet_cost_limit_cents: this.costGuard?.getLimitCents() ?? 0,
+            running_count,
+            paused_count,
+            fleet_mem_mb,
+            system_mem_gb,
         };
     }
     /** Load fleet.yaml and build routing table */
@@ -2926,7 +2961,8 @@ export class FleetManager {
                 await data.respond(text);
             }
             else if (data.command === "sysinfo") {
-                await data.respond(this.topicCommands.getSysInfoText());
+                // Slash commands are Discord-only; use plain lines (no markdown table)
+                await data.respond(this.topicCommands.getSysInfoText({ platform: "discord" }));
             }
             else if (data.command === "dashboard") {
                 // Reply is ephemeral (adapter defers non-chat commands ephemerally), so
@@ -3272,7 +3308,8 @@ export class FleetManager {
                 await data.respond(text);
             }
             else if (data.command === "sysinfo") {
-                await data.respond(this.topicCommands.getSysInfoText());
+                // Slash commands are Discord-only; use plain lines (no markdown table)
+                await data.respond(this.topicCommands.getSysInfoText({ platform: "discord" }));
             }
             else if (data.command === "dashboard") {
                 // Reply is ephemeral (adapter defers non-chat commands ephemerally), so
@@ -5404,14 +5441,6 @@ export class FleetManager {
             return;
         }
         const suppressed = seen?.suppressed ?? 0;
-        this.fleetErrorNotices.set(key, { at: now, suppressed: 0 });
-        // Bound the map: it is keyed by message text, and a message with a varying
-        // suffix (a path, an id) would otherwise grow it without limit.
-        if (this.fleetErrorNotices.size > 100) {
-            const oldest = this.fleetErrorNotices.keys().next().value;
-            if (oldest !== undefined)
-                this.fleetErrorNotices.delete(oldest);
-        }
         const body = suppressed > 0
             ? `${text}\n${t("fleet.error_suppressed", suppressed, Math.round(FleetManager.FLEET_ERROR_THROTTLE_MS / 60_000))}`
             : text;
@@ -5421,26 +5450,60 @@ export class FleetManager {
         // topic_id to post into it.
         const general = Object.entries(this.fleetConfig?.instances ?? {})
             .find(([, config]) => config.general_topic === true)?.[0];
+        // "dispatched", not "delivered": see notifyInstanceTopic. A platform that
+        // rejects the message afterwards still consumes the throttle window, which
+        // is the pre-existing semantic and not something this change alters.
+        let dispatched = false;
         if (general) {
-            this.notifyInstanceTopic(general, body);
-            return;
+            dispatched = this.notifyInstanceTopic(general, body);
         }
-        // No General instance — fall back to the primary channel's group.
-        const channelCfg = this.getChannelConfig();
-        const groupId = channelCfg?.group_id;
-        if (this.adapter && groupId) {
-            this.adapter.sendText(String(groupId), body)
-                .catch(err => this.logger.warn({ err }, "Failed to send fleet error notification"));
+        else {
+            // No General instance — fall back to the primary channel's group.
+            const groupId = this.getChannelConfig()?.group_id;
+            if (this.adapter && groupId) {
+                this.adapter.sendText(String(groupId), body)
+                    .catch(err => this.logger.warn({ err }, "Failed to send fleet error notification"));
+                dispatched = true;
+            }
+        }
+        if (!dispatched) {
+            // Do NOT record the throttle key. It used to be claimed before delivery
+            // was attempted, so a message nobody could receive — every fleet error
+            // raised before adapters exist, which is when startup faults happen —
+            // also suppressed the next ten minutes of identical ones. Leaving the key
+            // unspent lets the next caller, once an adapter is up, actually get
+            // through. The counter is not reset either: nothing was shown.
+            this.logger.warn({ text: body }, "Fleet error could not be delivered (no adapter or no target yet) — not consuming the throttle window");
             return;
         }
-        this.logger.warn({ text: body }, "Fleet error had no notification target (no General instance, no adapter)");
+        this.fleetErrorNotices.set(key, { at: now, suppressed: 0 });
+        // Bound the map: it is keyed by message text, and a message with a varying
+        // suffix (a path, an id) would otherwise grow it without limit.
+        if (this.fleetErrorNotices.size > 100) {
+            const oldest = this.fleetErrorNotices.keys().next().value;
+            if (oldest !== undefined)
+                this.fleetErrorNotices.delete(oldest);
+        }
     }
     static FLEET_ERROR_THROTTLE_MS = 10 * 60_000;
     fleetErrorNotices = new Map();
+    /**
+     * Post into an instance's topic. Returns whether a send was DISPATCHED — a
+     * target was resolved and sendText was called — so a caller that must not
+     * lose the message (notifyFleetError, which spends a throttle key) can tell
+     * that from silence. Existing callers ignore the result and are unaffected.
+     *
+     * Not a delivery guarantee: sendText is fire-and-forget, and a platform-side
+     * rejection surfaces only as a warn in its .catch. True delivery confirmation
+     * would have to make this async and change every caller.
+     */
     notifyInstanceTopic(instanceName, text, extraOpts) {
         const adapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
-        if (!adapter)
-            return;
+        if (!adapter) {
+            // Early startup: adapters are created well after the fleet object exists.
+            this.logger.warn({ instanceName }, "No adapter yet — instance topic notification not sent");
+            return false;
+        }
         const channelCfg = this.getChannelConfig(this.getInstanceAdapterId(instanceName));
         const groupId = channelCfg?.group_id;
         // Fleet topic instance
@@ -5448,20 +5511,23 @@ export class FleetManager {
         if (threadId != null && groupId) {
             adapter.sendText(String(groupId), text, { threadId: String(threadId), ...extraOpts })
                 .catch(e => this.logger.warn({ err: e, instanceName }, "Failed to send instance topic notification"));
-            return;
+            return true;
         }
         // Classic instance: find its channelId from the classic manager
         const classicChatId = this.classicChannels?.getChannelIdByInstance(instanceName);
         if (classicChatId) {
             adapter.sendText(classicChatId, text, extraOpts)
                 .catch(e => this.logger.warn({ err: e, instanceName }, "Failed to send classic notification"));
-            return;
+            return true;
         }
         // Fallback: send to group without threadId
         if (groupId) {
             adapter.sendText(String(groupId), text, extraOpts)
                 .catch(e => this.logger.warn({ err: e, instanceName }, "Failed to send notification (no topic)"));
+            return true;
         }
+        this.logger.warn({ instanceName }, "No group id — instance topic notification not sent");
+        return false;
     }
     // ── Nonce-armed button prompts (hang / assist / exit / clear) ──
     //
@@ -9802,72 +9868,81 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
             }
             // Fleet API (enriched for agent board)
             if (req.method === "GET" && req.url === "/api/fleet") {
-                const sysInfo = this.getSysInfo();
-                const fleetInstances = sysInfo.instances.map(inst => ({ ...inst, classic: false }));
-                const fleetNames = new Set(fleetInstances.map(inst => inst.name));
-                const classicInstances = (this.classicChannels?.getAll() ?? [])
-                    .filter(channel => !fleetNames.has(channel.instanceName))
-                    .map(channel => ({
-                    name: channel.instanceName,
-                    status: this.getInstanceStatus(channel.instanceName),
-                    state: this.getInstanceExecutionState(channel.instanceName),
-                    ipc: this.instanceIpcClients.has(channel.instanceName),
-                    costCents: this.costGuard?.getDailyCostCents(channel.instanceName) ?? 0,
-                    rateLimits: this.statuslineWatcher.getRateLimits(channel.instanceName) ?? null,
-                    classic: true,
-                    classicName: channel.name,
-                    channelId: channel.channelId,
-                    adapterId: channel.adapterId ?? null,
-                }));
-                const enriched = [...fleetInstances, ...classicInstances].map(inst => {
-                    const config = this.fleetConfig?.instances[inst.name];
-                    const persistedInboundAt = readLastInboundAt(this.getInstanceDir(inst.name));
-                    const lastActivity = inst.classic
-                        ? Math.max(persistedInboundAt ?? 0, readClassicLastActivityAt(this.dataDir, inst.name) ?? 0) || null
-                        : (persistedInboundAt ?? this.lastActivityMs(inst.name)) || null;
-                    const backend = this.backendNameForInstance(inst.name);
-                    const resolvedModel = this.resolveInstanceModel(inst.name);
-                    const effortStrategy = this.effortStrategyFor(inst.name);
-                    const resolvedEffort = this.resolveInstanceEffort(inst.name);
-                    // Find claimed tasks for this instance
-                    let currentTask = null;
-                    try {
-                        const tasks = this.scheduler?.db.listTasks({ assignee: inst.name, status: "claimed" });
-                        if (tasks?.length)
-                            currentTask = tasks[0].title;
-                    }
-                    catch (err) {
-                        this.logger.debug({ err, name: inst.name }, "Scheduler listTasks failed (/api/fleet)");
+                try {
+                    const sysInfo = this.getSysInfo();
+                    const fleetInstances = sysInfo.instances.map(inst => ({ ...inst, classic: false }));
+                    const fleetNames = new Set(fleetInstances.map(inst => inst.name));
+                    const classicInstances = (this.classicChannels?.getAll() ?? [])
+                        .filter(channel => !fleetNames.has(channel.instanceName))
+                        .map(channel => ({
+                        name: channel.instanceName,
+                        status: this.getInstanceStatus(channel.instanceName),
+                        state: this.getInstanceExecutionState(channel.instanceName),
+                        ipc: this.instanceIpcClients.has(channel.instanceName),
+                        costCents: this.costGuard?.getDailyCostCents(channel.instanceName) ?? 0,
+                        rateLimits: this.statuslineWatcher.getRateLimits(channel.instanceName) ?? null,
+                        classic: true,
+                        classicName: channel.name,
+                        channelId: channel.channelId,
+                        adapterId: channel.adapterId ?? null,
+                    }));
+                    const enriched = [...fleetInstances, ...classicInstances].map(inst => {
+                        const config = this.fleetConfig?.instances[inst.name];
+                        const persistedInboundAt = readLastInboundAt(this.getInstanceDir(inst.name));
+                        const lastActivity = inst.classic
+                            ? Math.max(persistedInboundAt ?? 0, readClassicLastActivityAt(this.dataDir, inst.name) ?? 0) || null
+                            : (persistedInboundAt ?? this.lastActivityMs(inst.name)) || null;
+                        const backend = this.backendNameForInstance(inst.name);
+                        const resolvedModel = this.resolveInstanceModel(inst.name);
+                        const effortStrategy = this.effortStrategyFor(inst.name);
+                        const resolvedEffort = this.resolveInstanceEffort(inst.name);
+                        // Find claimed tasks for this instance
+                        let currentTask = null;
+                        try {
+                            const tasks = this.scheduler?.db.listTasks({ assignee: inst.name, status: "claimed" });
+                            if (tasks?.length)
+                                currentTask = tasks[0].title;
+                        }
+                        catch (err) {
+                            this.logger.debug({ err, name: inst.name }, "Scheduler listTasks failed (/api/fleet)");
+                        }
+                        return {
+                            ...inst,
+                            description: config?.description ?? ("classicName" in inst ? inst.classicName : null),
+                            backend,
+                            // Settings renders these runtime-effective values rather than the
+                            // sparse user-authored YAML. `auto` means the supported CLI is
+                            // using its own effort default; null is reserved for unsupported.
+                            model: resolvedModel.model,
+                            model_display: resolvedModel.display,
+                            model_source: resolvedModel.source,
+                            effort: effortStrategy === "unsupported" ? null : (resolvedEffort.effort ?? "auto"),
+                            effort_supported: effortStrategy !== "unsupported",
+                            tool_set: config?.tool_set ?? "full",
+                            general_topic: config?.general_topic ?? false,
+                            // User activity is persisted by the daemon, so both the board and
+                            // auto-pause retain an accurate age across fleet restarts.
+                            lastActivity,
+                            currentTask,
+                            idle: this.getInstanceIdle(inst.name),
+                            state: this.getInstanceExecutionState(inst.name),
+                        };
+                    });
+                    res.setHeader("Access-Control-Allow-Origin", "*");
+                    res.writeHead(200);
+                    res.end(JSON.stringify({
+                        ...sysInfo,
+                        version: this.currentVersion,
+                        instances: enriched,
+                    }));
+                }
+                catch (err) {
+                    this.logger.error({ err }, "/api/fleet failed");
+                    if (!res.headersSent) {
+                        res.writeHead(500);
+                        res.end("Internal Server Error");
                     }
-                    return {
-                        ...inst,
-                        description: config?.description ?? ("classicName" in inst ? inst.classicName : null),
-                        backend,
-                        // Settings renders these runtime-effective values rather than the
-                        // sparse user-authored YAML. `auto` means the supported CLI is
-                        // using its own effort default; null is reserved for unsupported.
-                        model: resolvedModel.model,
-                        model_display: resolvedModel.display,
-                        model_source: resolvedModel.source,
-                        effort: effortStrategy === "unsupported" ? null : (resolvedEffort.effort ?? "auto"),
-                        effort_supported: effortStrategy !== "unsupported",
-                        tool_set: config?.tool_set ?? "full",
-                        general_topic: config?.general_topic ?? false,
-                        // User activity is persisted by the daemon, so both the board and
-                        // auto-pause retain an accurate age across fleet restarts.
-                        lastActivity,
-                        currentTask,
-                        idle: this.getInstanceIdle(inst.name),
-                        state: this.getInstanceExecutionState(inst.name),
-                    };
-                });
-                res.setHeader("Access-Control-Allow-Origin", "*");
-                res.writeHead(200);
-                res.end(JSON.stringify({
-                    ...sysInfo,
-                    version: this.currentVersion,
-                    instances: enriched,
-                }));
+                }
                 return;
             }
             // Activity API