@songsid/agend 2.1.0-beta.29 → 2.1.0-beta.30

package/dist/fleet-manager.js

@@ -52,6 +52,26 @@ export function resolveReplyThreadId(argsThreadId, instanceConfig) {
     }
     return instanceConfig?.topic_id != null ? String(instanceConfig.topic_id) : undefined;
 }
+/**
+ * Pure warm-cap victim selection (extracted for testability). Given the current
+ * warm (running) instance names and a cap, return the LRU idle instances to evict
+ * so the running count returns to the cap. Skips: the `exclude` instance, any
+ * already-evicting, general instances (never evicted), and non-idle instances
+ * (working/stuck can't be evicted). Oldest last-inbound is evicted first; a
+ * missing timestamp (0) sorts oldest. cap <= 0 (or non-integer) = unlimited → [].
+ */
+export function selectLruEvictions(warm, cap, opts) {
+    if (!Number.isInteger(cap) || cap <= 0)
+        return [];
+    if (warm.length <= cap)
+        return [];
+    const candidates = warm.filter(name => name !== opts.exclude
+        && !opts.isEvicting(name)
+        && !opts.isGeneral(name)
+        && opts.isIdle(name));
+    candidates.sort((a, b) => opts.lastInboundAt(a) - opts.lastInboundAt(b));
+    return candidates.slice(0, warm.length - cap);
+}
 /** Retry cadence for retiring a cancel button whose delete failed (e.g. a DC
  * forum thread the bot momentarily can't reach). 3 retries × 5min = 15min. */
 const CANCEL_BTN_RETRY_INTERVAL_MS = 5 * 60_000;
@@ -102,6 +122,8 @@ export class FleetManager {
     lastActivity = new Map();
     /** Latest pane-derived execution snapshot reported by each daemon. */
     instanceStateCache = new Map();
+    /** Instances currently being auto-paused by warm_cap, so concurrent checks don't double-evict. */
+    warmCapEvicting = new Set();
     /** Per-instance tail keeps cross-instance and scheduled deliveries FIFO. */
     idleGatedDeliveryTails = new Map();
     /** Non-user work must observe a fresh idle snapshot after the latest delivery. */
@@ -382,6 +404,46 @@ export class FleetManager {
         });
         for (const check of this.instanceIdleWaiters.get(name) ?? [])
             check();
+        // warm_cap: a fresh transition into idle may free this instance for eviction,
+        // or (more usefully) reveal that the fleet is now over cap. Only fire on the
+        // edge into idle, not on every idle heartbeat.
+        if (state === "idle" && previous?.state !== "idle")
+            this.enforceWarmCap();
+    }
+    /**
+     * Fleet-wide warm cap: if more than `defaults.warm_cap` instances are running,
+     * auto-pause the least-recently-active idle instances until back at the cap.
+     * Never evicts general instances (must stay warm) or working/stuck instances
+     * (only idle). 0/unset = unlimited. wake-before-deliver re-warms any evicted
+     * instance when a message next arrives.
+     *
+     * @param exclude instance to spare (e.g. one just woken to receive a delivery).
+     */
+    enforceWarmCap(exclude) {
+        const cap = this.fleetConfig?.defaults?.warm_cap ?? 0;
+        if (!Number.isInteger(cap) || cap <= 0)
+            return; // 0/invalid = unlimited
+        const warm = [];
+        for (const name of this.daemons.keys()) {
+            if (this.getInstanceStatus(name) === "running")
+                warm.push(name);
+        }
+        if (warm.length <= cap)
+            return;
+        const victims = selectLruEvictions(warm, cap, {
+            exclude,
+            isEvicting: name => this.warmCapEvicting.has(name),
+            isGeneral: name => this.fleetConfig?.instances[name]?.general_topic === true,
+            isIdle: name => this.getInstanceExecutionState(name) === "idle",
+            lastInboundAt: name => readLastInboundAt(this.getInstanceDir(name)) ?? 0,
+        });
+        for (const victim of victims) {
+            this.warmCapEvicting.add(victim);
+            this.logger.info({ instance: victim, warm: warm.length, cap }, "warm_cap exceeded — auto-pausing LRU idle instance");
+            this.lifecycle.pause(victim)
+                .catch(err => this.logger.warn({ err, instance: victim }, "warm_cap auto-pause failed"))
+                .finally(() => this.warmCapEvicting.delete(victim));
+        }
     }
     waitForInstanceIdle(instanceName, timeoutMs, idleObservedAfter = 0) {
         const isReady = () => {
@@ -427,6 +489,9 @@ export class FleetManager {
         if (this.lifecycle.isPaused(instanceName)) {
             const wakeStartedAt = Date.now();
             await this.lifecycle.wake(instanceName, 30_000);
+            // Waking added one to the warm count — make room by evicting a different
+            // LRU idle instance (never this one; it's about to work).
+            this.enforceWarmCap(instanceName);
             // Never satisfy a post-wake gate from a stale pre-pause cache entry.
             idleObservedAfter = Math.max(idleObservedAfter, wakeStartedAt);
         }
@@ -453,6 +518,7 @@ export class FleetManager {
         if (!waitForIdle) {
             if (this.lifecycle.isPaused(instanceName)) {
                 await this.lifecycle.wake(instanceName, 30_000);
+                this.enforceWarmCap(instanceName); // woke one → evict a different LRU idle if over cap
             }
             const ipc = this.instanceIpcClients.get(instanceName);
             if (!ipc?.connected)
@@ -483,6 +549,7 @@ export class FleetManager {
     async changeInstancePauseState(name, action) {
         if (action === "wake") {
             await this.lifecycle.wake(name, 30_000);
+            this.enforceWarmCap(name); // manual wake still respects the fleet warm cap
             return "awake";
         }
         await this.lifecycle.pause(name);
@@ -510,6 +577,13 @@ export class FleetManager {
             return;
         }
         if (config.general_topic) {
+            // antigravity (agy) does not read MCP instructions — fleet context and
+            // routing instructions are not injected, so it cannot act as a dispatcher.
+            const backend = config.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code";
+            if (backend === "antigravity") {
+                this.logger.warn({ name }, "antigravity backend does not support MCP instructions — general dispatcher will not work correctly");
+                this.notifyInstanceTopic(name, "⚠️ antigravity backend is not supported for General instances (no MCP instructions injection). Switch to claude-code or kiro-cli.");
+            }
             this.ensureGeneralInstructions(config.working_directory, config.backend);
         }
         await this.lifecycle.start(name, config, topicMode);
@@ -538,8 +612,21 @@ export class FleetManager {
      */
     async startInstancesWithConcurrency(entries, topicMode) {
         const raw = this.fleetConfig?.defaults?.startup;
-        const concurrency = Math.max(1, Math.min(20, raw?.concurrency ?? 10));
+        const explicitConcurrency = raw?.concurrency;
         const staggerMs = Math.max(0, Math.min(30_000, raw?.stagger_delay_ms ?? 500));
+        // Adaptive concurrency: if not explicitly set, estimate from available RAM.
+        // Each instance uses ~300MB (tmux + CLI process + model overhead).
+        const ESTIMATED_MB_PER_INSTANCE = 300;
+        const { freemem } = await import("node:os");
+        let concurrency;
+        if (explicitConcurrency != null) {
+            concurrency = Math.max(1, Math.min(20, explicitConcurrency));
+        }
+        else {
+            const freeMemMB = Math.round(freemem() / (1024 * 1024));
+            concurrency = Math.max(2, Math.min(10, Math.floor(freeMemMB / ESTIMATED_MB_PER_INSTANCE)));
+            this.logger.info({ concurrency, freeMemMB: freeMemMB, totalInstances: entries.length }, "Adaptive startup concurrency");
+        }
         const byWorkDir = new Map();
         for (const [name, config] of entries) {
             const dir = config.working_directory;
@@ -561,6 +648,17 @@ export class FleetManager {
                 if (pendingTimer)
                     return;
                 while (running < concurrency && idx < groups.length) {
+                    // Re-check memory if adaptive (no explicit concurrency set)
+                    if (explicitConcurrency == null && running > 0) {
+                        const nowFreeMB = Math.round(freemem() / (1024 * 1024));
+                        if (nowFreeMB < ESTIMATED_MB_PER_INSTANCE) {
+                            this.logger.warn({ freeMemMB: nowFreeMB, remaining: groups.length - idx }, "Low memory — pausing instance startup");
+                            // Wait and retry in 5s
+                            pendingTimer = true;
+                            setTimeout(() => { pendingTimer = false; startNext(); }, 5000);
+                            return;
+                        }
+                    }
                     const now = Date.now();
                     const elapsed = now - lastStartAt;
                     if (lastStartAt > 0 && elapsed < staggerMs) {
@@ -986,16 +1084,26 @@ export class FleetManager {
             const classicCount = this.classicChannels?.getAll().length ?? 0;
             const total = Object.keys(fleet.instances).length + classicCount;
             const started = this.daemons.size;
-            const failedNames = Object.keys(fleet.instances).filter(n => !this.daemons.has(n));
+            const allNotRunning = Object.keys(fleet.instances).filter(n => !this.daemons.has(n));
+            const pausedNames = allNotRunning.filter(n => this.lifecycle.isPaused(n));
+            const failedNames = allNotRunning.filter(n => !this.lifecycle.isPaused(n));
             const generalName = this.findGeneralInstance();
             const generalThreadId = generalName ? fleet.instances[generalName]?.topic_id : undefined;
             const { createRequire } = await import("node:module");
             const _require = createRequire(import.meta.url);
             const agendVersion = _require("../package.json").version ?? "unknown";
             if (this.adapter && fleet.channel?.group_id) {
-                const text = failedNames.length === 0
-                    ? t("fleet.ready", started, total, agendVersion)
-                    : t("fleet.ready_with_failed", started, total, agendVersion, failedNames.join(", "));
+                let text;
+                if (failedNames.length === 0 && pausedNames.length === 0) {
+                    text = t("fleet.ready", started, total, agendVersion);
+                }
+                else if (failedNames.length === 0) {
+                    text = t("fleet.ready", started, total, agendVersion) + `\n⏸ Paused: ${pausedNames.join(", ")}`;
+                }
+                else {
+                    text = t("fleet.ready_with_failed", started, total, agendVersion, failedNames.join(", "))
+                        + (pausedNames.length > 0 ? `\n⏸ Paused: ${pausedNames.join(", ")}` : "");
+                }
                 this.adapter.sendText(String(fleet.channel.group_id), text, {
                     threadId: generalThreadId != null ? String(generalThreadId) : undefined,
                 }).catch(e => this.logger.warn({ err: e }, "Failed to send fleet start notification"));
@@ -4544,13 +4652,23 @@ When users create specialized instances, suggest these configurations:
         if (groupId && this.adapter) {
             const total = Object.keys(fleet.instances).length;
             const started = this.daemons.size;
-            const failedNames = Object.keys(fleet.instances).filter(n => !this.daemons.has(n));
+            const allNotRunning2 = Object.keys(fleet.instances).filter(n => !this.daemons.has(n));
+            const pausedNames2 = allNotRunning2.filter(n => this.lifecycle.isPaused(n));
+            const failedNames = allNotRunning2.filter(n => !this.lifecycle.isPaused(n));
             const { createRequire } = await import("node:module");
             const _require2 = createRequire(import.meta.url);
             const agendVersion2 = _require2("../package.json").version ?? "unknown";
-            const restartText = failedNames.length === 0
-                ? t("fleet.ready", started, total, agendVersion2)
-                : t("fleet.ready_with_failed", started, total, agendVersion2, failedNames.join(", "));
+            let restartText;
+            if (failedNames.length === 0 && pausedNames2.length === 0) {
+                restartText = t("fleet.ready", started, total, agendVersion2);
+            }
+            else if (failedNames.length === 0) {
+                restartText = t("fleet.ready", started, total, agendVersion2) + `\n⏸ Paused: ${pausedNames2.join(", ")}`;
+            }
+            else {
+                restartText = t("fleet.ready_with_failed", started, total, agendVersion2, failedNames.join(", "))
+                    + (pausedNames2.length > 0 ? `\n⏸ Paused: ${pausedNames2.join(", ")}` : "");
+            }
             await this.adapter.sendText(String(groupId), restartText, notifyOpts)
                 .catch(e => this.logger.warn({ err: e }, "Failed to post restart completion notification"));
             // Notify each instance's channel — staggered to avoid rate limit storm