@songsid/agend 2.1.5-beta.3 → 2.1.5-beta.5

package/dist/fleet-manager.js

@@ -60,7 +60,7 @@ import { clearPausedMarker } from "./pause-marker.js";
 import { releaseProcessFleetLock } from "./fleet-lock.js";
 import { GENERAL_PAUSE_ERROR, isGeneralInstance } from "./general-instance.js";
 import { loadOrCreateWebToken, WEB_TOKEN_INVALID_MESSAGE } from "./web-auth.js";
-import { RestartProgress } from "./restart-progress.js";
+import { RESTART_PROGRESS_TERMINAL_TIMEOUT_MS, RestartProgress, } from "./restart-progress.js";
 import { collectRedundantInstanceDefaultPaths } from "./fleet-yaml-slim.js";
 import { StormWindow } from "./storm-window.js";
 import { SpawnGate } from "./spawn-gate.js";
@@ -228,7 +228,39 @@ const CLEAR_CONFIRM_TIMEOUT_MS = 15_000;
 /** Default lifetime for long-lived nonce prompts (clear overrides this to 15s). */
 const NONCE_BUTTON_TIMEOUT_MS = 15 * 60_000;
 const TIP_BUTTON_TIMEOUT_MS = 24 * 60 * 60_000;
-const CLI_ENV_TTL_MS = 24 * 60 * 60 * 1000; // /model reads cached CLI env within 24h
+const CLI_ENV_TTL_MS = 24 * 60 * 60 * 1000; // hard validity bound for the cached CLI env
+/**
+ * How old the cached CLI env may be before `/model` re-probes it live.
+ *
+ * The cache is a file under AGEND_HOME, so it outlives the process, and the only
+ * thing that refreshed it was the startup probe. That is why a newly released
+ * model showed up only after `agend stop` + `agend start`: a bare `agend restart`
+ * signals SIGUSR2 and restarts the instances inside the *same* manager process,
+ * so nothing re-probed and `/model` kept serving a list up to 24h old.
+ */
+const CLI_ENV_FRESH_MS = 60 * 60 * 1000;
+/**
+ * Upper bound on a live probe driven by `/model`.
+ *
+ * A probe chains bounded leaves but is not itself bounded: claude-code runs
+ * `--version` (5s) then listModels then listApiModels (an 8s AbortController
+ * against api.anthropic.com), and several backends' probes carry no explicit
+ * timeout at all. Awaiting that inline would make `/model` the next thing to
+ * hang, so it races this deadline and falls back to the cached list.
+ *
+ * Must stay above CLI_PROBE_LONGEST_CHAIN_MS — the probe's bounded steps run
+ * back to back, so clearing only the longest single leaf would be false
+ * confidence: a deadline above 8s but below the 13s chain still truncates a
+ * probe that would have succeeded. Bounding the chain rather than the leaf is
+ * the lesson from the usage hang, and the tests assert against the derived
+ * chain constant so raising either step cannot silently break it.
+ *
+ * The deadline exists to stop a probe hanging forever, not to cut short one that
+ * would have finished: truncating a legitimate probe serves the previous list
+ * and hides exactly the newly released model the user opened `/model` to find.
+ * The wait is announced before it starts, so it reads as progress, not a stall.
+ */
+export const CLI_ENV_PROBE_DEADLINE_MS = 16_000;
 export class FleetManager {
     dataDir;
     static signalTarget = null;
@@ -1751,16 +1783,60 @@ export class FleetManager {
         if (!generalName)
             return null;
         const adapter = this.getAdapterForInstance(generalName);
+        const adapterId = this.getInstanceAdapterId(generalName);
         const chatId = this.getGroupIdForInstance(generalName);
         if (!adapter || !chatId)
             return null;
         const topicId = this.fleetConfig?.instances[generalName]?.topic_id;
         return {
             adapter,
+            resolveAdapter: adapterId ? () => this.readyProgressAdapter(adapterId) : undefined,
             chatId,
             threadId: topicId != null ? String(topicId) : undefined,
         };
     }
+    /** Resolve only an adapter generation that can accept progress delivery.
+     * Discord exposes direct gateway readiness; adapters without a health
+     * snapshot use the fleet startup/retry state. */
+    readyProgressAdapter(adapterId) {
+        const adapter = this.adapters.get(adapterId);
+        if (!adapter)
+            return undefined;
+        const health = adapter.getHealthSnapshot?.();
+        if (health)
+            return health.isReady ? adapter : undefined;
+        return this.adapterState.get(adapterId)?.status === "connected" ? adapter : undefined;
+    }
+    /** Last-resort completion after RestartProgress could not deliver to its
+     * adopted target. It stays bounded and never wakes a not-ready gateway. */
+    async sendFleetStartCompletionFallback(chatId, text, threadId) {
+        const adapterId = this.getPrimaryAdapterId();
+        const adapter = adapterId ? this.readyProgressAdapter(adapterId) : undefined;
+        if (!adapter) {
+            this.logger.error({ adapterId }, "Fleet start completion fallback skipped because the primary adapter is not ready");
+            return false;
+        }
+        let timer;
+        const timeout = new Promise(resolve => {
+            timer = setTimeout(() => resolve({ status: "timeout" }), RESTART_PROGRESS_TERMINAL_TIMEOUT_MS);
+            timer.unref?.();
+        });
+        const delivery = Promise.resolve()
+            .then(() => adapter.sendText(chatId, text, { threadId }))
+            .then(() => ({ status: "sent" }), err => ({ status: "failed", err }));
+        const result = await Promise.race([delivery, timeout]);
+        if (timer)
+            clearTimeout(timer);
+        if (result.status === "sent")
+            return true;
+        if (result.status === "failed") {
+            this.logger.error({ err: result.err }, "Failed to send fleet start completion fallback");
+        }
+        else {
+            this.logger.error({ timeout_ms: RESTART_PROGRESS_TERMINAL_TIMEOUT_MS }, "Timed out sending fleet start completion fallback");
+        }
+        return false;
+    }
     async stopInstance(name) {
         this.explicitStopGeneration.set(name, (this.explicitStopGeneration.get(name) ?? 0) + 1);
         this.cancelStartupRetry(name);
@@ -2317,12 +2393,17 @@ export class FleetManager {
             progressStart = adapterStartup.then(() => {
                 if (pendingUpdateProgress) {
                     const saved = pendingUpdateProgress.progress.target;
-                    const adapter = this.adapters.get(saved.adapterId);
-                    const target = adapter ? {
-                        adapter,
+                    const target = {
+                        adapter: this.adapters.get(saved.adapterId),
+                        // Adapter retries replace the failed object in this map. Resolve at
+                        // every progress delivery so the adopted update message follows the
+                        // live, ready generation instead of remaining pinned to a stopped
+                        // client. This also waits when the first generation failed before
+                        // any adapter object was registered.
+                        resolveAdapter: () => this.readyProgressAdapter(saved.adapterId),
                         chatId: saved.chatId,
                         threadId: saved.threadId,
-                    } : null;
+                    };
                     return startupProgress.resume(target, saved.messageId);
                 }
                 return startupProgress.start(this.restartProgressTarget());
@@ -2439,7 +2520,7 @@ export class FleetManager {
                     })()
                     : undefined,
             });
-            if (!progressCompleted && this.adapter && fleet.channel?.group_id) {
+            if (!progressCompleted && fleet.channel?.group_id) {
                 let text;
                 if (failedNames.length === 0 && pausedNames.length === 0) {
                     text = t("fleet.ready", started, total, agendVersion);
@@ -2451,9 +2532,7 @@ export class FleetManager {
                     text = t("fleet.ready_with_failed", started, total, agendVersion, failedNames.join(", "))
                         + (pausedNames.length > 0 ? `\n⏸ Paused: ${pausedNames.join(", ")}` : "");
                 }
-                this.adapter.sendText(String(fleet.channel.group_id), text, {
-                    threadId: generalThreadId != null ? String(generalThreadId) : undefined,
-                }).catch(e => this.logger.warn({ err: e }, "Failed to send fleet start notification"));
+                await this.sendFleetStartCompletionFallback(String(fleet.channel.group_id), text, generalThreadId != null ? String(generalThreadId) : undefined);
             }
         }
         // Health HTTP endpoint
@@ -8376,6 +8455,32 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
         catch { /* missing / stale / corrupt */ }
         return null;
     }
+    /** True when a cached CLI env is old enough that `/model` should re-probe. */
+    cliEnvNeedsRefresh(env) {
+        return !env || typeof env.probedAt !== "number" || Date.now() - env.probedAt >= CLI_ENV_FRESH_MS;
+    }
+    /**
+     * Run a live probe under a deadline, falling back to whatever the cache holds.
+     * A model list is an aid: a vendor that stops answering must degrade to the
+     * previous list, never stall the command that asked for it.
+     */
+    async probeBackendBounded(backend) {
+        const work = this.probeBackend(backend);
+        work.catch(() => { });
+        let timer;
+        const deadline = new Promise(resolve => {
+            timer = setTimeout(() => {
+                this.logger.warn({ backend, deadlineMs: CLI_ENV_PROBE_DEADLINE_MS }, "CLI env live probe exceeded its deadline — serving the cached model list");
+                resolve(null);
+            }, CLI_ENV_PROBE_DEADLINE_MS);
+        });
+        try {
+            return await Promise.race([work, deadline]);
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    }
     /**
      * Resolve the effective model for a fleet or ClassicBot instance, plus where it
      * came from. Single source of truth for `/model` and `/ctx` — precedence:
@@ -8477,16 +8582,23 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
             void this.probeBackend(b);
     }
     /** Best-effort model list for `/model`: cached CLI env first, else live probe. Never throws. */
-    async getModelOptions(instanceName, refresh = false) {
+    async getModelOptions(instanceName, refresh = false, onLiveProbe) {
         const backendName = this.backendNameForInstance(instanceName);
-        if (!refresh) {
-            const cached = this.readCliEnv(backendName);
-            if (cached && cached.models.length)
-                return cached.models;
-        }
-        // Cache miss / stale / forced refresh → probe live (also refreshes the cache).
-        const env = await this.probeBackend(backendName);
-        return env?.models ?? [];
+        const cached = this.readCliEnv(backendName);
+        if (!refresh && cached?.models.length && !this.cliEnvNeedsRefresh(cached))
+            return cached.models;
+        // About to go to the vendor: let the caller say so. A silent 1–10s pause on
+        // an interactive command reads as another hang, which is the wrong lesson to
+        // teach a user who has just been bitten by one.
+        onLiveProbe?.();
+        // Stale, missing, or a forced refresh → probe live (also refreshes the cache).
+        // A newly released model is invisible until this runs, which is why staleness
+        // triggers it rather than waiting for the 24h hard expiry or a cold start.
+        const env = await this.probeBackendBounded(backendName);
+        if (env?.models.length)
+            return env.models;
+        // Probe failed or timed out: the previous list is still the best answer.
+        return cached?.models ?? [];
     }
     /**
      * Model catalog behind the `list_models` tool.
@@ -8777,7 +8889,9 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
             await data.respond(t("model.usage"));
             return;
         }
-        const options = await this.getModelOptions(name, isRefresh);
+        const options = await this.getModelOptions(name, isRefresh, () => {
+            void data.respond(t("model.refreshing")).catch(() => { });
+        });
         if (options.length === 0) {
             await data.respond(t("model.list_unavailable", name));
             return;
@@ -9788,6 +9902,13 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
             this.logger.error("Cannot restart: no config path (was startAll called?)");
             return;
         }
+        // A graceful restart keeps this manager process, so the startup probe does
+        // not run again. Without this, `agend restart` left the cached CLI env
+        // untouched and `/model` kept serving an old list until a cold start —
+        // exactly the "only stop+start works" report. Background, never blocking:
+        // /model re-probes on staleness anyway, this just makes a restart do the
+        // refreshing a user expects of it.
+        this.probeCliEnvs();
         const instanceNames = [...this.daemons.keys()];
         if (instanceNames.length === 0) {
             this.logger.info("No instances to restart");