@songsid/agend 2.1.5-beta.4 → 2.1.5-beta.6

package/dist/fleet-manager.js

@@ -7,7 +7,7 @@ import { join, dirname, basename } from "node:path";
 import { fileURLToPath } from "node:url";
 import { isDeepStrictEqual } from "node:util";
 import { getAgendHome, ensureWorkspaceGit } from "./paths.js";
-import { beginUpdateProgress as persistUpdateProgress, clearUpdateMarker, isUpdateInProgress, readUpdateProgress, setUpdateProgressStage, } from "./update-marker.js";
+import { beginFullRestartProgress as persistFullRestartProgress, beginUpdateProgress as persistUpdateProgress, clearUpdateMarker, isUpdateInProgress, readUpdateProgress, setUpdateProgressStage, updateProgressOperation, } from "./update-marker.js";
 import { formatUpdateProgress } from "./update-progress.js";
 import { sdNotify, sdNotifyBlocking } from "./sd-notify.js";
 import { readFleetMemory } from "./process-memory.js";
@@ -60,7 +60,8 @@ import { clearPausedMarker } from "./pause-marker.js";
 import { releaseProcessFleetLock } from "./fleet-lock.js";
 import { GENERAL_PAUSE_ERROR, isGeneralInstance } from "./general-instance.js";
 import { loadOrCreateWebToken, WEB_TOKEN_INVALID_MESSAGE } from "./web-auth.js";
-import { RESTART_PROGRESS_TERMINAL_TIMEOUT_MS, RestartProgress, } from "./restart-progress.js";
+import { formatRestartProgressCompletion, RESTART_PROGRESS_TERMINAL_TIMEOUT_MS, RestartProgress, } from "./restart-progress.js";
+import { launchFullRestartHelper } from "./full-restart.js";
 import { collectRedundantInstanceDefaultPaths } from "./fleet-yaml-slim.js";
 import { StormWindow } from "./storm-window.js";
 import { SpawnGate } from "./spawn-gate.js";
@@ -228,7 +229,39 @@ const CLEAR_CONFIRM_TIMEOUT_MS = 15_000;
 /** Default lifetime for long-lived nonce prompts (clear overrides this to 15s). */
 const NONCE_BUTTON_TIMEOUT_MS = 15 * 60_000;
 const TIP_BUTTON_TIMEOUT_MS = 24 * 60 * 60_000;
-const CLI_ENV_TTL_MS = 24 * 60 * 60 * 1000; // /model reads cached CLI env within 24h
+const CLI_ENV_TTL_MS = 24 * 60 * 60 * 1000; // hard validity bound for the cached CLI env
+/**
+ * How old the cached CLI env may be before `/model` re-probes it live.
+ *
+ * The cache is a file under AGEND_HOME, so it outlives the process, and the only
+ * thing that refreshed it was the startup probe. That is why a newly released
+ * model showed up only after `agend stop` + `agend start`: a bare `agend restart`
+ * signals SIGUSR2 and restarts the instances inside the *same* manager process,
+ * so nothing re-probed and `/model` kept serving a list up to 24h old.
+ */
+const CLI_ENV_FRESH_MS = 60 * 60 * 1000;
+/**
+ * Upper bound on a live probe driven by `/model`.
+ *
+ * A probe chains bounded leaves but is not itself bounded: claude-code runs
+ * `--version` (5s) then listModels then listApiModels (an 8s AbortController
+ * against api.anthropic.com), and several backends' probes carry no explicit
+ * timeout at all. Awaiting that inline would make `/model` the next thing to
+ * hang, so it races this deadline and falls back to the cached list.
+ *
+ * Must stay above CLI_PROBE_LONGEST_CHAIN_MS — the probe's bounded steps run
+ * back to back, so clearing only the longest single leaf would be false
+ * confidence: a deadline above 8s but below the 13s chain still truncates a
+ * probe that would have succeeded. Bounding the chain rather than the leaf is
+ * the lesson from the usage hang, and the tests assert against the derived
+ * chain constant so raising either step cannot silently break it.
+ *
+ * The deadline exists to stop a probe hanging forever, not to cut short one that
+ * would have finished: truncating a legitimate probe serves the previous list
+ * and hides exactly the newly released model the user opened `/model` to find.
+ * The wait is announced before it starts, so it reads as progress, not a stall.
+ */
+export const CLI_ENV_PROBE_DEADLINE_MS = 16_000;
 export class FleetManager {
     dataDir;
     static signalTarget = null;
@@ -379,6 +412,8 @@ export class FleetManager {
     updateProgressEditRunning = false;
     lastUpdateProgressText = null;
     updateCompletionTipText = null;
+    /** Injectable only to keep the chat→CLI reload hand-off deterministic in tests. */
+    fullRestartLauncher = launchFullRestartHelper;
     eventLogPruneTimer = null;
     logRotateTimer = null;
     /** Days of event/activity history to keep. */
@@ -765,6 +800,28 @@ export class FleetManager {
         child.once("error", err => this.failUpdateProgress(err.message));
         child.unref();
     }
+    async handleRestartSlash(data, adapterId) {
+        if (!this.isFleetAdmin(data.userId, adapterId)) {
+            await data.respond(t("not_authorized"));
+            return;
+        }
+        if (data.options?.mode !== "full") {
+            await data.respond(t("restart.graceful"));
+            process.kill(process.pid, "SIGUSR2");
+            return;
+        }
+        // Discord exposes only `full` as a choice. Keep a runtime check anyway: a
+        // briefly stale command schema must never turn an unknown value into SIGUSR1.
+        const messageId = await data.respond(t("restart.full_preparing"));
+        const adapter = this.adapters.get(adapterId) ?? this.adapter;
+        if (!messageId || !adapter) {
+            this.logger.error({ adapterId, hasMessageId: !!messageId }, "Full restart response could not be persisted — reload refused");
+            await data.respond(t("restart.full_launch_failed"));
+            return;
+        }
+        const chatId = String(this.getChannelConfig(adapterId)?.group_id ?? data.channelId);
+        await this.requestFullRestart(adapter, chatId, data.channelId, messageId);
+    }
     async handleTipsSlash(data, adapterId) {
         if (!this.fleetConfig)
             return;
@@ -1950,6 +2007,130 @@ export class FleetManager {
         this.updateCompletionTipText = null;
         this.startUpdateProgressMonitor(adapter);
     }
+    /** Persist the public response, wait for idle, then start the canonical service restart. */
+    async requestFullRestart(adapter, chatId, threadId, messageId) {
+        const target = {
+            adapterId: adapter.id,
+            chatId,
+            ...(threadId ? { threadId } : {}),
+            messageId,
+        };
+        // This check is synchronous with the marker write below. A second command
+        // cannot slip through between them on Node's event loop and replace the
+        // first command's delivery target or launch a competing restart helper.
+        if (this.shuttingDown || isUpdateInProgress(this.dataDir)) {
+            this.logger.warn({ adapterId: adapter.id }, "Full restart refused because another planned restart is active");
+            await this.reportFullRestartFailure(adapter, chatId, threadId, messageId, t("restart.full_busy"));
+            return false;
+        }
+        if (!persistFullRestartProgress(this.dataDir, target)) {
+            this.logger.error({ adapterId: adapter.id }, "Full restart marker could not be persisted — reload refused");
+            await this.reportFullRestartFailure(adapter, chatId, threadId, messageId, t("restart.full_launch_failed"));
+            return false;
+        }
+        const ownedMarker = readUpdateProgress(this.dataDir);
+        if (!ownedMarker) {
+            this.logger.error({ adapterId: adapter.id }, "Full restart marker disappeared after persistence — reload refused");
+            await this.reportFullRestartFailure(adapter, chatId, threadId, messageId, t("restart.full_launch_failed"));
+            return false;
+        }
+        this.lastUpdateProgressText = null;
+        this.updateCompletionTipText = null;
+        this.startUpdateProgressMonitor(adapter);
+        await this.waitForFullRestartIdleGrace();
+        // An update can begin while the idle wait yields. Never launch a second
+        // process replacement against a marker we no longer own.
+        if (this.shuttingDown || !this.isOwnedFullRestartMarker(ownedMarker.startedAt, target)) {
+            this.logger.warn({ adapterId: adapter.id }, "Full restart superseded during idle wait — reload refused");
+            if (this.isOwnedFullRestartMarker(ownedMarker.startedAt, target))
+                clearUpdateMarker(this.dataDir);
+            await this.reportFullRestartFailure(adapter, chatId, threadId, messageId, t("restart.full_busy"));
+            return false;
+        }
+        let helper;
+        try {
+            helper = await this.fullRestartLauncher();
+        }
+        catch (err) {
+            this.logger.error({ err }, "Full restart helper failed to spawn — reload refused");
+            if (this.isOwnedFullRestartMarker(ownedMarker.startedAt, target))
+                clearUpdateMarker(this.dataDir);
+            await this.reportFullRestartFailure(adapter, chatId, threadId, messageId, t("restart.full_launch_failed"));
+            return false;
+        }
+        void helper.completion.then(result => {
+            // A zero exit means the service manager accepted the restart. launchd
+            // may report that before its SIGTERM reaches us, so only an explicit
+            // helper error/non-zero exit is evidence of failure. A signal is also
+            // ambiguous under systemd because this helper shares the old cgroup.
+            if (this.shuttingDown || !this.isOwnedFullRestartMarker(ownedMarker.startedAt, target))
+                return;
+            if (!result.error && (result.code === 0 || result.code === null))
+                return;
+            const detail = result.error
+                ? "reload helper failed after launch"
+                : `reload helper exited before process hand-off (code ${result.code ?? "null"}, signal ${result.signal ?? "none"})`;
+            this.logger.error({ result }, "Full restart helper exited before the fleet began shutting down");
+            setUpdateProgressStage(this.dataDir, "failed", { error: detail });
+        });
+        return true;
+    }
+    isOwnedFullRestartMarker(startedAt, target) {
+        const marker = readUpdateProgress(this.dataDir);
+        if (!marker || marker.startedAt !== startedAt || marker.pid !== process.pid)
+            return false;
+        if (updateProgressOperation(marker.progress) !== "full-restart")
+            return false;
+        const current = marker.progress.target;
+        return current.adapterId === target.adapterId
+            && current.chatId === target.chatId
+            && current.threadId === target.threadId
+            && current.messageId === target.messageId;
+    }
+    /** Give current work the same bounded idle grace used by graceful reload. */
+    async waitForFullRestartIdleGrace() {
+        const instanceNames = [...this.daemons.keys()];
+        if (instanceNames.length === 0)
+            return;
+        const IDLE_TIMEOUT_MS = 5 * 60_000;
+        let timeoutHandle;
+        const deadline = new Promise((_, reject) => {
+            timeoutHandle = setTimeout(() => reject(new Error("Idle wait timed out after 5 minutes")), IDLE_TIMEOUT_MS);
+        });
+        try {
+            await Promise.race([
+                Promise.all(instanceNames.map(async (name) => {
+                    const daemon = this.daemons.get(name);
+                    if (!daemon)
+                        return;
+                    this.logger.info(`Full restart: waiting for ${name} to idle...`);
+                    await daemon.waitForIdle(10_000);
+                })),
+                deadline,
+            ]);
+        }
+        catch (err) {
+            this.logger.warn({ err }, "Full restart idle wait timed out — continuing with service restart");
+        }
+        finally {
+            clearTimeout(timeoutHandle);
+        }
+    }
+    async reportFullRestartFailure(adapter, chatId, threadId, messageId, text) {
+        try {
+            await adapter.editMessage(chatId, messageId, text, threadId);
+            return;
+        }
+        catch (err) {
+            this.logger.warn({ err, adapterId: adapter.id }, "Failed to edit rejected full-restart request; posting a fresh notice");
+        }
+        try {
+            await adapter.sendText(chatId, text, { threadId });
+        }
+        catch (err) {
+            this.logger.error({ err, adapterId: adapter.id }, "Failed to deliver full-restart rejection");
+        }
+    }
     failUpdateProgress(message) {
         setUpdateProgressStage(this.dataDir, "failed", { error: message });
     }
@@ -2032,6 +2213,9 @@ export class FleetManager {
             && savedUpdateProgress.progress.stage !== "complete"
             ? savedUpdateProgress
             : null;
+        const pendingProgressOperation = pendingUpdateProgress
+            ? updateProgressOperation(pendingUpdateProgress.progress)
+            : null;
         if (pendingUpdateProgress) {
             setUpdateProgressStage(this.dataDir, "starting", { version: pendingUpdateProgress.progress.version });
         }
@@ -2312,7 +2496,7 @@ export class FleetManager {
         const allEntries = Object.entries(fleet.instances);
         const generals = allEntries.filter(([_, cfg]) => cfg.general_topic);
         const others = allEntries.filter(([_, cfg]) => !cfg.general_topic);
-        const startupProgress = new RestartProgress(this.runnableStartupCount(fleet, topicMode), pendingUpdateProgress?.startedAt ?? startupStartedAt, this.logger, { mode: pendingUpdateProgress ? "update" : "restart" });
+        const startupProgress = new RestartProgress(this.runnableStartupCount(fleet, topicMode), pendingUpdateProgress?.startedAt ?? startupStartedAt, this.logger, { mode: pendingProgressOperation === "full-restart" ? "reload" : pendingUpdateProgress ? "update" : "restart" });
         if (generals.length > 0) {
             for (const [name, cfg] of generals) {
                 try {
@@ -2481,7 +2665,7 @@ export class FleetManager {
                 version: agendVersion,
                 pausedNames,
                 failedNames,
-                tipText: pendingUpdateProgress && this.tipsEnabled()
+                tipText: pendingProgressOperation === "update" && this.tipsEnabled()
                     ? (() => {
                         const tip = this.pickAvailableTip();
                         return tip ? this.formatTip(tip) : undefined;
@@ -2490,7 +2674,16 @@ export class FleetManager {
             });
             if (!progressCompleted && fleet.channel?.group_id) {
                 let text;
-                if (failedNames.length === 0 && pausedNames.length === 0) {
+                if (pendingProgressOperation === "full-restart") {
+                    text = formatRestartProgressCompletion("reload", {
+                        running: started,
+                        total,
+                        version: agendVersion,
+                        pausedNames,
+                        failedNames,
+                    }, pendingUpdateProgress.startedAt);
+                }
+                else if (failedNames.length === 0 && pausedNames.length === 0) {
                     text = t("fleet.ready", started, total, agendVersion);
                 }
                 else if (failedNames.length === 0) {
@@ -3036,13 +3229,7 @@ export class FleetManager {
                 await data.respond(this.topicCommands.getDashboardText());
             }
             else if (data.command === "restart") {
-                const allowed = this.fleetConfig?.channel?.access?.allowed_users ?? [];
-                if (allowed.length > 0 && !allowed.some(u => String(u) === String(data.userId))) {
-                    await data.respond(t("not_authorized"));
-                    return;
-                }
-                await data.respond(t("restart.graceful"));
-                process.kill(process.pid, "SIGUSR2");
+                await this.handleRestartSlash(data, adapterId);
             }
             else if (data.command === "compact") {
                 const name = this.resolveSlashTarget(data.channelId, adapterId);
@@ -3385,13 +3572,7 @@ export class FleetManager {
                 await data.respond(this.topicCommands.getDashboardText());
             }
             else if (data.command === "restart") {
-                const allowed = this.fleetConfig?.channel?.access?.allowed_users ?? [];
-                if (allowed.length > 0 && !allowed.some(u => String(u) === String(data.userId))) {
-                    await data.respond(t("not_authorized"));
-                    return;
-                }
-                await data.respond(t("restart.graceful"));
-                process.kill(process.pid, "SIGUSR2");
+                await this.handleRestartSlash(data, adapterId);
             }
             else if (data.command === "compact") {
                 const name = this.resolveSlashTarget(data.channelId, adapterId);
@@ -8423,6 +8604,32 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
         catch { /* missing / stale / corrupt */ }
         return null;
     }
+    /** True when a cached CLI env is old enough that `/model` should re-probe. */
+    cliEnvNeedsRefresh(env) {
+        return !env || typeof env.probedAt !== "number" || Date.now() - env.probedAt >= CLI_ENV_FRESH_MS;
+    }
+    /**
+     * Run a live probe under a deadline, falling back to whatever the cache holds.
+     * A model list is an aid: a vendor that stops answering must degrade to the
+     * previous list, never stall the command that asked for it.
+     */
+    async probeBackendBounded(backend) {
+        const work = this.probeBackend(backend);
+        work.catch(() => { });
+        let timer;
+        const deadline = new Promise(resolve => {
+            timer = setTimeout(() => {
+                this.logger.warn({ backend, deadlineMs: CLI_ENV_PROBE_DEADLINE_MS }, "CLI env live probe exceeded its deadline — serving the cached model list");
+                resolve(null);
+            }, CLI_ENV_PROBE_DEADLINE_MS);
+        });
+        try {
+            return await Promise.race([work, deadline]);
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    }
     /**
      * Resolve the effective model for a fleet or ClassicBot instance, plus where it
      * came from. Single source of truth for `/model` and `/ctx` — precedence:
@@ -8524,16 +8731,23 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
             void this.probeBackend(b);
     }
     /** Best-effort model list for `/model`: cached CLI env first, else live probe. Never throws. */
-    async getModelOptions(instanceName, refresh = false) {
+    async getModelOptions(instanceName, refresh = false, onLiveProbe) {
         const backendName = this.backendNameForInstance(instanceName);
-        if (!refresh) {
-            const cached = this.readCliEnv(backendName);
-            if (cached && cached.models.length)
-                return cached.models;
-        }
-        // Cache miss / stale / forced refresh → probe live (also refreshes the cache).
-        const env = await this.probeBackend(backendName);
-        return env?.models ?? [];
+        const cached = this.readCliEnv(backendName);
+        if (!refresh && cached?.models.length && !this.cliEnvNeedsRefresh(cached))
+            return cached.models;
+        // About to go to the vendor: let the caller say so. A silent 1–10s pause on
+        // an interactive command reads as another hang, which is the wrong lesson to
+        // teach a user who has just been bitten by one.
+        onLiveProbe?.();
+        // Stale, missing, or a forced refresh → probe live (also refreshes the cache).
+        // A newly released model is invisible until this runs, which is why staleness
+        // triggers it rather than waiting for the 24h hard expiry or a cold start.
+        const env = await this.probeBackendBounded(backendName);
+        if (env?.models.length)
+            return env.models;
+        // Probe failed or timed out: the previous list is still the best answer.
+        return cached?.models ?? [];
     }
     /**
      * Model catalog behind the `list_models` tool.
@@ -8824,7 +9038,9 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
             await data.respond(t("model.usage"));
             return;
         }
-        const options = await this.getModelOptions(name, isRefresh);
+        const options = await this.getModelOptions(name, isRefresh, () => {
+            void data.respond(t("model.refreshing")).catch(() => { });
+        });
         if (options.length === 0) {
             await data.respond(t("model.list_unavailable", name));
             return;
@@ -9658,8 +9874,18 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
             return;
         }
         this.logger.info(`Full restart: waiting for ${instanceNames.length} instances to idle...`);
+        const trackedProgress = readUpdateProgress(this.dataDir);
+        const trackedFullRestart = trackedProgress
+            && updateProgressOperation(trackedProgress.progress) === "full-restart"
+            && trackedProgress.progress.stage !== "failed"
+            && trackedProgress.progress.stage !== "complete";
+        if (trackedFullRestart) {
+            // `/restart full` already posted and persisted one public progress message.
+            // Keep that single message; the new process will adopt and finish it.
+            setUpdateProgressStage(this.dataDir, "stopping");
+        }
         const groupId = this.fleetConfig?.channel?.group_id;
-        if (groupId && this.adapter) {
+        if (!trackedFullRestart && groupId && this.adapter) {
             await this.adapter.sendText(String(groupId), t("restart.full_initiated"))
                 .catch(e => this.logger.warn({ err: e }, "Failed to post full restart notification"));
         }
@@ -9835,6 +10061,13 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
             this.logger.error("Cannot restart: no config path (was startAll called?)");
             return;
         }
+        // A graceful restart keeps this manager process, so the startup probe does
+        // not run again. Without this, `agend restart` left the cached CLI env
+        // untouched and `/model` kept serving an old list until a cold start —
+        // exactly the "only stop+start works" report. Background, never blocking:
+        // /model re-probes on staleness anyway, this just makes a restart do the
+        // refreshing a user expects of it.
+        this.probeCliEnvs();
         const instanceNames = [...this.daemons.keys()];
         if (instanceNames.length === 0) {
             this.logger.info("No instances to restart");