@songsid/agend 2.1.5-beta.5 → 2.1.5-beta.7

package/dist/fleet-context.d.ts

@@ -103,6 +103,8 @@ export interface FleetContext {
     /** Persist and edit one `/update` message across the fleet process restart. */
     beginUpdateProgress?(adapter: import("./channel/types.js").ChannelAdapter, chatId: string, threadId: string | undefined, messageId: string): void;
     failUpdateProgress?(message: string): void;
+    /** Persist `/restart full`, wait for idle, then launch the environment-aware service restart. */
+    requestFullRestart?(adapter: import("./channel/types.js").ChannelAdapter, chatId: string, threadId: string | undefined, messageId: string): Promise<boolean>;
     /** Post one random, not-yet-dismissed tip with a dismiss button. */
     promptTip?(generalName: string, adapter: import("./channel/types.js").ChannelAdapter, chatId: string, threadId?: string): Promise<"posted" | "empty" | "unavailable">;
     /** Persistently enable advanced tips without waiting for the dismissal threshold. */

package/dist/fleet-manager.d.ts

@@ -221,6 +221,8 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
     private updateProgressEditRunning;
     private lastUpdateProgressText;
     private updateCompletionTipText;
+    /** Injectable only to keep the chat→CLI reload hand-off deterministic in tests. */
+    private fullRestartLauncher;
     private eventLogPruneTimer;
     private logRotateTimer;
     /** Days of event/activity history to keep. */
@@ -309,6 +311,7 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
     private isModelAdmin;
     private handlePauseWakeSlash;
     private handleUpdateSlash;
+    private handleRestartSlash;
     private handleTipsSlash;
     /** Admin-only full conversation reset for fleet-topic and Classic instances. */
     private handleClearSlash;
@@ -508,6 +511,12 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
         token: string | null;
     };
     beginUpdateProgress(adapter: ChannelAdapter, chatId: string, threadId: string | undefined, messageId: string): void;
+    /** Persist the public response, wait for idle, then start the canonical service restart. */
+    requestFullRestart(adapter: ChannelAdapter, chatId: string, threadId: string | undefined, messageId: string): Promise<boolean>;
+    private isOwnedFullRestartMarker;
+    /** Give current work the same bounded idle grace used by graceful reload. */
+    private waitForFullRestartIdleGrace;
+    private reportFullRestartFailure;
     failUpdateProgress(message: string): void;
     /** The old fleet edits CLI install stages; the new fleet adopts the same message below. */
     private startUpdateProgressMonitor;

package/dist/fleet-manager.js

@@ -7,7 +7,7 @@ import { join, dirname, basename } from "node:path";
 import { fileURLToPath } from "node:url";
 import { isDeepStrictEqual } from "node:util";
 import { getAgendHome, ensureWorkspaceGit } from "./paths.js";
-import { beginUpdateProgress as persistUpdateProgress, clearUpdateMarker, isUpdateInProgress, readUpdateProgress, setUpdateProgressStage, } from "./update-marker.js";
+import { beginFullRestartProgress as persistFullRestartProgress, beginUpdateProgress as persistUpdateProgress, clearUpdateMarker, isUpdateInProgress, readUpdateProgress, setUpdateProgressStage, updateProgressOperation, } from "./update-marker.js";
 import { formatUpdateProgress } from "./update-progress.js";
 import { sdNotify, sdNotifyBlocking } from "./sd-notify.js";
 import { readFleetMemory } from "./process-memory.js";
@@ -60,7 +60,8 @@ import { clearPausedMarker } from "./pause-marker.js";
 import { releaseProcessFleetLock } from "./fleet-lock.js";
 import { GENERAL_PAUSE_ERROR, isGeneralInstance } from "./general-instance.js";
 import { loadOrCreateWebToken, WEB_TOKEN_INVALID_MESSAGE } from "./web-auth.js";
-import { RESTART_PROGRESS_TERMINAL_TIMEOUT_MS, RestartProgress, } from "./restart-progress.js";
+import { formatRestartProgressCompletion, RESTART_PROGRESS_TERMINAL_TIMEOUT_MS, RestartProgress, } from "./restart-progress.js";
+import { launchFullRestartHelper } from "./full-restart.js";
 import { collectRedundantInstanceDefaultPaths } from "./fleet-yaml-slim.js";
 import { StormWindow } from "./storm-window.js";
 import { SpawnGate } from "./spawn-gate.js";
@@ -411,6 +412,8 @@ export class FleetManager {
     updateProgressEditRunning = false;
     lastUpdateProgressText = null;
     updateCompletionTipText = null;
+    /** Injectable only to keep the chat→CLI reload hand-off deterministic in tests. */
+    fullRestartLauncher = launchFullRestartHelper;
     eventLogPruneTimer = null;
     logRotateTimer = null;
     /** Days of event/activity history to keep. */
@@ -797,6 +800,28 @@ export class FleetManager {
         child.once("error", err => this.failUpdateProgress(err.message));
         child.unref();
     }
+    async handleRestartSlash(data, adapterId) {
+        if (!this.isFleetAdmin(data.userId, adapterId)) {
+            await data.respond(t("not_authorized"));
+            return;
+        }
+        if (data.options?.mode !== "full") {
+            await data.respond(t("restart.graceful"));
+            process.kill(process.pid, "SIGUSR2");
+            return;
+        }
+        // Discord exposes only `full` as a choice. Keep a runtime check anyway: a
+        // briefly stale command schema must never turn an unknown value into SIGUSR1.
+        const messageId = await data.respond(t("restart.full_preparing"));
+        const adapter = this.adapters.get(adapterId) ?? this.adapter;
+        if (!messageId || !adapter) {
+            this.logger.error({ adapterId, hasMessageId: !!messageId }, "Full restart response could not be persisted — reload refused");
+            await data.respond(t("restart.full_launch_failed"));
+            return;
+        }
+        const chatId = String(this.getChannelConfig(adapterId)?.group_id ?? data.channelId);
+        await this.requestFullRestart(adapter, chatId, data.channelId, messageId);
+    }
     async handleTipsSlash(data, adapterId) {
         if (!this.fleetConfig)
             return;
@@ -1982,6 +2007,130 @@ export class FleetManager {
         this.updateCompletionTipText = null;
         this.startUpdateProgressMonitor(adapter);
     }
+    /** Persist the public response, wait for idle, then start the canonical service restart. */
+    async requestFullRestart(adapter, chatId, threadId, messageId) {
+        const target = {
+            adapterId: adapter.id,
+            chatId,
+            ...(threadId ? { threadId } : {}),
+            messageId,
+        };
+        // This check is synchronous with the marker write below. A second command
+        // cannot slip through between them on Node's event loop and replace the
+        // first command's delivery target or launch a competing restart helper.
+        if (this.shuttingDown || isUpdateInProgress(this.dataDir)) {
+            this.logger.warn({ adapterId: adapter.id }, "Full restart refused because another planned restart is active");
+            await this.reportFullRestartFailure(adapter, chatId, threadId, messageId, t("restart.full_busy"));
+            return false;
+        }
+        if (!persistFullRestartProgress(this.dataDir, target)) {
+            this.logger.error({ adapterId: adapter.id }, "Full restart marker could not be persisted — reload refused");
+            await this.reportFullRestartFailure(adapter, chatId, threadId, messageId, t("restart.full_launch_failed"));
+            return false;
+        }
+        const ownedMarker = readUpdateProgress(this.dataDir);
+        if (!ownedMarker) {
+            this.logger.error({ adapterId: adapter.id }, "Full restart marker disappeared after persistence — reload refused");
+            await this.reportFullRestartFailure(adapter, chatId, threadId, messageId, t("restart.full_launch_failed"));
+            return false;
+        }
+        this.lastUpdateProgressText = null;
+        this.updateCompletionTipText = null;
+        this.startUpdateProgressMonitor(adapter);
+        await this.waitForFullRestartIdleGrace();
+        // An update can begin while the idle wait yields. Never launch a second
+        // process replacement against a marker we no longer own.
+        if (this.shuttingDown || !this.isOwnedFullRestartMarker(ownedMarker.startedAt, target)) {
+            this.logger.warn({ adapterId: adapter.id }, "Full restart superseded during idle wait — reload refused");
+            if (this.isOwnedFullRestartMarker(ownedMarker.startedAt, target))
+                clearUpdateMarker(this.dataDir);
+            await this.reportFullRestartFailure(adapter, chatId, threadId, messageId, t("restart.full_busy"));
+            return false;
+        }
+        let helper;
+        try {
+            helper = await this.fullRestartLauncher();
+        }
+        catch (err) {
+            this.logger.error({ err }, "Full restart helper failed to spawn — reload refused");
+            if (this.isOwnedFullRestartMarker(ownedMarker.startedAt, target))
+                clearUpdateMarker(this.dataDir);
+            await this.reportFullRestartFailure(adapter, chatId, threadId, messageId, t("restart.full_launch_failed"));
+            return false;
+        }
+        void helper.completion.then(result => {
+            // A zero exit means the service manager accepted the restart. launchd
+            // may report that before its SIGTERM reaches us, so only an explicit
+            // helper error/non-zero exit is evidence of failure. A signal is also
+            // ambiguous under systemd because this helper shares the old cgroup.
+            if (this.shuttingDown || !this.isOwnedFullRestartMarker(ownedMarker.startedAt, target))
+                return;
+            if (!result.error && (result.code === 0 || result.code === null))
+                return;
+            const detail = result.error
+                ? "reload helper failed after launch"
+                : `reload helper exited before process hand-off (code ${result.code ?? "null"}, signal ${result.signal ?? "none"})`;
+            this.logger.error({ result }, "Full restart helper exited before the fleet began shutting down");
+            setUpdateProgressStage(this.dataDir, "failed", { error: detail });
+        });
+        return true;
+    }
+    isOwnedFullRestartMarker(startedAt, target) {
+        const marker = readUpdateProgress(this.dataDir);
+        if (!marker || marker.startedAt !== startedAt || marker.pid !== process.pid)
+            return false;
+        if (updateProgressOperation(marker.progress) !== "full-restart")
+            return false;
+        const current = marker.progress.target;
+        return current.adapterId === target.adapterId
+            && current.chatId === target.chatId
+            && current.threadId === target.threadId
+            && current.messageId === target.messageId;
+    }
+    /** Give current work the same bounded idle grace used by graceful reload. */
+    async waitForFullRestartIdleGrace() {
+        const instanceNames = [...this.daemons.keys()];
+        if (instanceNames.length === 0)
+            return;
+        const IDLE_TIMEOUT_MS = 5 * 60_000;
+        let timeoutHandle;
+        const deadline = new Promise((_, reject) => {
+            timeoutHandle = setTimeout(() => reject(new Error("Idle wait timed out after 5 minutes")), IDLE_TIMEOUT_MS);
+        });
+        try {
+            await Promise.race([
+                Promise.all(instanceNames.map(async (name) => {
+                    const daemon = this.daemons.get(name);
+                    if (!daemon)
+                        return;
+                    this.logger.info(`Full restart: waiting for ${name} to idle...`);
+                    await daemon.waitForIdle(10_000);
+                })),
+                deadline,
+            ]);
+        }
+        catch (err) {
+            this.logger.warn({ err }, "Full restart idle wait timed out — continuing with service restart");
+        }
+        finally {
+            clearTimeout(timeoutHandle);
+        }
+    }
+    async reportFullRestartFailure(adapter, chatId, threadId, messageId, text) {
+        try {
+            await adapter.editMessage(chatId, messageId, text, threadId);
+            return;
+        }
+        catch (err) {
+            this.logger.warn({ err, adapterId: adapter.id }, "Failed to edit rejected full-restart request; posting a fresh notice");
+        }
+        try {
+            await adapter.sendText(chatId, text, { threadId });
+        }
+        catch (err) {
+            this.logger.error({ err, adapterId: adapter.id }, "Failed to deliver full-restart rejection");
+        }
+    }
     failUpdateProgress(message) {
         setUpdateProgressStage(this.dataDir, "failed", { error: message });
     }
@@ -2064,6 +2213,9 @@ export class FleetManager {
             && savedUpdateProgress.progress.stage !== "complete"
             ? savedUpdateProgress
             : null;
+        const pendingProgressOperation = pendingUpdateProgress
+            ? updateProgressOperation(pendingUpdateProgress.progress)
+            : null;
         if (pendingUpdateProgress) {
             setUpdateProgressStage(this.dataDir, "starting", { version: pendingUpdateProgress.progress.version });
         }
@@ -2344,7 +2496,7 @@ export class FleetManager {
         const allEntries = Object.entries(fleet.instances);
         const generals = allEntries.filter(([_, cfg]) => cfg.general_topic);
         const others = allEntries.filter(([_, cfg]) => !cfg.general_topic);
-        const startupProgress = new RestartProgress(this.runnableStartupCount(fleet, topicMode), pendingUpdateProgress?.startedAt ?? startupStartedAt, this.logger, { mode: pendingUpdateProgress ? "update" : "restart" });
+        const startupProgress = new RestartProgress(this.runnableStartupCount(fleet, topicMode), pendingUpdateProgress?.startedAt ?? startupStartedAt, this.logger, { mode: pendingProgressOperation === "full-restart" ? "reload" : pendingUpdateProgress ? "update" : "restart" });
         if (generals.length > 0) {
             for (const [name, cfg] of generals) {
                 try {
@@ -2513,7 +2665,7 @@ export class FleetManager {
                 version: agendVersion,
                 pausedNames,
                 failedNames,
-                tipText: pendingUpdateProgress && this.tipsEnabled()
+                tipText: pendingProgressOperation === "update" && this.tipsEnabled()
                     ? (() => {
                         const tip = this.pickAvailableTip();
                         return tip ? this.formatTip(tip) : undefined;
@@ -2522,7 +2674,16 @@ export class FleetManager {
             });
             if (!progressCompleted && fleet.channel?.group_id) {
                 let text;
-                if (failedNames.length === 0 && pausedNames.length === 0) {
+                if (pendingProgressOperation === "full-restart") {
+                    text = formatRestartProgressCompletion("reload", {
+                        running: started,
+                        total,
+                        version: agendVersion,
+                        pausedNames,
+                        failedNames,
+                    }, pendingUpdateProgress.startedAt);
+                }
+                else if (failedNames.length === 0 && pausedNames.length === 0) {
                     text = t("fleet.ready", started, total, agendVersion);
                 }
                 else if (failedNames.length === 0) {
@@ -3068,13 +3229,7 @@ export class FleetManager {
                 await data.respond(this.topicCommands.getDashboardText());
             }
             else if (data.command === "restart") {
-                const allowed = this.fleetConfig?.channel?.access?.allowed_users ?? [];
-                if (allowed.length > 0 && !allowed.some(u => String(u) === String(data.userId))) {
-                    await data.respond(t("not_authorized"));
-                    return;
-                }
-                await data.respond(t("restart.graceful"));
-                process.kill(process.pid, "SIGUSR2");
+                await this.handleRestartSlash(data, adapterId);
             }
             else if (data.command === "compact") {
                 const name = this.resolveSlashTarget(data.channelId, adapterId);
@@ -3417,13 +3572,7 @@ export class FleetManager {
                 await data.respond(this.topicCommands.getDashboardText());
             }
             else if (data.command === "restart") {
-                const allowed = this.fleetConfig?.channel?.access?.allowed_users ?? [];
-                if (allowed.length > 0 && !allowed.some(u => String(u) === String(data.userId))) {
-                    await data.respond(t("not_authorized"));
-                    return;
-                }
-                await data.respond(t("restart.graceful"));
-                process.kill(process.pid, "SIGUSR2");
+                await this.handleRestartSlash(data, adapterId);
             }
             else if (data.command === "compact") {
                 const name = this.resolveSlashTarget(data.channelId, adapterId);
@@ -9725,8 +9874,18 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
             return;
         }
         this.logger.info(`Full restart: waiting for ${instanceNames.length} instances to idle...`);
+        const trackedProgress = readUpdateProgress(this.dataDir);
+        const trackedFullRestart = trackedProgress
+            && updateProgressOperation(trackedProgress.progress) === "full-restart"
+            && trackedProgress.progress.stage !== "failed"
+            && trackedProgress.progress.stage !== "complete";
+        if (trackedFullRestart) {
+            // `/restart full` already posted and persisted one public progress message.
+            // Keep that single message; the new process will adopt and finish it.
+            setUpdateProgressStage(this.dataDir, "stopping");
+        }
         const groupId = this.fleetConfig?.channel?.group_id;
-        if (groupId && this.adapter) {
+        if (!trackedFullRestart && groupId && this.adapter) {
             await this.adapter.sendText(String(groupId), t("restart.full_initiated"))
                 .catch(e => this.logger.warn({ err: e }, "Failed to post full restart notification"));
         }