@songsid/agend 2.1.2-beta.48 → 2.1.2-beta.49

package/dist/fleet-manager.d.ts

@@ -303,6 +303,8 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
      * TODO: per-instance startup timeout (existing issue, not introduced here)
      */
     private startInstancesWithConcurrency;
+    private runnableStartupCount;
+    private restartProgressTarget;
     stopInstance(name: string): Promise<void>;
     /** Restart a single instance, reloading fleet.yaml first to pick up config changes. */
     restartSingleInstance(name: string, opts?: {

package/dist/fleet-manager.js

@@ -52,6 +52,7 @@ import { clearPausedMarker } from "./pause-marker.js";
 import { releaseProcessFleetLock } from "./fleet-lock.js";
 import { GENERAL_PAUSE_ERROR, isGeneralInstance } from "./general-instance.js";
 import { loadOrCreateWebToken, WEB_TOKEN_INVALID_MESSAGE } from "./web-auth.js";
+import { RestartProgress } from "./restart-progress.js";
 import { getTmuxSession } from "./config.js";
 export function resolveReplyThreadId(argsThreadId, instanceConfig) {
     if (typeof argsThreadId === "string" && argsThreadId.length > 0) {
@@ -1061,7 +1062,7 @@ export class FleetManager {
      * to avoid config file races. Stagger delay is group-to-group, not instance-to-instance.
      * TODO: per-instance startup timeout (existing issue, not introduced here)
      */
-    async startInstancesWithConcurrency(entries, topicMode) {
+    async startInstancesWithConcurrency(entries, topicMode, onReady) {
         // Persisted pauses are intentionally preserved across fleet restarts. Filter
         // them before grouping/staggering: startInstance() retains its own guard as
         // a final backstop, but putting a no-op entry in this queue still consumes a
@@ -1133,7 +1134,14 @@ export class FleetManager {
                     lastStartAt = Date.now();
                     (async () => {
                         for (const [name, config] of group) {
-                            await this.startInstance(name, config, topicMode).catch((err) => this.logger.error({ err, name }, "Failed to start instance"));
+                            try {
+                                await this.startInstance(name, config, topicMode);
+                                if (this.daemons.has(name))
+                                    onReady?.(name);
+                            }
+                            catch (err) {
+                                this.logger.error({ err, name }, "Failed to start instance");
+                            }
                         }
                     })().finally(() => {
                         running--;
@@ -1147,6 +1155,34 @@ export class FleetManager {
             startNext();
         });
     }
+    runnableStartupCount(fleet, includeClassic) {
+        const names = new Set(Object.keys(fleet.instances));
+        if (includeClassic) {
+            for (const channel of this.classicChannels?.getAll() ?? [])
+                names.add(channel.instanceName);
+        }
+        let count = 0;
+        for (const name of names) {
+            if (!this.lifecycle.isPaused(name))
+                count++;
+        }
+        return count;
+    }
+    restartProgressTarget() {
+        const generalName = this.findGeneralInstance();
+        if (!generalName)
+            return null;
+        const adapter = this.getAdapterForInstance(generalName);
+        const chatId = this.getGroupIdForInstance(generalName);
+        if (!adapter || !chatId)
+            return null;
+        const topicId = this.fleetConfig?.instances[generalName]?.topic_id;
+        return {
+            adapter,
+            chatId,
+            threadId: topicId != null ? String(topicId) : undefined,
+        };
+    }
     async stopInstance(name) {
         this.failoverActive.delete(name);
         this.cancelIdleButtonRetirement(name);
@@ -1255,6 +1291,7 @@ export class FleetManager {
     }
     /** Start all instances from fleet config */
     async startAll(configPath) {
+        const startupStartedAt = Date.now();
         FleetManager.signalTarget = this;
         this.startupComplete = false;
         // Cleared here, not at the end of doStopAll: a stop has an async tail, and
@@ -1522,10 +1559,13 @@ export class FleetManager {
         const allEntries = Object.entries(fleet.instances);
         const generals = allEntries.filter(([_, cfg]) => cfg.general_topic);
         const others = allEntries.filter(([_, cfg]) => !cfg.general_topic);
+        const startupProgress = new RestartProgress(this.runnableStartupCount(fleet, topicMode), startupStartedAt, this.logger);
         if (generals.length > 0) {
             for (const [name, cfg] of generals) {
                 try {
                     await this.startInstance(name, cfg, topicMode);
+                    if (this.daemons.has(name))
+                        startupProgress.markReady();
                 }
                 catch (err) {
                     this.logger.error({ err, name }, "Failed to start general instance");
@@ -1540,6 +1580,26 @@ export class FleetManager {
                 }
             }
         }
+        // The adapter must exist before General can receive the progress message.
+        // Start it after General is ready, in parallel with the remaining CLIs, so
+        // progress is visible without adding adapter startup time to the critical path.
+        let adapterStartup = null;
+        let progressStart = Promise.resolve(false);
+        if (topicMode && (fleet.channel || fleet.channels?.length)) {
+            // An adapter becoming reachable during startup can receive messages; make
+            // all existing topic ids routable before opening that inbound path.
+            this.routing.rebuild(fleet);
+            this.reregisterClassicChannels();
+            adapterStartup = (async () => {
+                try {
+                    await this.startSharedAdapter(fleet);
+                }
+                catch (err) {
+                    this.logger.error({ err }, "startSharedAdapter failed — fleet continues without some adapters");
+                }
+            })();
+            progressStart = adapterStartup.then(() => startupProgress.start(this.restartProgressTarget()));
+        }
         // The systemd watchdog answers exactly one question: is this process still
         // turning its event loop? Pinging from a timer proves that, and after the
         // blocking child-process calls were made async it is a meaningful signal —
@@ -1565,15 +1625,10 @@ export class FleetManager {
         this.logRotateTimer.unref?.();
         // Phase 2: Start remaining instances with staggered concurrency
         if (others.length > 0) {
-            await this.startInstancesWithConcurrency(others, topicMode);
+            await this.startInstancesWithConcurrency(others, topicMode, () => startupProgress.markReady());
         }
         if (topicMode && (fleet.channel || fleet.channels?.length)) {
-            try {
-                await this.startSharedAdapter(fleet);
-            }
-            catch (err) {
-                this.logger.error({ err }, "startSharedAdapter failed — fleet continues without some adapters");
-            }
+            await adapterStartup;
             // Bind every fleet instance deterministically. Explicit channel_id wins;
             // otherwise channels[0] is authoritative. Do not infer identity from
             // concurrent adapter startup or whichever bot receives a message first.
@@ -1614,18 +1669,30 @@ export class FleetManager {
             // Start classic channel instances (parallel, concurrency 3)
             if (this.classicChannels) {
                 const fleetBackend = this.fleetConfig?.defaults?.backend;
-                const channels = this.classicChannels.getAll();
+                const channels = this.classicChannels.getAll()
+                    .filter(ch => !this.lifecycle.isPaused(ch.instanceName));
                 const concurrency = 3;
                 let idx = 0;
                 while (idx < channels.length) {
                     const batch = channels.slice(idx, idx + concurrency);
-                    await Promise.allSettled(batch.map(ch => this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after)).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance"))));
+                    await Promise.allSettled(batch.map(async (ch) => {
+                        try {
+                            await this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
+                            if (this.daemons.has(ch.instanceName))
+                                startupProgress.markReady();
+                        }
+                        catch (err) {
+                            this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance");
+                        }
+                    }));
                     idx += concurrency;
                 }
             }
             for (const name of Object.keys(fleet.instances)) {
                 this.startStatuslineWatcher(name);
             }
+            await progressStart;
+            const progressCompleted = await startupProgress.finish();
             // Notify General topic that fleet is up
             const classicCount = this.classicChannels?.getAll().length ?? 0;
             const total = Object.keys(fleet.instances).length + classicCount;
@@ -1638,7 +1705,7 @@ export class FleetManager {
             const { createRequire } = await import("node:module");
             const _require = createRequire(import.meta.url);
             const agendVersion = _require("../package.json").version ?? "unknown";
-            if (this.adapter && fleet.channel?.group_id) {
+            if (!progressCompleted && this.adapter && fleet.channel?.group_id) {
                 let text;
                 if (failedNames.length === 0 && pausedNames.length === 0) {
                     text = t("fleet.ready", started, total, agendVersion);
@@ -6618,6 +6685,10 @@ When users create specialized instances, suggest these configurations:
             clearTimeout(timeoutHandle);
         }
         this.logger.info("All instances idle — restarting...");
+        const restartStartedAt = Date.now();
+        // Capture the live adapter/topic before General's daemon is stopped. The
+        // channel adapter remains connected throughout an in-process restart.
+        const progressTarget = this.restartProgressTarget();
         this.clearStatuslineWatchers();
         for (const [, ipc] of this.instanceIpcClients) {
             await ipc.close();
@@ -6642,15 +6713,25 @@ When users create specialized instances, suggest these configurations:
         const fleet = this.loadConfig(this.configPath);
         this.fleetConfig = fleet;
         const topicMode = fleet.channel?.mode === "topic" || !!fleet.channels?.some(ch => ch.mode === "topic");
+        const restartProgress = new RestartProgress(this.runnableStartupCount(fleet, topicMode), restartStartedAt, this.logger);
         // Phase 1: generals first
         const restartEntries = Object.entries(fleet.instances);
         const restartGenerals = restartEntries.filter(([_, cfg]) => cfg.general_topic);
         const restartOthers = restartEntries.filter(([_, cfg]) => !cfg.general_topic);
         for (const [name, cfg] of restartGenerals) {
-            await this.startInstance(name, cfg, topicMode).catch(err => this.logger.error({ err, name }, "Failed to start general instance"));
+            try {
+                await this.startInstance(name, cfg, topicMode);
+                if (this.daemons.has(name))
+                    restartProgress.markReady();
+            }
+            catch (err) {
+                this.logger.error({ err, name }, "Failed to start general instance");
+            }
         }
+        // General is ready again; now its topic can own the live progress message.
+        await restartProgress.start(progressTarget);
         if (restartOthers.length > 0) {
-            await this.startInstancesWithConcurrency(restartOthers, topicMode);
+            await this.startInstancesWithConcurrency(restartOthers, topicMode, () => restartProgress.markReady());
         }
         if (topicMode) {
             this.routing.rebuild(this.fleetConfig);
@@ -6659,12 +6740,22 @@ When users create specialized instances, suggest these configurations:
             // Restart classic channel instances (killed during orphan cleanup)
             if (this.classicChannels) {
                 const fleetBackend = this.fleetConfig?.defaults?.backend;
-                const channels = this.classicChannels.getAll();
+                const channels = this.classicChannels.getAll()
+                    .filter(ch => !this.lifecycle.isPaused(ch.instanceName));
                 const concurrency = 3;
                 let idx = 0;
                 while (idx < channels.length) {
                     const batch = channels.slice(idx, idx + concurrency);
-                    await Promise.allSettled(batch.map(ch => this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after)).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance"))));
+                    await Promise.allSettled(batch.map(async (ch) => {
+                        try {
+                            await this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
+                            if (this.daemons.has(ch.instanceName))
+                                restartProgress.markReady();
+                        }
+                        catch (err) {
+                            this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance");
+                        }
+                    }));
                     idx += concurrency;
                 }
             }
@@ -6673,6 +6764,7 @@ When users create specialized instances, suggest these configurations:
             }
         }
         this.logger.info("Graceful restart complete");
+        const progressCompleted = await restartProgress.finish();
         if (groupId && this.adapter) {
             const total = Object.keys(fleet.instances).length;
             const started = this.daemons.size;
@@ -6693,8 +6785,10 @@ When users create specialized instances, suggest these configurations:
                 restartText = t("fleet.ready_with_failed", started, total, agendVersion2, failedNames.join(", "))
                     + (pausedNames2.length > 0 ? `\n⏸ Paused: ${pausedNames2.join(", ")}` : "");
             }
-            await this.adapter.sendText(String(groupId), restartText, notifyOpts)
-                .catch(e => this.logger.warn({ err: e }, "Failed to post restart completion notification"));
+            if (!progressCompleted) {
+                await this.adapter.sendText(String(groupId), restartText, notifyOpts)
+                    .catch(e => this.logger.warn({ err: e }, "Failed to post restart completion notification"));
+            }
             // Notify each instance's channel — staggered to avoid rate limit storm
             const instances = Object.entries(this.fleetConfig?.instances ?? {});
             this.logger.info({ count: instances.length }, "Sending restart notification to instances (staggered)");