@songsid/agend 2.1.2-beta.47 → 2.1.2-beta.49

package/dist/fleet-manager.js

@@ -51,6 +51,8 @@ import { readLastInboundAt } from "./daemon.js";
 import { clearPausedMarker } from "./pause-marker.js";
 import { releaseProcessFleetLock } from "./fleet-lock.js";
 import { GENERAL_PAUSE_ERROR, isGeneralInstance } from "./general-instance.js";
+import { loadOrCreateWebToken, WEB_TOKEN_INVALID_MESSAGE } from "./web-auth.js";
+import { RestartProgress } from "./restart-progress.js";
 import { getTmuxSession } from "./config.js";
 export function resolveReplyThreadId(argsThreadId, instanceConfig) {
     if (typeof argsThreadId === "string" && argsThreadId.length > 0) {
@@ -285,6 +287,7 @@ export class FleetManager {
     sseClients = new Set();
     webToken = null;
     viewToken = null;
+    healthServerListening = false;
     constructor(dataDir) {
         this.dataDir = dataDir;
         FleetManager.signalTarget = this;
@@ -1059,7 +1062,18 @@ export class FleetManager {
      * to avoid config file races. Stagger delay is group-to-group, not instance-to-instance.
      * TODO: per-instance startup timeout (existing issue, not introduced here)
      */
-    async startInstancesWithConcurrency(entries, topicMode) {
+    async startInstancesWithConcurrency(entries, topicMode, onReady) {
+        // Persisted pauses are intentionally preserved across fleet restarts. Filter
+        // them before grouping/staggering: startInstance() retains its own guard as
+        // a final backstop, but putting a no-op entry in this queue still consumes a
+        // full stagger slot for every distinct working directory.
+        const runnableEntries = entries.filter(([name]) => !this.lifecycle.isPaused(name));
+        const pausedCount = entries.length - runnableEntries.length;
+        if (pausedCount > 0) {
+            this.logger.info({ pausedCount }, "Paused instances excluded from startup queue");
+        }
+        if (runnableEntries.length === 0)
+            return;
         const raw = this.fleetConfig?.defaults?.startup;
         const explicitConcurrency = raw?.concurrency;
         const staggerMs = Math.max(0, Math.min(30_000, raw?.stagger_delay_ms ?? 500));
@@ -1074,10 +1088,10 @@ export class FleetManager {
         else {
             const freeMemMB = Math.round(freemem() / (1024 * 1024));
             concurrency = Math.max(2, Math.min(10, Math.floor(freeMemMB / ESTIMATED_MB_PER_INSTANCE)));
-            this.logger.info({ concurrency, freeMemMB: freeMemMB, totalInstances: entries.length }, "Adaptive startup concurrency");
+            this.logger.info({ concurrency, freeMemMB: freeMemMB, totalInstances: runnableEntries.length }, "Adaptive startup concurrency");
         }
         const byWorkDir = new Map();
-        for (const [name, config] of entries) {
+        for (const [name, config] of runnableEntries) {
             const dir = config.working_directory;
             if (!byWorkDir.has(dir))
                 byWorkDir.set(dir, []);
@@ -1120,7 +1134,14 @@ export class FleetManager {
                     lastStartAt = Date.now();
                     (async () => {
                         for (const [name, config] of group) {
-                            await this.startInstance(name, config, topicMode).catch((err) => this.logger.error({ err, name }, "Failed to start instance"));
+                            try {
+                                await this.startInstance(name, config, topicMode);
+                                if (this.daemons.has(name))
+                                    onReady?.(name);
+                            }
+                            catch (err) {
+                                this.logger.error({ err, name }, "Failed to start instance");
+                            }
                         }
                     })().finally(() => {
                         running--;
@@ -1134,6 +1155,34 @@ export class FleetManager {
             startNext();
         });
     }
+    runnableStartupCount(fleet, includeClassic) {
+        const names = new Set(Object.keys(fleet.instances));
+        if (includeClassic) {
+            for (const channel of this.classicChannels?.getAll() ?? [])
+                names.add(channel.instanceName);
+        }
+        let count = 0;
+        for (const name of names) {
+            if (!this.lifecycle.isPaused(name))
+                count++;
+        }
+        return count;
+    }
+    restartProgressTarget() {
+        const generalName = this.findGeneralInstance();
+        if (!generalName)
+            return null;
+        const adapter = this.getAdapterForInstance(generalName);
+        const chatId = this.getGroupIdForInstance(generalName);
+        if (!adapter || !chatId)
+            return null;
+        const topicId = this.fleetConfig?.instances[generalName]?.topic_id;
+        return {
+            adapter,
+            chatId,
+            threadId: topicId != null ? String(topicId) : undefined,
+        };
+    }
     async stopInstance(name) {
         this.failoverActive.delete(name);
         this.cancelIdleButtonRetirement(name);
@@ -1225,8 +1274,24 @@ export class FleetManager {
             process.env[key] = value;
         }
     }
+    /** Initialize auth before any adapter can answer /dashboard. */
+    initializeWebAuthTokens() {
+        this.webToken = loadOrCreateWebToken(this.dataDir);
+        this.viewToken = randomBytes(24).toString("hex");
+        const viewTokenPath = join(this.dataDir, "view.token");
+        writeFileSync(viewTokenPath, this.viewToken, { encoding: "utf8", mode: 0o600 });
+        try {
+            chmodSync(viewTokenPath, 0o600);
+        }
+        catch { /* best effort */ }
+        this.healthServerListening = false;
+    }
+    getDashboardAccess() {
+        return { ready: this.healthServerListening, token: this.webToken };
+    }
     /** Start all instances from fleet config */
     async startAll(configPath) {
+        const startupStartedAt = Date.now();
         FleetManager.signalTarget = this;
         this.startupComplete = false;
         // Cleared here, not at the end of doStopAll: a stop has an async tail, and
@@ -1238,6 +1303,7 @@ export class FleetManager {
         rotateLogIfNeeded(join(this.dataDir, "fleet.log"));
         const fleet = this.loadConfig(configPath);
         setLocale(detectLocale(fleet)); // user-facing text language (fleet.yaml defaults.locale / timezone)
+        this.initializeWebAuthTokens();
         const topicMode = fleet.channel?.mode === "topic" || !!fleet.channels?.some(ch => ch.mode === "topic");
         // Set tmux socket isolation for custom AGEND_HOME
         const { getTmuxSocketName: getSocket } = await import("./paths.js");
@@ -1493,10 +1559,13 @@ export class FleetManager {
         const allEntries = Object.entries(fleet.instances);
         const generals = allEntries.filter(([_, cfg]) => cfg.general_topic);
         const others = allEntries.filter(([_, cfg]) => !cfg.general_topic);
+        const startupProgress = new RestartProgress(this.runnableStartupCount(fleet, topicMode), startupStartedAt, this.logger);
         if (generals.length > 0) {
             for (const [name, cfg] of generals) {
                 try {
                     await this.startInstance(name, cfg, topicMode);
+                    if (this.daemons.has(name))
+                        startupProgress.markReady();
                 }
                 catch (err) {
                     this.logger.error({ err, name }, "Failed to start general instance");
@@ -1511,6 +1580,26 @@ export class FleetManager {
                 }
             }
         }
+        // The adapter must exist before General can receive the progress message.
+        // Start it after General is ready, in parallel with the remaining CLIs, so
+        // progress is visible without adding adapter startup time to the critical path.
+        let adapterStartup = null;
+        let progressStart = Promise.resolve(false);
+        if (topicMode && (fleet.channel || fleet.channels?.length)) {
+            // An adapter becoming reachable during startup can receive messages; make
+            // all existing topic ids routable before opening that inbound path.
+            this.routing.rebuild(fleet);
+            this.reregisterClassicChannels();
+            adapterStartup = (async () => {
+                try {
+                    await this.startSharedAdapter(fleet);
+                }
+                catch (err) {
+                    this.logger.error({ err }, "startSharedAdapter failed — fleet continues without some adapters");
+                }
+            })();
+            progressStart = adapterStartup.then(() => startupProgress.start(this.restartProgressTarget()));
+        }
         // The systemd watchdog answers exactly one question: is this process still
         // turning its event loop? Pinging from a timer proves that, and after the
         // blocking child-process calls were made async it is a meaningful signal —
@@ -1536,15 +1625,10 @@ export class FleetManager {
         this.logRotateTimer.unref?.();
         // Phase 2: Start remaining instances with staggered concurrency
         if (others.length > 0) {
-            await this.startInstancesWithConcurrency(others, topicMode);
+            await this.startInstancesWithConcurrency(others, topicMode, () => startupProgress.markReady());
         }
         if (topicMode && (fleet.channel || fleet.channels?.length)) {
-            try {
-                await this.startSharedAdapter(fleet);
-            }
-            catch (err) {
-                this.logger.error({ err }, "startSharedAdapter failed — fleet continues without some adapters");
-            }
+            await adapterStartup;
             // Bind every fleet instance deterministically. Explicit channel_id wins;
             // otherwise channels[0] is authoritative. Do not infer identity from
             // concurrent adapter startup or whichever bot receives a message first.
@@ -1585,18 +1669,30 @@ export class FleetManager {
             // Start classic channel instances (parallel, concurrency 3)
             if (this.classicChannels) {
                 const fleetBackend = this.fleetConfig?.defaults?.backend;
-                const channels = this.classicChannels.getAll();
+                const channels = this.classicChannels.getAll()
+                    .filter(ch => !this.lifecycle.isPaused(ch.instanceName));
                 const concurrency = 3;
                 let idx = 0;
                 while (idx < channels.length) {
                     const batch = channels.slice(idx, idx + concurrency);
-                    await Promise.allSettled(batch.map(ch => this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after)).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance"))));
+                    await Promise.allSettled(batch.map(async (ch) => {
+                        try {
+                            await this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
+                            if (this.daemons.has(ch.instanceName))
+                                startupProgress.markReady();
+                        }
+                        catch (err) {
+                            this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance");
+                        }
+                    }));
                     idx += concurrency;
                 }
             }
             for (const name of Object.keys(fleet.instances)) {
                 this.startStatuslineWatcher(name);
             }
+            await progressStart;
+            const progressCompleted = await startupProgress.finish();
             // Notify General topic that fleet is up
             const classicCount = this.classicChannels?.getAll().length ?? 0;
             const total = Object.keys(fleet.instances).length + classicCount;
@@ -1609,7 +1705,7 @@ export class FleetManager {
             const { createRequire } = await import("node:module");
             const _require = createRequire(import.meta.url);
             const agendVersion = _require("../package.json").version ?? "unknown";
-            if (this.adapter && fleet.channel?.group_id) {
+            if (!progressCompleted && this.adapter && fleet.channel?.group_id) {
                 let text;
                 if (failedNames.length === 0 && pausedNames.length === 0) {
                     text = t("fleet.ready", started, total, agendVersion);
@@ -6331,6 +6427,7 @@ When users create specialized instances, suggest these configurations:
         this.controlClient?.stop();
         this.controlClient = null;
         if (this.healthServer) {
+            this.healthServerListening = false;
             this.healthServer.close();
             this.healthServer = null;
         }
@@ -6588,6 +6685,10 @@ When users create specialized instances, suggest these configurations:
             clearTimeout(timeoutHandle);
         }
         this.logger.info("All instances idle — restarting...");
+        const restartStartedAt = Date.now();
+        // Capture the live adapter/topic before General's daemon is stopped. The
+        // channel adapter remains connected throughout an in-process restart.
+        const progressTarget = this.restartProgressTarget();
         this.clearStatuslineWatchers();
         for (const [, ipc] of this.instanceIpcClients) {
             await ipc.close();
@@ -6612,15 +6713,25 @@ When users create specialized instances, suggest these configurations:
         const fleet = this.loadConfig(this.configPath);
         this.fleetConfig = fleet;
         const topicMode = fleet.channel?.mode === "topic" || !!fleet.channels?.some(ch => ch.mode === "topic");
+        const restartProgress = new RestartProgress(this.runnableStartupCount(fleet, topicMode), restartStartedAt, this.logger);
         // Phase 1: generals first
         const restartEntries = Object.entries(fleet.instances);
         const restartGenerals = restartEntries.filter(([_, cfg]) => cfg.general_topic);
         const restartOthers = restartEntries.filter(([_, cfg]) => !cfg.general_topic);
         for (const [name, cfg] of restartGenerals) {
-            await this.startInstance(name, cfg, topicMode).catch(err => this.logger.error({ err, name }, "Failed to start general instance"));
+            try {
+                await this.startInstance(name, cfg, topicMode);
+                if (this.daemons.has(name))
+                    restartProgress.markReady();
+            }
+            catch (err) {
+                this.logger.error({ err, name }, "Failed to start general instance");
+            }
         }
+        // General is ready again; now its topic can own the live progress message.
+        await restartProgress.start(progressTarget);
         if (restartOthers.length > 0) {
-            await this.startInstancesWithConcurrency(restartOthers, topicMode);
+            await this.startInstancesWithConcurrency(restartOthers, topicMode, () => restartProgress.markReady());
         }
         if (topicMode) {
             this.routing.rebuild(this.fleetConfig);
@@ -6629,12 +6740,22 @@ When users create specialized instances, suggest these configurations:
             // Restart classic channel instances (killed during orphan cleanup)
             if (this.classicChannels) {
                 const fleetBackend = this.fleetConfig?.defaults?.backend;
-                const channels = this.classicChannels.getAll();
+                const channels = this.classicChannels.getAll()
+                    .filter(ch => !this.lifecycle.isPaused(ch.instanceName));
                 const concurrency = 3;
                 let idx = 0;
                 while (idx < channels.length) {
                     const batch = channels.slice(idx, idx + concurrency);
-                    await Promise.allSettled(batch.map(ch => this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after)).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance"))));
+                    await Promise.allSettled(batch.map(async (ch) => {
+                        try {
+                            await this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
+                            if (this.daemons.has(ch.instanceName))
+                                restartProgress.markReady();
+                        }
+                        catch (err) {
+                            this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance");
+                        }
+                    }));
                     idx += concurrency;
                 }
             }
@@ -6643,6 +6764,7 @@ When users create specialized instances, suggest these configurations:
             }
         }
         this.logger.info("Graceful restart complete");
+        const progressCompleted = await restartProgress.finish();
         if (groupId && this.adapter) {
             const total = Object.keys(fleet.instances).length;
             const started = this.daemons.size;
@@ -6663,8 +6785,10 @@ When users create specialized instances, suggest these configurations:
                 restartText = t("fleet.ready_with_failed", started, total, agendVersion2, failedNames.join(", "))
                     + (pausedNames2.length > 0 ? `\n⏸ Paused: ${pausedNames2.join(", ")}` : "");
             }
-            await this.adapter.sendText(String(groupId), restartText, notifyOpts)
-                .catch(e => this.logger.warn({ err: e }, "Failed to post restart completion notification"));
+            if (!progressCompleted) {
+                await this.adapter.sendText(String(groupId), restartText, notifyOpts)
+                    .catch(e => this.logger.warn({ err: e }, "Failed to post restart completion notification"));
+            }
             // Notify each instance's channel — staggered to avoid rate limit storm
             const instances = Object.entries(this.fleetConfig?.instances ?? {});
             this.logger.info({ count: instances.length }, "Sending restart notification to instances (staggered)");
@@ -6799,26 +6923,11 @@ When users create specialized instances, suggest these configurations:
     // ── Health HTTP endpoint ─────────────────────────────────────────────
     startHealthServer(port) {
         this.startedAt = Date.now();
-        // Generate web token before server starts so auth is enforced from the first request.
-        this.webToken = randomBytes(24).toString("hex");
-        const tokenPath = join(this.dataDir, "web.token");
-        writeFileSync(tokenPath, this.webToken, { mode: 0o600 });
-        // Defensive: if file existed previously with looser perms, tighten it.
-        try {
-            chmodSync(tokenPath, 0o600);
-        }
-        catch {
-            // best-effort
-        }
-        // Separate read-only token for the /view page: grants terminal-view + profile
-        // read, but never write (POSTs still require the full web token).
-        this.viewToken = randomBytes(24).toString("hex");
-        const viewTokenPath = join(this.dataDir, "view.token");
-        writeFileSync(viewTokenPath, this.viewToken, { mode: 0o600 });
-        try {
-            chmodSync(viewTokenPath, 0o600);
-        }
-        catch { /* best-effort */ }
+        this.healthServerListening = false;
+        this.healthPortRetried = false;
+        // Defensive for direct/unit callers; normal startup initializes these before adapters.
+        if (!this.webToken || !this.viewToken)
+            this.initializeWebAuthTokens();
         this.healthServer = createServer((req, res) => {
             res.setHeader("Content-Type", "application/json");
             // Public health probe — no auth required.
@@ -6845,7 +6954,7 @@ When users create specialized instances, suggest these configurations:
                     ?? (typeof headerToken === "string" ? headerToken : null);
                 if (!this.webToken || providedToken !== this.webToken) {
                     res.writeHead(401);
-                    res.end(JSON.stringify({ error: "Unauthorized" }));
+                    res.end(JSON.stringify({ error: WEB_TOKEN_INVALID_MESSAGE }));
                     return;
                 }
             }
@@ -7060,10 +7169,20 @@ When users create specialized instances, suggest these configurations:
             res.writeHead(404);
             res.end(JSON.stringify({ error: "not found" }));
         });
+        const markListening = (afterTakeover = false) => {
+            this.healthServerListening = true;
+            this.logger.info({ port }, afterTakeover
+                ? "Health endpoint listening (after takeover)"
+                : "Health endpoint listening");
+            this.logger.info({ url: `http://localhost:${port}/ui?token=${this.webToken}` }, "Web UI available");
+            this.logger.info({ url: `http://localhost:${port}/view?token=${this.viewToken}` }, "Web View available");
+        };
         this.healthServer.on("error", (err) => {
+            this.healthServerListening = false;
             if (err.code === "EADDRINUSE") {
                 if (this.healthPortRetried) {
-                    this.logger.debug({ port }, "Health port still in use after takeover — skipping health endpoint");
+                    this.logger.error({ err, port }, "Health port still in use after takeover — dashboard disabled");
+                    this.notifyFleetError(`⚠️ Dashboard unavailable — health port ${port} is already in use. Stop the conflicting process or configure a different health_port.`);
                     return;
                 }
                 this.healthPortRetried = true;
@@ -7084,19 +7203,14 @@ When users create specialized instances, suggest these configurations:
                 setTimeout(() => {
                     if (!this.healthServer)
                         return;
-                    this.healthServer.listen(port, "127.0.0.1", () => {
-                        this.logger.info({ port }, "Health endpoint listening (after takeover)");
-                    });
+                    this.healthServer.listen(port, "127.0.0.1", () => markListening(true));
                 }, 1500);
                 return;
             }
             this.logger.error({ err, port }, "Health server error");
+            this.notifyFleetError(`⚠️ Dashboard unavailable — health server failed: ${err.message}`);
         });
-        this.healthServer.listen(port, "127.0.0.1", () => {
-            this.logger.info({ port }, "Health endpoint listening");
-        });
-        this.logger.info({ url: `http://localhost:${port}/ui?token=${this.webToken}` }, "Web UI available");
-        this.logger.info({ url: `http://localhost:${port}/view?token=${this.viewToken}` }, "Web View available");
+        this.healthServer.listen(port, "127.0.0.1", () => markListening());
     }
     getUiStatus() {
         const fleetNames = Object.keys(this.fleetConfig?.instances ?? {});