@songsid/agend 2.1.4-beta.47 → 2.1.4-beta.49

package/dist/fleet-manager.js

@@ -62,6 +62,7 @@ import { RestartProgress } from "./restart-progress.js";
 import { collectRedundantInstanceDefaultPaths } from "./fleet-yaml-slim.js";
 import { StormWindow } from "./storm-window.js";
 import { SpawnGate } from "./spawn-gate.js";
+import { BackendOutageTracker } from "./backend-outage.js";
 import { canUnlockAdvancedTips, DailyTipScheduler, selectTip, visibleTipLevels, } from "./tips.js";
 import { getTmuxSession } from "./config.js";
 export function resolveReplyThreadId(argsThreadId, instanceConfig) {
@@ -216,6 +217,7 @@ const CLEAR_CONFIRM_CALLBACK_PREFIX = "clear-confirm:";
 const TIP_DISMISS_CALLBACK_PREFIX = "tip-dismiss:";
 const TIP_UNLOCK_CALLBACK_PREFIX = "tip-unlock:";
 const LOGIN_CALLBACK_PREFIX = "login:";
+const INSTALL_CALLBACK_PREFIX = "install-select:";
 const LOGIN_MENU_CALLBACK_PREFIX = "login-menu:";
 const LOGIN_CONFIRM_CALLBACK_PREFIX = "login-confirm:";
 const INSTALL_LOGIN_CALLBACK_PREFIX = "install-login:";
@@ -232,6 +234,8 @@ export class FleetManager {
     lifecycle;
     stormWindow;
     spawnGate;
+    /** Fleet-level backend reachability memory (fed by pty_error / startup panes). */
+    backendOutage = new BackendOutageTracker();
     /** Live view of lifecycle.daemons — used throughout; not deprecated. */
     get daemons() { return this.lifecycle.daemons; }
     fleetConfig = null;
@@ -313,6 +317,28 @@ export class FleetManager {
     ipcWaitTails = new Map();
     /** instanceName → restart currently executing; concurrent callers join it. */
     restartsInFlight = new Map();
+    /**
+     * Delayed automatic retries for instances whose startup failed. Before this a
+     * failed start was logged once and the instance stayed `stopped` until an
+     * operator noticed (2026-09-03: 4 kiro instances, all victims of the same
+     * backend outage during a post-update herd). instanceName → pending retry.
+     */
+    startupRetries = new Map();
+    /** Bumped by every explicit stop (incl. the stop half of a restart); fences the outage hand-off. */
+    explicitStopGeneration = new Map();
+    /** instanceName → outage hand-off currently executing; a repeat joins it. */
+    handOffsInFlight = new Map();
+    /** instanceName → explicit stop currently executing; the outage hand-off waits for it. */
+    stopsInFlight = new Map();
+    /** Aggregation window for the "N instances failed to start" notice. */
+    startupRetryNotices = new Map();
+    /** Backoff between automatic startup retries; the last step repeats while the backend is down. */
+    static STARTUP_RETRY_BACKOFF_MS = [60_000, 5 * 60_000, 15 * 60_000];
+    /** Hard cap on automatic startup retries (3 backoff steps + up to 3 more during a backend outage). */
+    static STARTUP_RETRY_MAX_ATTEMPTS = 6;
+    /** Re-check interval when a retry is due but the tmux storm window still blocks spawns. */
+    static STARTUP_RETRY_STORM_DEFER_MS = 60_000;
+    static STARTUP_RETRY_NOTICE_AGGREGATE_MS = 1_000;
     // Last user message delivered to each instance — used to react ✅ on completion.
     lastInboundMsg = new Map();
     topicArchiver;
@@ -1324,6 +1350,9 @@ export class FleetManager {
      * across a fleet restart.
      */
     resumePaused = false) {
+        // Any start supersedes a pending automatic retry (it would otherwise fire
+        // into a running instance — harmless, but noisy — or race this start).
+        this.cancelStartupRetry(name);
         if (resumePaused && this.lifecycle.isPaused(name)) {
             await this.lifecycle.wake(name, 30_000);
             // A successful wake clears the persisted pause marker and produces a
@@ -1431,15 +1460,231 @@ export class FleetManager {
             workingDirectory: config.working_directory,
             reason: "startup",
         }, async () => {
-            try {
-                await this.startInstance(name, config, topicMode);
-                if (this.daemons.has(name))
-                    onReady?.(name);
+            if (await this.startInstanceUnattended(name, config, topicMode, "instance"))
+                onReady?.(name);
+        })));
+    }
+    /**
+     * Start from an UNATTENDED path (fleet startup, full restart, config
+     * reconcile): nobody is watching the result, so a failure is logged and
+     * handed to the delayed automatic retry instead of leaving the instance
+     * `stopped` forever. Explicit operator/API starts call startInstance directly
+     * and keep their synchronous error. Returns whether the instance is up.
+     */
+    async startInstanceUnattended(name, config, topicMode, what) {
+        try {
+            await this.startInstance(name, config, topicMode);
+            return this.daemons.has(name);
+        }
+        catch (err) {
+            this.logger.error({ err, name }, `Failed to start ${what}`);
+            this.scheduleStartupRetry(name, 0);
+            return false;
+        }
+    }
+    // ── Delayed automatic startup retries ──────────────────────────────────
+    /**
+     * Schedule attempt `attempt` (0-based) of the automatic startup retry for an
+     * instance whose start just failed. Backoff 1 → 5 → 15 min; while the
+     * instance's backend is known to be unreachable the 15-min step repeats up to
+     * STARTUP_RETRY_MAX_ATTEMPTS, after which we give up with one notice. Each
+     * retry re-checks the world (still configured, not running, not paused, no
+     * tmux storm) and runs through the SpawnGate as "recovery" — so a herd of
+     * failed instances comes back at the gate's concurrency, never all at once.
+     */
+    scheduleStartupRetry(name, attempt) {
+        if (this.shuttingDown)
+            return;
+        if (this.startupRetries.has(name))
+            return;
+        const backoff = FleetManager.STARTUP_RETRY_BACKOFF_MS;
+        const outage = this.backendOutage.isActive(this.backendNameOf(name));
+        const exhausted = attempt >= backoff.length && !(outage && attempt < FleetManager.STARTUP_RETRY_MAX_ATTEMPTS);
+        if (attempt >= FleetManager.STARTUP_RETRY_MAX_ATTEMPTS || exhausted) {
+            this.logger.error({ name, attempts: attempt }, "Giving up automatic startup retries");
+            this.queueStartupRetryNotice("gave_up", name, 0);
+            return;
+        }
+        const delayMs = backoff[Math.min(attempt, backoff.length - 1)];
+        const timer = setTimeout(() => { void this.runStartupRetry(name, attempt); }, delayMs);
+        timer.unref?.();
+        this.startupRetries.set(name, { attempt, timer });
+        this.logger.warn({ name, attempt: attempt + 1, delayMs, backendOutage: outage }, "Startup failed — automatic retry scheduled");
+        if (attempt === 0)
+            this.queueStartupRetryNotice("scheduled", name, delayMs);
+    }
+    /** Drop a pending automatic retry (an operator start/stop/restart supersedes it). */
+    cancelStartupRetry(name) {
+        const pending = this.startupRetries.get(name);
+        if (!pending)
+            return;
+        clearTimeout(pending.timer);
+        this.startupRetries.delete(name);
+        this.logger.info({ name }, "Pending automatic startup retry cancelled");
+    }
+    /** Pending automatic retry, if any (status display / tests). */
+    pendingStartupRetry(name) {
+        const pending = this.startupRetries.get(name);
+        return pending ? { attempt: pending.attempt } : null;
+    }
+    /**
+     * A daemon's crash-respawn hit the backend outage (startup_backend_unreachable):
+     * stop THAT daemon and schedule the delayed retry. Serialized against the
+     * operator paths: an in-flight restart is awaited first (it replaces the
+     * daemon itself), the stop is identity-checked so a fresh daemon registered
+     * meanwhile is never deleted, and an explicit stop/restart that began during
+     * the hand-off (generation bump) owns the outcome — no retry is scheduled
+     * behind an operator's back.
+     */
+    async handOffToStartupRetry(name, daemon) {
+        const joined = this.handOffsInFlight.get(name);
+        if (joined)
+            return joined;
+        const run = (async () => {
+            // Let any operator restart/stop that is already executing finish first:
+            // a restart replaces the daemon itself, a stop removes it — either way
+            // the identity check below then sees "not ours any more" and we do
+            // nothing. Loop: one operation can be chained behind another.
+            for (;;) {
+                const inFlight = this.restartsInFlight.get(name) ?? this.stopsInFlight.get(name);
+                if (!inFlight)
+                    break;
+                await inFlight.catch(() => { });
             }
-            catch (err) {
-                this.logger.error({ err, name }, "Failed to start instance");
+            const stopGen = this.explicitStopGeneration.get(name) ?? 0;
+            if (this.lifecycle.daemons.get(name) !== daemon)
+                return; // already replaced or stopped
+            await this.lifecycle.stopIfCurrent(name, daemon);
+            if (this.shuttingDown)
+                return;
+            if ((this.explicitStopGeneration.get(name) ?? 0) !== stopGen) {
+                this.logger.info({ name }, "Outage hand-off superseded by an explicit stop/restart — no automatic retry");
+                return;
             }
-        })));
+            if (this.lifecycle.daemons.has(name) || this.lifecycle.isPaused(name))
+                return;
+            this.scheduleStartupRetry(name, 0);
+        })().finally(() => this.handOffsInFlight.delete(name));
+        this.handOffsInFlight.set(name, run);
+        return run;
+    }
+    async runStartupRetry(name, attempt) {
+        this.startupRetries.delete(name);
+        if (this.shuttingDown)
+            return;
+        if (!this.isConfiguredInstance(name))
+            return; // removed from fleet.yaml / classic channel meanwhile
+        if (this.lifecycle.isPaused(name) || this.daemons.has(name))
+            return; // operator acted meanwhile
+        if (this.stormWindow.isSpawnBlocked()) {
+            // The tmux server is being restarted; joining that herd is what we are
+            // trying to avoid. Same attempt again after the storm's own recovery.
+            const timer = setTimeout(() => { void this.runStartupRetry(name, attempt); }, FleetManager.STARTUP_RETRY_STORM_DEFER_MS);
+            timer.unref?.();
+            this.startupRetries.set(name, { attempt, timer });
+            this.logger.info({ name, attempt: attempt + 1 }, "Startup retry deferred — tmux storm window is blocking spawns");
+            return;
+        }
+        const topicMode = this.fleetConfig?.channel?.mode === "topic"
+            || !!this.fleetConfig?.channels?.some(channel => channel.mode === "topic");
+        try {
+            await this.spawnGate.run({
+                instanceName: name,
+                workingDirectory: this.fleetConfig?.instances[name]?.working_directory || this.getInstanceDir(name),
+                reason: "recovery",
+            }, () => this.startConfiguredInstance(name, topicMode));
+            if (this.daemons.has(name)) {
+                this.logger.info({ name, attempt: attempt + 1 }, "Automatic startup retry succeeded");
+            }
+        }
+        catch (err) {
+            this.logger.error({ err, name, attempt: attempt + 1 }, "Automatic startup retry failed");
+            this.scheduleStartupRetry(name, attempt + 1);
+        }
+    }
+    /** Fleet-topic (fleet.yaml) or ClassicBot (classic channel) — both are retryable; anything else is gone. */
+    isConfiguredInstance(name) {
+        if (this.fleetConfig?.instances[name])
+            return true;
+        return !!this.classicChannels?.getAll().some(channel => channel.instanceName === name);
+    }
+    /**
+     * Kind-aware start for the automatic retry: fleet-topic instances come from
+     * fleet.yaml, ClassicBot instances exist only in the classic channel manager
+     * and must be rebuilt through startClassicInstance (a fleet.yaml lookup alone
+     * silently dropped them — the retry timer fired and nothing happened).
+     */
+    async startConfiguredInstance(name, topicMode) {
+        const config = this.fleetConfig?.instances[name];
+        if (config) {
+            await this.startInstance(name, config, topicMode);
+            return;
+        }
+        const channel = this.classicChannels?.getAll().find(item => item.instanceName === name);
+        if (!channel || !this.classicChannels)
+            throw new Error(`Instance '${name}' is no longer configured`);
+        await this.startClassicInstance(name, this.classicChannels.getBackendByInstance(name, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(channel.channelId, channel.adapterId), this.classicChannels.getModel(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
+    }
+    /**
+     * Unattended ClassicBot start (fleet startup batch, full-restart batch, the
+     * classicBot.yaml reconcile): same contract as startInstanceUnattended —
+     * failures are logged and handed to the delayed automatic retry, whose
+     * kind-aware startConfiguredInstance rebuilds the Classic instance. Returns
+     * whether the instance is up.
+     */
+    async startClassicInstanceUnattended(ch, what) {
+        try {
+            await this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
+            return this.daemons.has(ch.instanceName);
+        }
+        catch (err) {
+            this.logger.warn({ err, instanceName: ch.instanceName }, `Failed to start ${what}`);
+            this.scheduleStartupRetry(ch.instanceName, 0);
+            return false;
+        }
+    }
+    backendNameOf(name) {
+        const fleetDefault = this.fleetConfig?.defaults?.backend;
+        const configured = this.fleetConfig?.instances[name]?.backend;
+        if (configured)
+            return configured;
+        // ClassicBot channels pick their own backend; the fleet default is only the fallback.
+        if (this.classicChannels?.getAll().some(channel => channel.instanceName === name)) {
+            return this.classicChannels.getBackendByInstance(name, fleetDefault);
+        }
+        return fleetDefault ?? "claude-code";
+    }
+    /**
+     * One fleet-level notice per burst, not one per instance: a post-update herd
+     * fails many instances within the same second. Two notices per incident at
+     * most — "N failed, retrying in X" and, if it comes to that, "gave up on N".
+     */
+    queueStartupRetryNotice(kind, name, delayMs) {
+        const pending = this.startupRetryNotices.get(kind);
+        if (pending) {
+            if (!pending.names.includes(name))
+                pending.names.push(name);
+            return;
+        }
+        const timer = setTimeout(() => {
+            const entry = this.startupRetryNotices.get(kind);
+            this.startupRetryNotices.delete(kind);
+            if (!entry)
+                return;
+            const list = entry.names.join(", ");
+            if (kind === "scheduled") {
+                let text = t("fleet.startup_retry_scheduled", entry.names.length, list, this.formatStormDelay(entry.delayMs));
+                const downBackends = [...new Set(entry.names.map(n => this.backendNameOf(n)))].filter(b => this.backendOutage.isActive(b));
+                if (downBackends.length)
+                    text += `\n${t("fleet.startup_retry_outage", downBackends.join(", "))}`;
+                this.notifyFleetError(text);
+            }
+            else {
+                this.notifyFleetError(t("fleet.startup_retry_gave_up", entry.names.length, list));
+            }
+        }, FleetManager.STARTUP_RETRY_NOTICE_AGGREGATE_MS);
+        timer.unref?.();
+        this.startupRetryNotices.set(kind, { names: [name], delayMs, timer });
     }
     runnableStartupCount(fleet, includeClassic) {
         const names = this.configuredStartupInstanceNames(fleet, includeClassic);
@@ -1474,6 +1719,8 @@ export class FleetManager {
         };
     }
     async stopInstance(name) {
+        this.explicitStopGeneration.set(name, (this.explicitStopGeneration.get(name) ?? 0) + 1);
+        this.cancelStartupRetry(name);
         this.failoverActive.delete(name);
         this.cancelIdleButtonRetirement(name);
         this.instanceStateCache.delete(name);
@@ -1485,12 +1732,20 @@ export class FleetManager {
         // interactive-prompt / clean-exit events whose handlers post fresh prompts
         // while the stop is still awaiting (the TOCTOU sol's review called out).
         this.clearNoncePromptsForInstance(name);
-        try {
-            return await this.lifecycle.stop(name);
-        }
-        finally {
-            this.clearNoncePromptsForInstance(name);
-        }
+        // Published so the outage hand-off can wait for an explicit stop that began
+        // BEFORE it (the generation fence alone only catches stops that begin after
+        // the hand-off snapshotted it).
+        const run = (async () => {
+            try {
+                await this.lifecycle.stop(name);
+            }
+            finally {
+                this.clearNoncePromptsForInstance(name);
+            }
+        })().finally(() => { if (this.stopsInFlight.get(name) === run)
+            this.stopsInFlight.delete(name); });
+        this.stopsInFlight.set(name, run);
+        return run;
     }
     /** Restart a single instance, reloading fleet.yaml first to pick up config changes. */
     async restartSingleInstance(name, opts) {
@@ -1780,7 +2035,10 @@ export class FleetManager {
                         await this.stopInstance(ch.instanceName).catch(() => { });
                         // Small delay to let tmux window clean up
                         await new Promise(r => setTimeout(r, 2000));
-                        await this.startClassicInstance(ch.instanceName, newBackend, this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), newModel, newAutoPause).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to restart classic instance"));
+                        // The manager already holds the new backend/model/auto-pause; the
+                        // unattended helper reads them from it and schedules the delayed
+                        // retry on failure like every other unattended start.
+                        await this.startClassicInstanceUnattended(ch, "classic instance after backend/model change");
                     }
                 }
             }
@@ -1967,6 +2225,9 @@ export class FleetManager {
                 }
                 catch (err) {
                     this.logger.error({ err, name }, "Failed to start general instance");
+                    // General is the most important instance to bring back: it also gets
+                    // the delayed automatic retry (the topic notice below still goes out).
+                    this.scheduleStartupRetry(name, 0);
                     const errorMsg = err instanceof Error ? err.message : String(err);
                     const topicId = cfg.topic_id ? String(cfg.topic_id) : undefined;
                     if (this.adapter && topicId) {
@@ -2086,14 +2347,8 @@ export class FleetManager {
                 while (idx < channels.length) {
                     const batch = channels.slice(idx, idx + concurrency);
                     await Promise.allSettled(batch.map(async (ch) => {
-                        try {
-                            await this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
-                            if (this.daemons.has(ch.instanceName))
-                                startupProgress.markReady();
-                        }
-                        catch (err) {
-                            this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance");
-                        }
+                        if (await this.startClassicInstanceUnattended(ch, "classic instance"))
+                            startupProgress.markReady();
                     }));
                     idx += concurrency;
                 }
@@ -2419,6 +2674,8 @@ export class FleetManager {
                 return;
             if (await this.handleLoginBackendSelect(data, adapterId, this.adapter ?? undefined))
                 return;
+            if (await this.handleInstallBackendSelect(data, adapterId, this.adapter ?? undefined))
+                return;
             if (await this.handleLoginMenuSelect(data, adapterId, this.adapter ?? undefined))
                 return;
             if (await this.handleLoginConfirm(data, adapterId, this.adapter ?? undefined))
@@ -2813,6 +3070,8 @@ export class FleetManager {
                 return;
             if (await this.handleLoginBackendSelect(data, adapterId, adapter))
                 return;
+            if (await this.handleInstallBackendSelect(data, adapterId, adapter))
+                return;
             if (await this.handleLoginMenuSelect(data, adapterId, adapter))
                 return;
             if (await this.handleLoginConfirm(data, adapterId, adapter))
@@ -5170,8 +5429,13 @@ export class FleetManager {
      */
     async postNonceButtonPrompt(opts) {
         // 16 bytes = the 128-bit capability the design claims. Telegram's 64-byte
-        // callback_data cap still holds: longest id is "interactive-assist:" (19)
-        // + 32 hex + ":confirm" (8) = 59 bytes.
+        // callback_data cap still holds, with two prefixes tied at the longest:
+        // "interactive-assist:" (19) + 32 hex + ":confirm" (8) = 59, and
+        // "install-select:" (15) + 32 hex + ":" + the longest backend name
+        // ("claude-code"/"antigravity", 11) = 59. Only 5 bytes of headroom: a
+        // backend name of 17+ characters, or a longer prefix, would be silently
+        // rejected by Telegram — see the callback_data assertion in
+        // install-backend-menu.test.ts.
         const nonce = randomBytes(16).toString("hex");
         const entry = {
             prefix: opts.prefix,
@@ -6410,6 +6674,56 @@ export class FleetManager {
             expiredText: t("buttons.stale"),
         });
     }
+    /**
+     * Backend chooser for a bare `/install-cli`, mirroring promptLoginBackends so
+     * both commands feel the same. Built on postNonceButtonPrompt rather than the
+     * `/model` selection coordinator: that is the mechanism `/login` already uses,
+     * and the one whose canonical-address binding (#682) makes the buttons answer
+     * in a Telegram General topic.
+     *
+     * Unlike the login chooser this does NOT filter to backends the fleet already
+     * runs. Installing is how you get a backend you do not have yet, so filtering
+     * by configured backends would hide the only entry the admin came for.
+     *
+     * gemini-cli is omitted: it is deprecated (see backend/factory.ts). Typing
+     * `/install-cli gemini-cli` still works — this only stops recommending it.
+     */
+    async promptInstallBackends(chat) {
+        const choices = Object.keys(BACKEND_INSTALLATION_INFO)
+            .filter(backend => backend !== "gemini-cli")
+            .map(backend => ({ action: backend, label: backend }));
+        await this.postNonceButtonPrompt({
+            prefix: INSTALL_CALLBACK_PREFIX,
+            alertType: "install",
+            instanceName: "install",
+            adapter: chat.adapter,
+            adapterId: chat.adapterId,
+            chatId: chat.chatId,
+            threadId: chat.threadId,
+            message: t("install.choose_backend"),
+            choices,
+            expiredText: t("buttons.stale"),
+        });
+    }
+    /** Backend chooser button → start that backend's install session. */
+    async handleInstallBackendSelect(data, callbackAdapterId, receivingAdapter) {
+        const claimed = this.consumeNonceCallback(INSTALL_CALLBACK_PREFIX, /^install-select:([0-9a-f]+):([a-z][a-z-]*)$/, data, callbackAdapterId, receivingAdapter);
+        if (claimed === null)
+            return false;
+        if (claimed === "consumed")
+            return true;
+        const { entry, action: backend } = claimed;
+        await this.retireNonceButtons(entry, entry.messageId ?? data.messageId, t("install.starting_backend", backend));
+        const text = await this.startInstallSession(backend, {
+            adapter: entry.adapter,
+            adapterId: entry.adapterId,
+            chatId: entry.chatId,
+            threadId: entry.threadId,
+        });
+        if (text)
+            await entry.adapter.sendText(entry.chatId, text, { threadId: entry.threadId }).catch(() => { });
+        return true;
+    }
     /**
      * Start a login session for one backend. Caller enforces admin.
      * Returns a status line to post, or null when a confirmation prompt was
@@ -6801,7 +7115,11 @@ export class FleetManager {
         }
         const backend = String(data.options?.backend ?? "").trim();
         if (!backend) {
-            await data.respond(t("install.usage"));
+            // Bare call: offer the backends instead of printing a usage line the user
+            // then has to retype. A Discord slash that DID pick the native `backend`
+            // choice never lands here, so the two paths cannot both fire.
+            await this.promptInstallBackends({ adapter, adapterId, chatId: data.channelId });
+            await data.respond(t("install.chooser_posted"));
             return;
         }
         await data.respond(await this.startInstallSession(backend, { adapter, adapterId, chatId: data.channelId }));
@@ -8608,6 +8926,12 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
         // make `agend stop` wait forever for its own queue.
         this.stormWindow.shutdown();
         this.spawnGate.shutdown();
+        for (const pending of this.startupRetries.values())
+            clearTimeout(pending.timer);
+        this.startupRetries.clear();
+        for (const pending of this.startupRetryNotices.values())
+            clearTimeout(pending.timer);
+        this.startupRetryNotices.clear();
         if (this.stormOpenNotifyTimer) {
             clearTimeout(this.stormOpenNotifyTimer);
             this.stormOpenNotifyTimer = null;
@@ -8933,7 +9257,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
             if (!this.daemons.has(name)) {
                 // New instance — startInstance already calls connectIpcToInstance
                 this.logger.info({ name }, "New instance in config — starting");
-                await this.startInstance(name, config, topicMode).catch(err => this.logger.error({ err, name }, "Failed to start new instance"));
+                await this.startInstanceUnattended(name, config, topicMode, "new instance");
             }
             else if (oldConfig?.instances[name]) {
                 const daemon = this.daemons.get(name);
@@ -8944,7 +9268,7 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
                 if (!isDeepStrictEqual(oldParts.cold, newParts.cold)) {
                     this.logger.info({ name }, "Instance config changed — restarting");
                     await this.stopInstance(name).catch(() => { });
-                    await this.startInstance(name, config, topicMode).catch(err => this.logger.error({ err, name }, "Failed to restart modified instance"));
+                    await this.startInstanceUnattended(name, config, topicMode, "modified instance");
                 }
                 else if (!isDeepStrictEqual(oldParts.hot, newParts.hot)) {
                     const update = hotConfigUpdate(config);
@@ -9043,14 +9367,8 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
         const restartGenerals = restartEntries.filter(([_, cfg]) => cfg.general_topic);
         const restartOthers = restartEntries.filter(([_, cfg]) => !cfg.general_topic);
         for (const [name, cfg] of restartGenerals) {
-            try {
-                await this.startInstance(name, cfg, topicMode);
-                if (this.daemons.has(name))
-                    restartProgress.markReady();
-            }
-            catch (err) {
-                this.logger.error({ err, name }, "Failed to start general instance");
-            }
+            if (await this.startInstanceUnattended(name, cfg, topicMode, "general instance"))
+                restartProgress.markReady();
         }
         // General is ready again; now its topic can own the live progress message.
         await restartProgress.start(progressTarget);
@@ -9071,14 +9389,8 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
                 while (idx < channels.length) {
                     const batch = channels.slice(idx, idx + concurrency);
                     await Promise.allSettled(batch.map(async (ch) => {
-                        try {
-                            await this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
-                            if (this.daemons.has(ch.instanceName))
-                                restartProgress.markReady();
-                        }
-                        catch (err) {
-                            this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance");
-                        }
+                        if (await this.startClassicInstanceUnattended(ch, "classic instance"))
+                            restartProgress.markReady();
                     }));
                     idx += concurrency;
                 }