@songsid/agend 2.1.4-beta.9 → 2.1.4

package/dist/instance-lifecycle.js

@@ -4,7 +4,7 @@ import { join, basename, dirname, resolve, sep as pathSep } from "node:path";
 import { access, unlink } from "node:fs/promises";
 import { getAgendHome, ensureWorkspaceGit } from "./paths.js";
 import { DEFAULT_INSTANCE_CONFIG } from "./config.js";
-import { sanitizeInstanceName } from "./topic-commands.js";
+import { readStatuslineModel, sanitizeInstanceName } from "./topic-commands.js";
 import { isModelCompatible } from "./backend/types.js";
 import { safeHandler } from "./safe-async.js";
 import { t } from "./locale.js";
@@ -12,13 +12,17 @@ import { clearPausedMarker, hasPausedMarker, readPausedAt, writePausedMarker } f
 import { reportProviderRateLimit } from "./usage/provider-alerts.js";
 import { isFleetStartCommandLine } from "./fleet-lock.js";
 import { GENERAL_PAUSE_ERROR, isGeneralInstance } from "./general-instance.js";
+import { BACKEND_AUTH_CHECKS, checkAuthStatus, LOGIN_FLOWS } from "./login-flows.js";
+import { fetchClaudeUsage, fetchCodexUsage } from "./usage/providers.js";
 export { isFleetStartCommandLine } from "./fleet-lock.js";
 /** Shared CLI metadata used by startup validation and ClassicBot onboarding. */
 export const BACKEND_INSTALLATION_INFO = {
     "claude-code": { binary: "claude", install: "curl -fsSL https://claude.ai/install.sh | bash" },
     "gemini-cli": { binary: "gemini", install: "npm i -g @google/gemini-cli" },
-    "kiro-cli": { binary: "kiro-cli", install: "brew install --cask kiro-cli" },
-    codex: { binary: "codex", install: "npm i -g @openai/codex" },
+    // The curl installer serves Linux and macOS (verified: 200, text/x-shellscript);
+    // the previous `brew install --cask` form only worked on macOS.
+    "kiro-cli": { binary: "kiro-cli", install: "curl -fsSL https://cli.kiro.dev/install | bash" },
+    codex: { binary: "codex", install: "curl -fsSL https://chatgpt.com/codex/install.sh | sh" },
     opencode: { binary: "opencode", install: "curl -fsSL https://opencode.ai/install | bash" },
     antigravity: { binary: "agy", install: "curl -fsSL https://antigravity.google/cli/install.sh | bash" },
     grok: { binary: "grok", install: "curl -fsSL https://x.ai/cli/install.sh | bash" },
@@ -77,12 +81,116 @@ export function getUnsafeInstanceDaemonPidReason(pid, dataDir) {
 }
 /** Suppress duplicate auth alerts for the same backend within this window. */
 const AUTH_ALERT_COOLDOWN_MS = 5 * 60_000;
+/**
+ * Reuse one auth verification per backend for this long. Shared credentials
+ * expiring makes EVERY instance of the backend emit auth_error at once — one
+ * token-free check answers for all of them.
+ */
+const AUTH_VERIFY_CACHE_MS = 60_000;
+const CODEX_QUOTA_VERIFY_TIMEOUT_MS = 5_000;
+const CLAUDE_QUOTA_VERIFY_TIMEOUT_MS = 5_000;
+/**
+ * Convert the live Codex usage row into a conservative quota verdict. A
+ * successful response with at least one window below 100% proves that stale
+ * terminal text is no longer current only when no other window is exhausted.
+ * Missing credentials, API errors, and metric-less responses prove nothing.
+ */
+export function codexQuotaVerdictFromUsage(usage) {
+    if (usage.status !== "ok")
+        return "unknown";
+    const windows = usage.metrics.filter(metric => metric.type === "percent"
+        && typeof metric.used === "number"
+        && Number.isFinite(metric.used)
+        && metric.windowMs != null);
+    if (windows.length === 0)
+        return "unknown";
+    return windows.some(metric => (metric.used ?? 0) >= 100) ? "exhausted" : "available";
+}
+/** Run only the Codex usage provider, bounded independently of its network timeout. */
+export async function verifyCodexQuotaStatus(fetchUsage = fetchCodexUsage, timeoutMs = CODEX_QUOTA_VERIFY_TIMEOUT_MS) {
+    let timer;
+    try {
+        const usage = await Promise.race([
+            fetchUsage(),
+            new Promise((_, reject) => {
+                timer = setTimeout(() => reject(new Error("Codex quota verification timed out")), timeoutMs);
+                timer.unref?.();
+            }),
+        ]);
+        return codexQuotaVerdictFromUsage(usage);
+    }
+    catch {
+        return "unknown";
+    }
+    finally {
+        if (timer)
+            clearTimeout(timer);
+    }
+}
+/**
+ * Convert a LIVE Claude usage API row into a conservative verdict. The
+ * provider can degrade to statusline data when the network/API is unavailable;
+ * that fallback is intentionally unknown because stale local limits cannot
+ * disprove a current API credit-balance error.
+ */
+export function claudeQuotaVerdictFromUsage(usage) {
+    if (usage.status !== "ok" || /statusline/i.test(usage.hint ?? ""))
+        return "unknown";
+    const bounded = usage.metrics.filter(metric => {
+        if (typeof metric.used !== "number" || !Number.isFinite(metric.used))
+            return false;
+        if (metric.type === "percent")
+            return true;
+        return metric.type === "dollars"
+            && typeof metric.limit === "number"
+            && Number.isFinite(metric.limit)
+            && metric.limit > 0;
+    });
+    if (bounded.length === 0)
+        return "unknown";
+    const exhausted = bounded.some(metric => metric.type === "percent"
+        ? (metric.used ?? 0) >= 100
+        : (metric.used ?? 0) >= (metric.limit ?? Number.POSITIVE_INFINITY));
+    return exhausted ? "exhausted" : "available";
+}
+/** Run only the Claude usage provider, bounded independently of its network timeout. */
+export async function verifyClaudeQuotaStatus(fetchUsage = fetchClaudeUsage, timeoutMs = CLAUDE_QUOTA_VERIFY_TIMEOUT_MS) {
+    let timer;
+    try {
+        const usage = await Promise.race([
+            fetchUsage(),
+            new Promise((_, reject) => {
+                timer = setTimeout(() => reject(new Error("Claude quota verification timed out")), timeoutMs);
+                timer.unref?.();
+            }),
+        ]);
+        return claudeQuotaVerdictFromUsage(usage);
+    }
+    catch {
+        return "unknown";
+    }
+    finally {
+        if (timer)
+            clearTimeout(timer);
+    }
+}
 export class InstanceLifecycle {
     ctx;
     /** Active daemon processes: instanceName → Daemon */
     daemons = new Map();
     /** backend → last auth-error alert time, so one expiry sends one alert. */
     lastAuthAlertAt = new Map();
+    /**
+     * backend → cached token-free auth verification (see AUTH_VERIFY_CACHE_MS).
+     * The PROMISE is cached, not the result: a shared credential expiring makes
+     * every instance of the backend fire in the same tick, and they must join
+     * one in-flight check instead of each spawning their own.
+     */
+    authVerifyCache = new Map();
+    /** Same-tick Codex quota alerts join one live usage probe. */
+    codexQuotaVerifyInFlight = null;
+    /** Same-tick Claude quota alerts join one live usage probe. */
+    claudeQuotaVerifyInFlight = null;
     /**
      * Minimum gap between MCP-revival auto-restarts of one instance. Kept here —
      * not in the daemon — because each restart replaces the daemon object, which
@@ -108,12 +216,37 @@ export class InstanceLifecycle {
      * suppresses the chat message, not the record. A crash outside a planned
      * restart notifies exactly as before.
      */
+    /** @returns whether the notice was actually dispatched (false when suppressed). */
     notifyIncident(name, kind, text) {
         if (this.ctx.isPlannedRestart()) {
             this.ctx.logger.info({ name, kind }, "Incident notification suppressed — planned restart in progress");
-            return;
+            return false;
         }
-        this.ctx.notifyInstanceTopic(name, text);
+        if (this.ctx.stormSuppressed?.(kind)) {
+            this.ctx.logger.info({ name, kind }, "Incident notification suppressed — included in tmux storm summary");
+            return false;
+        }
+        // A production `false` (no adapter / no route) means nobody saw it — it
+        // must not count as dispatched, or a later retraction would be an orphan.
+        const dispatched = this.ctx.notifyInstanceTopic(name, text);
+        return dispatched !== false;
+    }
+    /**
+     * Per-instance MCP incident fence. The mcp_died handler awaits an auth probe
+     * (up to 5s) before it notifies; a recovery that lands inside that window
+     * bumps the generation so the late death handler sends nothing — otherwise
+     * the user would see "retracted" followed by the stale red alarm. The
+     * retraction itself is only sent when a death notice really reached the
+     * user (not when it was storm- or planned-restart-suppressed).
+     */
+    mcpIncidents = new Map();
+    mcpIncident(name) {
+        let s = this.mcpIncidents.get(name);
+        if (!s) {
+            s = { gen: 0, deathNoticeSent: false };
+            this.mcpIncidents.set(name, s);
+        }
+        return s;
     }
     /** Backend a running instance uses (config → fleet default). */
     backendOf(name) {
@@ -121,6 +254,67 @@ export class InstanceLifecycle {
             ?? this.ctx.fleetConfig?.defaults?.backend
             ?? "claude-code";
     }
+    /**
+     * Confirm a pane-detected auth error with the backend's token-free status
+     * probe before pausing anything: pattern matching over terminal text also
+     * fires on an agent DISCUSSING a 401. "valid" means false positive — ignore.
+     * "invalid" and "unknown" (timeout, missing binary) both pause: for a
+     * suspected expiry, pausing too much is recoverable, delivering into a dead
+     * CLI is not. Cached per backend so simultaneous alerts run one check.
+     */
+    verifyAuthError(name) {
+        return this.verifyBackendAuth(this.backendOf(name));
+    }
+    verifyCodexQuota() {
+        if (this.codexQuotaVerifyInFlight)
+            return this.codexQuotaVerifyInFlight;
+        const verify = this.ctx.verifyCodexQuota ?? (() => verifyCodexQuotaStatus());
+        const promise = verify().catch(() => "unknown");
+        this.codexQuotaVerifyInFlight = promise;
+        void promise.finally(() => {
+            if (this.codexQuotaVerifyInFlight === promise)
+                this.codexQuotaVerifyInFlight = null;
+        });
+        return promise;
+    }
+    verifyClaudeQuota() {
+        if (this.claudeQuotaVerifyInFlight)
+            return this.claudeQuotaVerifyInFlight;
+        const verify = this.ctx.verifyClaudeQuota ?? (() => verifyClaudeQuotaStatus());
+        const promise = verify().catch(() => "unknown");
+        this.claudeQuotaVerifyInFlight = promise;
+        void promise.finally(() => {
+            if (this.claudeQuotaVerifyInFlight === promise)
+                this.claudeQuotaVerifyInFlight = null;
+        });
+        return promise;
+    }
+    /** Same verification keyed by backend (used by startup pre-flight priming). */
+    verifyBackendAuth(backend) {
+        const cached = this.authVerifyCache.get(backend);
+        if (cached && Date.now() - cached.at < AUTH_VERIFY_CACHE_MS)
+            return cached.promise;
+        const check = LOGIN_FLOWS[backend]?.authCheck ?? BACKEND_AUTH_CHECKS[backend];
+        const promise = check ? checkAuthStatus(check) : Promise.resolve("unknown");
+        this.authVerifyCache.set(backend, { at: Date.now(), promise });
+        return promise;
+    }
+    /**
+     * Startup pre-flight: warm the per-backend auth verification cache so the
+     * detectors that fire seconds later (login-screen scan, MCP-died gate) get an
+     * instant answer. Advisory only — a pre-flight result alone never pauses
+     * anything, because e.g. codex on a custom provider runs fine while
+     * `codex login status` reports logged out (live-verified on this fleet).
+     */
+    primeAuthVerification(backends) {
+        for (const backend of new Set(backends)) {
+            void this.verifyBackendAuth(backend).then(result => {
+                if (result === "invalid") {
+                    this.ctx.logger.warn({ backend }, "Pre-flight auth check failed — marking backend as auth-suspect");
+                }
+            }).catch(() => { });
+        }
+    }
     /**
      * One alert per backend per cooldown, naming every affected instance — a CLI's
      * credentials are shared, so N instances failing is ONE problem with ONE fix
@@ -141,7 +335,30 @@ export class InstanceLifecycle {
         const scope = others.length
             ? `${affected.length} instances on \`${backend}\`: ${affected.join(", ")}`
             : `\`${name}\` (${backend})`;
-        this.notifyIncident(notificationTarget, "auth_error", `🔑 ${message}\n\nAffects ${scope}. Credentials are shared per backend — one re-login restores all of them; affected instances pause until then.`);
+        const remedy = backend === "opencode"
+            ? "Run `opencode auth login` in a terminal, then wake the affected instance(s)."
+            : `Use \`/login ${backend}\` to re-login remotely.`;
+        this.notifyIncident(notificationTarget, "auth_error", `🔑 ${message}\n\nAffects ${scope}. Credentials are shared per backend — one re-login restores all of them; affected instances pause until then. ${remedy}`);
+    }
+    /**
+     * A fleet-wide backend outage (see backend-outage.ts): record the sighting so
+     * startup stops spending `--resume` attempts on it, and notify ONCE per
+     * outage at fleet level — kiro prints the line every 10s on every instance,
+     * so per-instance incidents would be N × (one message per cooldown).
+     */
+    noteBackendOutage(name, message, notificationTarget) {
+        const backend = this.backendOf(name);
+        const result = this.ctx.backendOutage?.record(backend, name, message);
+        this.ctx.eventLog?.insert(name, "backend_unreachable", { backend, message });
+        if (result && !result.isNew) {
+            this.ctx.logger.info({ name, backend }, "backend outage sighting (already alerted)");
+            return;
+        }
+        const text = t("inst.backend_unreachable", backend, message);
+        if (this.ctx.notifyFleetError)
+            this.ctx.notifyFleetError(text);
+        else if (notificationTarget)
+            this.notifyIncident(notificationTarget, "pty_error", text);
     }
     /**
      * System errors from a ClassicBot belong in the operator's General topic,
@@ -179,6 +396,10 @@ export class InstanceLifecycle {
                     this.ctx.logger.info({ name }, "Hang notification suppressed — planned restart in progress");
                     return;
                 }
+                if (this.ctx.stormSuppressed?.("hang")) {
+                    this.ctx.logger.info({ name }, "Hang notification suppressed — included in tmux storm summary");
+                    return;
+                }
                 // Check if instance has claimed tasks — nudge it to continue
                 const claimedTasks = this.ctx.listClaimedTasks(name);
                 if (claimedTasks.length > 0) {
@@ -207,6 +428,24 @@ export class InstanceLifecycle {
                 this.notifyIncident(generalName, "crash_respawn", t("inst.crashed_respawned_log", name));
             }
         }, this.ctx.logger, `daemon.crash_respawn[${name}]`));
+        daemon.on("startup_backend_unreachable", safeHandler(async (data) => {
+            // The daemon's crash-respawn hit the backend outage and paused itself.
+            // Stop it cleanly (session kept) and let the fleet's delayed retry bring
+            // it back once the backend answers — instead of `crashed` forever.
+            this.ctx.eventLog?.insert(name, "startup_backend_unreachable", { backend: data.backend });
+            this.ctx.logger.warn({ name, backend: data.backend }, "Respawn blocked by backend outage — stopping the daemon for a delayed startup retry");
+            if (this.ctx.handOffToStartupRetry) {
+                await this.ctx.handOffToStartupRetry(name, daemon);
+                return;
+            }
+            // No fleet coordinator: stop only if this daemon is still the registered one.
+            if (await this.stopIfCurrent(name, daemon))
+                this.ctx.scheduleStartupRetry?.(name, 0);
+        }, this.ctx.logger, `daemon.startup_backend_unreachable[${name}]`));
+        daemon.on("tmux_server_crash", safeHandler(() => {
+            this.ctx.eventLog?.insert(name, "tmux_server_crash", {});
+            this.ctx.logger.error({ name }, "tmux server crash joined fleet storm window");
+        }, this.ctx.logger, `daemon.tmux_server_crash[${name}]`));
         daemon.on("snapshot_failed", safeHandler(() => {
             this.ctx.eventLog?.insert(name, "snapshot_failed", {});
             this.notifyIncident(name, "snapshot_failed", t("inst.restarted_no_context", name));
@@ -242,26 +481,82 @@ export class InstanceLifecycle {
             this.notifyIncident(name, "crash_loop", t("inst.respawn_paused", name));
             this.ctx.setTopicIcon(name, "red");
         }, this.ctx.logger, `daemon.crash_loop[${name}]`));
-        daemon.on("mcp_died", safeHandler((data) => {
+        daemon.on("mcp_died", safeHandler(async (data) => {
+            const stormAtDetection = this.ctx.stormWindow?.isActive() === true;
+            const incident = this.mcpIncident(name);
+            const generation = ++incident.gen;
+            incident.deathNoticeSent = false;
             this.ctx.eventLog?.insert(name, "mcp_died", { pid: data.pid });
             this.ctx.logger.error({ name, pid: data.pid }, "MCP server died — instance cannot use agend tools");
             this.ctx.webhookEmit("mcp_died", name, { pid: data.pid });
+            // Auth outranks MCP: when the CLI lost authentication, "MCP server died"
+            // is a symptom and the fix is /login, not a restart. The daemon flags a
+            // suspicion it already confirmed (login screen / 401 pattern); otherwise
+            // ask the cached token-free probe. Only a CONFIRMED invalid swaps the
+            // message — valid or uncertain keeps the accurate MCP report below.
+            const verdict = data.authSuspected ? "invalid" : await this.verifyAuthError(name);
+            if (incident.gen !== generation) {
+                // The server recovered (or a newer death superseded this one) while the
+                // probe ran. Anything we would say now is stale — say nothing.
+                this.ctx.logger.info({ name, pid: data.pid }, "MCP death report superseded during the auth probe — nothing sent");
+                return;
+            }
+            if (verdict === "invalid") {
+                this.notifyAuthErrorOnce(name, "Sign-in expired — the CLI cannot run its MCP server (agend tools are down) until it is re-authenticated.", this.ptyErrorNotificationTarget(name) ?? name);
+                return;
+            }
+            if (stormAtDetection) {
+                this.ctx.stormSuppressed?.("mcp_died");
+                this.ctx.logger.info({ name, pid: data.pid }, "MCP death notification suppressed — included in tmux storm summary");
+                return;
+            }
             // The CLI owns the MCP server's stdio pipes, so only restarting the CLI can
             // restore its tools. With mcp_auto_restart (default) the daemon requests an
             // idle-gated restart itself — an immediate one would interrupt whatever the
             // agent is doing. With it off, tell the operator what to run, as before.
-            this.notifyIncident(name, "mcp_died", `⚠️ \`${name}\` 的 MCP server 已終止 — 這個 instance 目前無法使用 agend 工具（無法 reply / 跨 instance 通訊）。\n`
+            incident.deathNoticeSent = this.notifyIncident(name, "mcp_died", `⚠️ \`${name}\` 的 MCP server 已終止 — 這個 instance 目前無法使用 agend 工具（無法 reply / 跨 instance 通訊）。\n`
                 + (data.autoRestart
                     ? "CLI 本身還在執行；等它閒置後會自動重啟以恢復工具（進行中的工作不會被打斷，session 會保留）。"
                     : `CLI 本身還在執行。工具只能由 CLI 自己重新啟動 MCP server，請用 \`restart_instance("${name}")\` 或 \`/restart\` 恢復。`));
         }, this.ctx.logger, `daemon.mcp_died[${name}]`));
+        daemon.on("mcp_recovered", safeHandler((data) => {
+            // Retract an earlier "MCP died" report: a live server for this instance
+            // is serving (seen at the IPC layer — a connection, mcp_ready, or a tool
+            // call — never inferred from the LLM). No restart will follow.
+            const incident = this.mcpIncident(name);
+            incident.gen++; // fences a death handler still waiting on its auth probe
+            this.ctx.eventLog?.insert(name, "mcp_recovered", { source: data.source, pid: data.pid });
+            this.ctx.logger.info({ name, source: data.source, pid: data.pid }, "MCP server recovered — earlier death report retracted");
+            if (!incident.deathNoticeSent) {
+                this.ctx.logger.info({ name }, "No MCP death notice reached the user — nothing to retract");
+                return;
+            }
+            incident.deathNoticeSent = false;
+            this.notifyIncident(name, "mcp_recovered", t("inst.mcp_recovered", name));
+        }, this.ctx.logger, `daemon.mcp_recovered[${name}]`));
         daemon.on("mcp_proxy_reply", safeHandler((data) => {
             // The message itself goes out through the daemon's fleet_outbound path;
             // this is the audit trail that it happened (and why the channel saw a ⚠️).
             this.ctx.eventLog?.insert(name, "mcp_proxy_reply", { correlationId: data.correlationId });
             this.ctx.logger.warn({ name, correlationId: data.correlationId }, "MCP dead at turn end with no reply — daemon relayed the pane text to the channel");
         }, this.ctx.logger, `daemon.mcp_proxy_reply[${name}]`));
+        daemon.on("malformed_tool_call", safeHandler((data) => {
+            this.ctx.eventLog?.insert(name, "malformed_tool_call", {
+                correlationId: data.correlationId,
+                recovered: data.recovered,
+            });
+            const notificationTarget = this.ptyErrorNotificationTarget(name);
+            if (notificationTarget) {
+                this.notifyIncident(notificationTarget, "malformed_tool_call", t(data.recovered ? "inst.malformed_tool_call_recovered" : "inst.malformed_tool_call_unrecoverable", name));
+            }
+        }, this.ctx.logger, `daemon.malformed_tool_call[${name}]`));
         daemon.on("mcp_restart_requested", safeHandler((data) => {
+            if (this.ctx.stormWindow?.isActive()) {
+                this.ctx.eventLog?.insert(name, "mcp_auto_restart_suppressed", { trigger: data.trigger, reason: "tmux_storm" });
+                this.ctx.stormSuppressed?.("mcp_auto_restart");
+                this.ctx.logger.info({ name, trigger: data.trigger }, "MCP revival restart suppressed — tmux storm recovery will replace MCP");
+                return;
+            }
             // The daemon object dies with the restart it asks for, so the loop guard
             // lives here: if the previous auto-restart was under the cooldown, the new
             // MCP server evidently died right back (broken install, OOM pressure) and
@@ -288,6 +583,24 @@ export class InstanceLifecycle {
             // server.
             this.ctx.restartSingleInstance(name).catch(err => this.ctx.logger.error({ err, name }, "MCP auto-restart failed"));
         }, this.ctx.logger, `daemon.mcp_restart_requested[${name}]`));
+        daemon.on("dialog_parked", safeHandler((data) => {
+            // A CLI dialog the daemon will NOT answer on its own. No assist buttons
+            // here on purpose: the General "Confirm" assist sends Enter, and Enter is
+            // exactly the destructive default for Claude's resume prompt. The text
+            // differs by case: a hold-only variant means we do not know the cursor
+            // position, so the human must LOOK and pick the option explicitly — no
+            // fixed key sequence; a known dialog whose auto-dismiss did not take gets
+            // the keys we would have pressed.
+            this.ctx.eventLog?.insert(name, "dialog_parked", { description: data.description, holdOnly: data.holdOnly });
+            this.ctx.logger.warn({ name, description: data.description, holdOnly: data.holdOnly }, "Instance is parked on a CLI dialog — not auto-answering");
+            if (this.ctx.isPlannedRestart())
+                return;
+            const text = data.holdOnly
+                ? t("inst.dialog_parked_hold", name, data.description)
+                : t("inst.dialog_parked_stuck", name, data.description);
+            this.notifyIncident(name, "dialog_parked", text);
+            this.ctx.notifyFleetError?.(t("fleet.dialog_parked", name, data.description));
+        }, this.ctx.logger, `daemon.dialog_parked[${name}]`));
         daemon.on("interactive_prompt", safeHandler(async (data) => {
             this.ctx.eventLog?.insert(name, "interactive_prompt", { kind: data.kind });
             this.ctx.logger.warn({ name, kind: data.kind, prompt: data.prompt }, "Instance is waiting for interactive terminal input");
@@ -297,7 +610,7 @@ export class InstanceLifecycle {
             }
             await this.ctx.notifyInteractivePrompt(name, data.kind);
         }, this.ctx.logger, `daemon.interactive_prompt[${name}]`));
-        daemon.on("pty_error", safeHandler((data) => {
+        daemon.on("pty_error", safeHandler(async (data) => {
             this.ctx.eventLog?.insert(name, "pty_error", { type: data.type, action: data.action });
             this.ctx.logger.warn({ name, errorType: data.type, action: data.action }, `PTY error: ${data.message}`);
             // Antigravity's account-level cap is visible ONLY here: the quota summary
@@ -307,18 +620,69 @@ export class InstanceLifecycle {
             if (data.type === "quota" && this.backendOf(name) === "antigravity") {
                 reportProviderRateLimit("antigravity", data.message);
             }
+            // Codex keeps old errors in pane scrollback. After a restart, the new
+            // daemon has no occurrence baseline and can mistake yesterday's
+            // `You've hit your usage limit` for a current failure, pause, wake, then
+            // repeat forever. A live, non-LLM usage query distinguishes an
+            // available account from that stale text before any notification/pause.
+            // Only pause-class quota errors need this gate; low-quota notifications
+            // remain immediate. Unknown (timeout/auth/API failure) stays fail-closed.
+            if (data.type === "quota" && data.action === "pause" && this.backendOf(name) === "codex") {
+                const verdict = await this.verifyCodexQuota();
+                if (verdict === "available") {
+                    this.ctx.logger.debug({ name, backend: "codex" }, "quota pattern ignored — live usage has capacity (stale pane history)");
+                    return;
+                }
+                if (verdict === "unknown") {
+                    this.ctx.logger.warn({ name, backend: "codex" }, "Codex quota verification failed or timed out — pausing conservatively");
+                }
+            }
+            // Claude's credit-balance message also remains in pane scrollback. Only
+            // a fresh usage API row can prove it stale; provider fallback to a local
+            // statusline, timeout, or missing credentials remains fail-closed.
+            if (data.type === "quota" && data.action === "pause" && this.backendOf(name) === "claude-code") {
+                const verdict = await this.verifyClaudeQuota();
+                if (verdict === "available") {
+                    this.ctx.logger.debug({ name, backend: "claude-code" }, "quota pattern ignored — live usage has capacity (stale pane history)");
+                    return;
+                }
+                if (verdict === "unknown") {
+                    this.ctx.logger.warn({ name, backend: "claude-code" }, "Claude quota verification failed, degraded, or timed out — pausing conservatively");
+                }
+            }
+            // Pattern-matched auth errors get a second opinion from the real CLI
+            // before any pause/alert: an agent quoting "401 Unauthorized" in prose
+            // must not pause the fleet. A working credential ends the incident here.
+            if (data.type === "auth_error") {
+                const verdict = await this.verifyAuthError(name);
+                if (verdict === "valid") {
+                    this.ctx.logger.info({ name, backend: this.backendOf(name) }, "auth-error pattern ignored — token-free auth check passed (likely conversation text)");
+                    return;
+                }
+            }
             const emoji = data.type === "rate_limit" || data.type === "timeout" ? "⏳" : data.type === "auth_error" ? "🔑" : "⚠️";
+            const incidentMessage = data.type === "model_error" && this.backendOf(name) === "claude-code"
+                ? (() => {
+                    const liveModel = readStatuslineModel(this.ctx.dataDir, name);
+                    return liveModel
+                        ? t("inst.claude_model_fallback", liveModel)
+                        : t("inst.claude_model_unavailable");
+                })()
+                : data.message;
             const notificationTarget = this.ptyErrorNotificationTarget(name);
             // Auth failures are a property of the BACKEND's shared credentials, not of
             // one instance: every instance on that CLI fails at once, and one re-login
             // fixes them all. Notify once per backend (listing who's affected) instead
             // of N near-identical alerts, and suppress repeats fleet-wide.
-            if (data.type === "auth_error") {
+            if (data.type === "network" && data.fleetWide) {
+                this.noteBackendOutage(name, data.message, notificationTarget);
+            }
+            else if (data.type === "auth_error") {
                 if (notificationTarget)
                     this.notifyAuthErrorOnce(name, data.message, notificationTarget);
             }
             else if (notificationTarget) {
-                this.notifyIncident(notificationTarget, "pty_error", t("inst.notification", emoji, name, data.message, data.action));
+                this.notifyIncident(notificationTarget, "pty_error", t("inst.notification", emoji, name, incidentMessage, data.action));
             }
             this.ctx.webhookEmit("pty_error", name, { type: data.type, action: data.action, message: data.message });
             // The CLI interrupted itself on this error, so any pending Cancel button is
@@ -386,12 +750,12 @@ export class InstanceLifecycle {
             kind: "fleet-topic",
             backend: backendName,
             model: config.model ?? "default",
-        });
+        }, this.ctx.spawnGate, this.ctx.stormWindow, this.ctx.backendOutage);
         // Catch errors from daemon internals (e.g. IPC server) to prevent crashing the fleet process
         daemon.on("error", (err) => {
             this.ctx.logger.error({ err, name }, "Daemon emitted error — instance isolated");
         });
-        await daemon.start();
+        await InstanceLifecycle.startOrDispose(daemon, name, this.ctx.logger);
         this.daemons.set(name, daemon);
         daemon.on("auto_pause_requested", safeHandler(async () => {
             await this.pause(name);
@@ -485,12 +849,38 @@ export class InstanceLifecycle {
         }
         this.ctx.startStatuslineWatcher(name);
     }
+    /**
+     * Ownership boundary for a daemon that is not yet registered: if start()
+     * rejects, nothing else will ever dispose it, so do it here before
+     * rethrowing. The abort is best-effort — the start error is the one to
+     * surface.
+     */
+    static async startOrDispose(daemon, name, logger) {
+        try {
+            await daemon.start();
+        }
+        catch (err) {
+            await daemon.abortStartup().catch(abortErr => logger.warn({ err: abortErr, name }, "Failed to dispose a daemon whose start() rejected"));
+            throw err;
+        }
+    }
+    /** Stop the registered daemon only if it is still `daemon` (a concurrent restart may have replaced it). */
+    async stopIfCurrent(name, daemon) {
+        if (this.daemons.get(name) !== daemon)
+            return false;
+        await this.stop(name);
+        return true;
+    }
     async stop(name) {
         this.ctx.setTopicIcon(name, "remove");
         const daemon = this.daemons.get(name);
         if (daemon) {
             await daemon.stop();
-            this.daemons.delete(name);
+            // Identity-safe: while we awaited, a concurrent restart may have
+            // registered a FRESH daemon under this name — deleting by name alone
+            // would drop it from supervision while its CLI keeps running.
+            if (this.daemons.get(name) === daemon)
+                this.daemons.delete(name);
         }
         else {
             const instanceDir = this.ctx.getInstanceDir(name);