@songsid/agend 2.1.1-beta.2 → 2.1.1-beta.20

package/dist/fleet-manager.js

@@ -4,7 +4,8 @@ import { createServer } from "node:http";
 import { join, dirname, basename } from "node:path";
 import { fileURLToPath } from "node:url";
 import { getAgendHome, ensureWorkspaceGit } from "./paths.js";
-import { sdNotify } from "./sd-notify.js";
+import { sdNotify, sdNotifyBlocking } from "./sd-notify.js";
+import { readFleetMemory } from "./process-memory.js";
 import { isScalar, parseDocument } from "yaml";
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
@@ -21,12 +22,12 @@ import { IpcClient } from "./channel/ipc-bridge.js";
 import { createAdapter } from "./channel/factory.js";
 import { createBackend } from "./backend/factory.js";
 import { isModelCompatible } from "./backend/types.js";
-import { createLogger } from "./logger.js";
+import { createLogger, rotateLogIfNeeded } from "./logger.js";
 import { processAttachments } from "./channel/attachment-handler.js";
 import { routeToolCall } from "./channel/tool-router.js";
 import { Scheduler } from "./scheduler/index.js";
 import { DEFAULT_SCHEDULER_CONFIG } from "./scheduler/index.js";
-import { TopicCommands, saveCommandForBackend, parseSaveFilename, parsePauseWakeCommand, SAVE_FILENAME_RE, SAVE_UNSUPPORTED_MSG } from "./topic-commands.js";
+import { TopicCommands, saveCommandForBackend, parseSaveFilename, parsePauseWakeCommand, SAVE_FILENAME_RE, SAVE_UNSUPPORTED_MSG, resolveInstanceContext, forgetInstanceContext } from "./topic-commands.js";
 import { DailySummary } from "./daily-summary.js";
 import { WebhookEmitter } from "./webhook-emitter.js";
 import { TmuxControlClient } from "./tmux-control.js";
@@ -38,6 +39,7 @@ import { StatuslineWatcher } from "./statusline-watcher.js";
 import { outboundHandlers } from "./outbound-handlers.js";
 import { handleWebRequest, broadcastSseEvent } from "./web-api.js";
 import { handleViewRequest, isViewPath } from "./view-api.js";
+import { handleUsageRequest, isUsagePath } from "./usage/usage-api.js";
 import { handleSettingsRequest } from "./settings-api.js";
 import { setLocale, detectLocale, t } from "./locale.js";
 import { handleAgentRequest } from "./agent-endpoint.js";
@@ -83,6 +85,27 @@ const CANCEL_BTN_MAX_RETRIES = 3;
  * buttons no clear trigger reached (e.g. a scheduled/HTTP turn that never called
  * reply). 5min (not the old 2s idle-watch) so Thinking isn't misread as idle. */
 const CANCEL_BTN_IDLE_CHECK_INTERVAL_MS = 5 * 60_000;
+/**
+ * How often the cancel button's text is refreshed with elapsed working time.
+ *
+ * One edit per working instance per interval — at 60s that is trivial for both
+ * platforms' rate limits, and it reads as a live counter rather than a stale
+ * snapshot. Nothing new is posted, so the channel is never spammed: there is
+ * exactly one progress message per turn, and it is the cancel button itself.
+ */
+const PROGRESS_UPDATE_INTERVAL_MS = 60_000;
+/** Elapsed time is only shown once work has clearly outlasted a quick answer. */
+const PROGRESS_MIN_ELAPSED_MS = 2 * 60_000;
+/** How much of a tool summary the progress line will show before eliding. */
+const PROGRESS_ACTIVITY_MAX_CHARS = 48;
+/**
+ * How long a delivery waits out a disconnected instance IPC before giving up.
+ *
+ * Sized for a daemon restart (socket close → respawn → CLI ready), which is the
+ * event this exists for. Past it the delivery fails loudly as it always did.
+ */
+const IPC_RECONNECT_GRACE_MS = 30_000;
+const IPC_RECONNECT_POLL_MS = 250;
 const CLASSIC_BACKEND_SELECTION_TIMEOUT_MS = 60_000;
 const CLASSIC_BACKEND_CALLBACK_PREFIX = "classic-backend:";
 const MODEL_SELECT_CALLBACK_PREFIX = "model-select:";
@@ -93,7 +116,7 @@ export class FleetManager {
     static sighupHandlerInstalled = false;
     children = new Map();
     lifecycle;
-    /** @deprecated Use lifecycle.daemons — kept for backward compat */
+    /** Live view of lifecycle.daemons — used throughout; not deprecated. */
     get daemons() { return this.lifecycle.daemons; }
     fleetConfig = null;
     rawFleetConfig = {};
@@ -151,6 +174,10 @@ export class FleetManager {
     // reply, on cancel, or when a newer button supersedes it for the same
     // instance. Per-button tracking means a failed delete never strands a button.
     cancelButtons = new Map();
+    /** instanceName → what it is doing right now, when the backend can tell us. */
+    instanceActivity = new Map();
+    /** instanceName → tail of deliveries waiting for its IPC to come back. */
+    ipcWaitTails = new Map();
     // Last user message delivered to each instance — used to react ✅ on completion.
     lastInboundMsg = new Map();
     topicArchiver;
@@ -163,6 +190,10 @@ export class FleetManager {
     failoverActive = new Map(); // instance → current failover model
     // IPC reconnect: tracks instances being intentionally stopped (skip reconnect)
     ipcStoppingInstances = new Set();
+    /** Coalesce concurrent connection attempts for the same daemon socket. */
+    ipcConnectInFlight = new Map();
+    /** At most one reconnect/backoff loop may exist per instance. */
+    ipcReconnectInFlight = new Map();
     // Adapter restart: prevents re-entrant restart attempts
     adapterRestarting = new Set();
     // Adapter isolation: track state per adapter for retry + visibility
@@ -172,6 +203,10 @@ export class FleetManager {
     healthServer = null;
     healthPortRetried = false;
     updateCheckTimer = null;
+    eventLogPruneTimer = null;
+    logRotateTimer = null;
+    /** Days of event/activity history to keep. */
+    static EVENT_LOG_RETENTION_DAYS = 30;
     watchdogTimer = null;
     startedAt = 0;
     // Mirror topic: buffer cross-instance messages, flush every 3s
@@ -210,7 +245,13 @@ export class FleetManager {
         }
         this.reloadPending = false;
         this.reconcileInFlight = this.reconcileInstances()
-            .catch(err => this.logger.error({ err }, "SIGHUP config reload failed"))
+            .catch(err => {
+            // Almost always a YAML parse error. Log-only meant the user edited
+            // fleet.yaml, sent SIGHUP, and got no reaction and no explanation.
+            this.logger.error({ err }, "SIGHUP config reload failed");
+            const message = err instanceof Error ? err.message : String(err);
+            this.notifyFleetError(`⚠️ fleet.yaml reload FAILED — ${message}\nThe previous configuration is still running.`);
+        })
             .finally(() => {
             this.reconcileInFlight = null;
             if (this.reloadPending && this.startupComplete) {
@@ -473,8 +514,12 @@ export class FleetManager {
         // warm_cap: a fresh transition into idle may free this instance for eviction,
         // or (more usefully) reveal that the fleet is now over cap. Only fire on the
         // edge into idle, not on every idle heartbeat.
-        if (state === "idle" && previous?.state !== "idle")
+        if (state === "idle" && previous?.state !== "idle") {
             this.enforceWarmCap();
+            // The turn is genuinely over — retire the cancel/progress button now rather
+            // than waiting for the 5-minute idle backstop to notice.
+            this.retireInstanceButtons(name);
+        }
     }
     cacheInstanceProcessStatus(name, status) {
         if (status === "running") {
@@ -579,12 +624,70 @@ export class FleetManager {
         if (!idle) {
             this.logger.warn({ instanceName, timeoutMs }, "Idle gate timed out; forcing delivery");
         }
-        const ipc = this.instanceIpcClients.get(instanceName);
-        if (!ipc?.connected)
-            throw new Error(`Instance '${instanceName}' IPC is unavailable`);
-        ipc.send(payload);
+        await this.sendWhenConnected(instanceName, payload);
         this.lastDeliveryAt.set(instanceName, Date.now());
     }
+    /**
+     * Hand a payload to an instance's IPC, waiting out a *transient* disconnect.
+     *
+     * A daemon that is restarting — `/restart`, crash recovery, a model switch —
+     * drops its socket for a few seconds. Any message arriving in that window used
+     * to fail instantly: the caller logged a warning, put ❌ on the user's message,
+     * and the message was gone. The user had to notice the ❌ and retype it. That is
+     * the "instance 訊息不容易掉" goal failing on the most predictable event there is.
+     *
+     * The wait is bounded. If the instance is genuinely down, this still throws and
+     * the ❌ still appears — just for a real failure rather than a restart.
+     *
+     * Ordering is preserved by serialising behind any waiter already queued for this
+     * instance, *including* when the socket happens to be up: otherwise a message
+     * arriving after the reconnect could overtake one that has been waiting for it.
+     */
+    async sendWhenConnected(instanceName, payload) {
+        const queued = this.ipcWaitTails.get(instanceName);
+        if (!queued) {
+            const ipc = this.instanceIpcClients.get(instanceName);
+            if (ipc?.connected && ipc.send(payload))
+                return;
+        }
+        const attempt = (queued ?? Promise.resolve())
+            .catch(() => { })
+            .then(() => this.sendAfterIpcReturns(instanceName, payload));
+        // The chain stores a settled-either-way promise so one failed delivery cannot
+        // wedge every later one, and so `queued` above is safe to await unguarded.
+        const tail = attempt.catch(() => { });
+        this.ipcWaitTails.set(instanceName, tail);
+        try {
+            await attempt;
+        }
+        finally {
+            // Only the last waiter clears the chain; while a queue is still draining the
+            // map must keep pointing at it or ordering is lost.
+            if (this.ipcWaitTails.get(instanceName) === tail) {
+                this.ipcWaitTails.delete(instanceName);
+            }
+        }
+    }
+    /** Poll for the instance's IPC to come back, then send. Throws if it does not. */
+    async sendAfterIpcReturns(instanceName, payload) {
+        const deadline = Date.now() + IPC_RECONNECT_GRACE_MS;
+        let warned = false;
+        for (;;) {
+            // Re-read every round: a reconnect replaces the IpcClient object entirely,
+            // so a cached reference would stay dead forever.
+            const ipc = this.instanceIpcClients.get(instanceName);
+            if (ipc?.connected && ipc.send(payload))
+                return;
+            if (Date.now() >= deadline) {
+                throw new Error(`Instance '${instanceName}' IPC is unavailable`);
+            }
+            if (!warned) {
+                warned = true;
+                this.logger.info({ instanceName }, "Instance IPC is down — holding delivery until it reconnects");
+            }
+            await new Promise(resolve => setTimeout(resolve, IPC_RECONNECT_POLL_MS));
+        }
+    }
     /** Single delivery facade: wake paused CLIs and serialize non-user work behind idle. */
     async deliverToInstance(instanceName, payload, options = {}) {
         const meta = payload.meta && typeof payload.meta === "object"
@@ -600,10 +703,7 @@ export class FleetManager {
                 await this.lifecycle.wake(instanceName, 30_000);
                 this.enforceWarmCap(instanceName); // woke one → evict a different LRU idle if over cap
             }
-            const ipc = this.instanceIpcClients.get(instanceName);
-            if (!ipc?.connected)
-                throw new Error(`Instance '${instanceName}' IPC is unavailable`);
-            ipc.send(payload);
+            await this.sendWhenConnected(instanceName, payload);
             // A cross-instance item arriving before the daemon observes this turn as
             // working must not trust the stale idle snapshot from before the send.
             this.lastDeliveryAt.set(instanceName, Date.now());
@@ -651,7 +751,7 @@ export class FleetManager {
         await new Promise(resolve => setTimeout(resolve, 250));
         await this.startClassicInstance(instanceName, this.classicChannels.getBackendByInstance(instanceName, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(channel.channelId, channel.adapterId), this.classicChannels.getModel(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
     }
-    async startInstance(name, config, topicMode) {
+    async startInstance(name, config, topicMode, kind = "fleet-topic") {
         if (this.lifecycle.isPaused(name)) {
             this.logger.info({ name }, "Persisted paused instance — skipping startup");
             return;
@@ -667,7 +767,11 @@ export class FleetManager {
             this.ensureGeneralInstructions(config.working_directory, config.backend);
         }
         this.instanceProcessStatus.delete(name);
-        await this.lifecycle.start(name, config, topicMode);
+        await this.lifecycle.start(name, config, topicMode, {
+            kind,
+            backend: config.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code",
+            model: this.resolveInstanceModel(name).display,
+        });
         // Auto-connect IPC — daemon.start() ensures socket is ready before resolving
         await this.connectIpcToInstance(name);
     }
@@ -845,7 +949,6 @@ export class FleetManager {
         this.configPath = configPath;
         this.loadEnvFile();
         // Rotate fleet.log if oversized (before any logging)
-        const { rotateLogIfNeeded } = await import("./logger.js");
         rotateLogIfNeeded(join(this.dataDir, "fleet.log"));
         const fleet = this.loadConfig(configPath);
         setLocale(detectLocale(fleet)); // user-facing text language (fleet.yaml defaults.locale / timezone)
@@ -889,7 +992,7 @@ export class FleetManager {
         }
         const pidPath = join(this.dataDir, "fleet.pid");
         writeFileSync(pidPath, String(process.pid), "utf-8");
-        this.eventLog = new EventLog(join(this.dataDir, "events.db"));
+        this.eventLog = this.openEventLog();
         // Initialize classic channel manager. The primary adapter (channels[0])
         // migrates legacy single-bot entries and names without a suffix. Classic
         // routing does NOT go through the routing engine (single-key, can't hold two
@@ -978,6 +1081,10 @@ export class FleetManager {
             // Rotate fleet.log daily too (besides the startup size check above), so a
             // long-running fleet doesn't accumulate an unbounded log.
             rotateLogIfNeeded(join(this.dataDir, "fleet.log"));
+            // Instance output.log is pipe-pane (TUI ANSI). Daemon health ticks rotate a
+            // running instance's own log; this sweep is the safety net for every other
+            // kind. One implementation, so the two cannot cover different sets.
+            this.rotateAllInstanceLogs();
         }, () => {
             const instances = Object.keys(this.fleetConfig?.instances ?? {});
             const costMap = new Map();
@@ -1117,9 +1224,29 @@ export class FleetManager {
                 }
             }
         }
-        // Signal systemd: generals ready
-        sdNotify("READY=1");
+        // The systemd watchdog answers exactly one question: is this process still
+        // turning its event loop? Pinging from a timer proves that, and after the
+        // blocking child-process calls were made async it is a meaningful signal —
+        // a deadlocked or frozen fleet stops pinging and systemd restarts it.
+        //
+        // It deliberately does NOT gate on fleet health. "No adapter connected" or
+        // "an instance crashed" must not kill the process: the fleet would be restarted
+        // into the same broken state, and a user who has legitimately stopped every
+        // instance would get a restart loop. Those conditions surface through /health
+        // (which now returns 503) and through the General-topic notifications instead.
         this.watchdogTimer = setInterval(() => sdNotify("WATCHDOG=1"), 30_000);
+        // EventLog.prune() existed but was never called, so `events` and `activity`
+        // grew without bound for the life of the install. Prune once at startup and
+        // daily after that; the timer is unref'd so it never holds the loop open.
+        this.pruneEventLog();
+        this.eventLogPruneTimer = setInterval(() => this.pruneEventLog(), 24 * 60 * 60_000);
+        this.eventLogPruneTimer.unref?.();
+        // Same shape for pipe-pane logs, and for the same reason: the only sweep that
+        // covered them lived inside the daily-summary callback, so it did not run at
+        // all when summaries were off.
+        this.rotateAllInstanceLogs();
+        this.logRotateTimer = setInterval(() => this.rotateAllInstanceLogs(), 24 * 60 * 60_000);
+        this.logRotateTimer.unref?.();
         // Phase 2: Start remaining instances with staggered concurrency
         if (others.length > 0) {
             await this.startInstancesWithConcurrency(others, topicMode);
@@ -1255,6 +1382,15 @@ export class FleetManager {
         // rest of startup finishes. Replay one coalesced reload only after all
         // startup-owned lifecycle work and signal handlers are in place.
         this.finishStartup();
+        // Tell systemd we are ready only now. This used to fire right after the
+        // generals started — before adapters, classic instances, topic creation and the
+        // health server — so `systemctl start` returned success while the fleet was
+        // still deaf: no path existed for a user message to arrive.
+        sdNotify("READY=1");
+        const health = this.getFleetHealth();
+        if (health.status !== "ok") {
+            this.logger.warn({ health }, "Fleet started with problems — see /health");
+        }
     }
     /**
      * Delete inbox files older than retentionDays (by mtime). Cleans the shared
@@ -1371,6 +1507,67 @@ export class FleetManager {
     getAdapterStates() {
         return this.adapterState;
     }
+    /**
+     * Real, checkable fleet health for `/health` and the operator.
+     *
+     * `status` is:
+     *  - `ok`       — at least one adapter connected and every configured instance
+     *                 that should be running is running
+     *  - `degraded` — reachable, but something the operator should look at (an
+     *                 adapter retrying, an instance crashed or stopped)
+     *  - `down`     — the fleet cannot do its job: no adapter is connected, so no
+     *                 message can arrive or be answered
+     *
+     * Deliberately does NOT gate the systemd watchdog — see the comment at the
+     * WATCHDOG timer for why.
+     */
+    getFleetHealth() {
+        const names = Object.keys(this.fleetConfig?.instances ?? {});
+        const counts = { configured: names.length, running: 0, crashed: 0, paused: 0, stopped: 0 };
+        for (const name of names) {
+            const state = this.getInstanceStatus(name);
+            if (state === "running")
+                counts.running++;
+            else if (state === "crashed")
+                counts.crashed++;
+            else if (state === "paused")
+                counts.paused++;
+            else
+                counts.stopped++;
+        }
+        const states = {};
+        let connected = 0;
+        for (const [id, state] of this.adapterState) {
+            states[id] = state.status;
+            if (state.status === "connected")
+                connected++;
+        }
+        const problems = [];
+        if (this.adapterState.size > 0 && connected === 0)
+            problems.push("no channel adapter is connected");
+        if (counts.crashed > 0)
+            problems.push(`${counts.crashed} instance(s) crashed`);
+        for (const [id, state] of this.adapterState) {
+            if (state.status !== "connected")
+                problems.push(`adapter ${id} is ${state.status}`);
+        }
+        if (!this.startupComplete)
+            problems.push("startup has not completed");
+        // "down" is reserved for "cannot receive or answer a message at all". A fleet
+        // with adapters configured but none connected is exactly that.
+        const status = this.adapterState.size > 0 && connected === 0
+            ? "down"
+            : problems.length > 0 ? "degraded" : "ok";
+        return {
+            status,
+            uptime: Math.floor((Date.now() - this.startedAt) / 1000),
+            instances: counts,
+            adapters: { total: this.adapterState.size, connected, states },
+            startupComplete: this.startupComplete,
+            memory: readFleetMemory(),
+            problems,
+        };
+    }
     /** Start the primary adapter (backward-compatible, sets this.adapter) */
     async startSingleAdapter(fleet, channelConfig) {
         const botToken = process.env[channelConfig.bot_token_env];
@@ -1397,6 +1594,9 @@ export class FleetManager {
         this.adapter.on("message", safeHandler(async (msg) => {
             await this.handleInboundMessage(msg);
         }, this.logger, "adapter.message"));
+        this.adapter.on("reaction", safeHandler(async (r) => {
+            await this.handleInboundReaction(r);
+        }, this.logger, "adapter.reaction"));
         this.adapter.on("callback_query", safeHandler(async (data) => {
             if (await this.handleClassicBackendSelection(data))
                 return;
@@ -1571,17 +1771,7 @@ export class FleetManager {
                     await data.respond(t("not_authorized"));
                     return;
                 }
-                try {
-                    const { execSync } = await import("node:child_process");
-                    const backend = this.fleetConfig?.defaults?.backend || "claude-code";
-                    const result = execSync(`agend backend doctor ${backend}`, { timeout: 30_000, encoding: "utf-8" });
-                    const clean = result.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
-                    await data.respond(clean || "No output");
-                }
-                catch (err) {
-                    const output = (err.stdout ?? err.message ?? "Doctor failed").replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
-                    await data.respond(output);
-                }
+                await data.respond(await this.runBackendDoctor());
             }
             else if (data.command === "status") {
                 const text = await this.topicCommands.getStatusText();
@@ -1692,6 +1882,9 @@ export class FleetManager {
         adapter.on("message", safeHandler(async (msg) => {
             await this.handleInboundMessage(msg);
         }, this.logger, `adapter[${adapterId}].message`));
+        adapter.on("reaction", safeHandler(async (r) => {
+            await this.handleInboundReaction(r);
+        }, this.logger, `adapter[${adapterId}].reaction`));
         adapter.on("callback_query", safeHandler(async (data) => {
             if (await this.handleClassicBackendSelection(data))
                 return;
@@ -1853,17 +2046,7 @@ export class FleetManager {
                     await data.respond(t("not_authorized"));
                     return;
                 }
-                try {
-                    const { execSync } = await import("node:child_process");
-                    const backend = this.fleetConfig?.defaults?.backend || "claude-code";
-                    const result = execSync(`agend backend doctor ${backend}`, { timeout: 30_000, encoding: "utf-8" });
-                    const clean = result.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
-                    await data.respond(clean || "No output");
-                }
-                catch (err) {
-                    const output = (err.stdout ?? err.message ?? "Doctor failed").replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
-                    await data.respond(output);
-                }
+                await data.respond(await this.runBackendDoctor());
             }
             else if (data.command === "status") {
                 const text = await this.topicCommands.getStatusText();
@@ -1931,19 +2114,38 @@ export class FleetManager {
         this.logger.info({ adapterId, type: channelConfig.type }, "Additional adapter started");
     }
     /** Connect IPC to a single instance with all handlers */
-    async connectIpcToInstance(name) {
+    connectIpcToInstance(name) {
+        const inFlight = this.ipcConnectInFlight.get(name);
+        if (inFlight)
+            return inFlight;
+        const connection = this.connectIpcToInstanceInternal(name)
+            .finally(() => {
+            if (this.ipcConnectInFlight.get(name) === connection) {
+                this.ipcConnectInFlight.delete(name);
+            }
+        });
+        this.ipcConnectInFlight.set(name, connection);
+        return connection;
+    }
+    async connectIpcToInstanceInternal(name) {
         // Close existing client to prevent socket leak on reconnect
         const existing = this.instanceIpcClients.get(name);
         if (existing) {
-            this.ipcStoppingInstances.add(name);
+            // Remove application listeners before destroying the socket. Even if a
+            // future regression creates two clients, the replaced one cannot keep
+            // handling fleet_outbound messages as an orphan.
+            existing.removeAllListeners();
             try {
-                existing.close();
+                await existing.close();
             }
             catch (err) {
                 this.logger.debug({ err, name }, "IPC client close failed (likely already closed)");
             }
-            this.instanceIpcClients.delete(name);
-            this.ipcStoppingInstances.delete(name);
+            finally {
+                if (this.instanceIpcClients.get(name) === existing) {
+                    this.instanceIpcClients.delete(name);
+                }
+            }
         }
         const sockPath = join(this.getInstanceDir(name), "channel.sock");
         if (!existsSync(sockPath))
@@ -2002,6 +2204,9 @@ export class FleetManager {
                 else if (msg.type === "instance_process_state") {
                     this.cacheInstanceProcessStatus(name, msg.status);
                 }
+                else if (msg.type === "instance_activity") {
+                    this.cacheInstanceActivity(name, msg.activity);
+                }
                 else if (msg.type === "instance_state" || msg.type === "instance_state_response") {
                     this.cacheInstanceExecutionState(name, msg);
                     if (msg.type === "instance_state_response") {
@@ -2021,6 +2226,10 @@ export class FleetManager {
             }
             // Auto-reconnect on disconnect (unless intentionally stopping)
             ipc.on("disconnect", () => {
+                // A delayed event from a replaced/stale client must never delete the
+                // current connection or start another reconnect loop.
+                if (this.instanceIpcClients.get(name) !== ipc)
+                    return;
                 this.instanceIpcClients.delete(name);
                 if (this.ipcStoppingInstances.has(name))
                     return;
@@ -2032,7 +2241,20 @@ export class FleetManager {
         }
     }
     /** Attempt IPC reconnection with exponential backoff */
-    async ipcReconnect(name) {
+    ipcReconnect(name) {
+        const inFlight = this.ipcReconnectInFlight.get(name);
+        if (inFlight)
+            return inFlight;
+        const reconnect = this.runIpcReconnect(name)
+            .finally(() => {
+            if (this.ipcReconnectInFlight.get(name) === reconnect) {
+                this.ipcReconnectInFlight.delete(name);
+            }
+        });
+        this.ipcReconnectInFlight.set(name, reconnect);
+        return reconnect;
+    }
+    async runIpcReconnect(name) {
         for (let attempt = 1;; attempt++) {
             if (this.ipcStoppingInstances.has(name) || !this.daemons.has(name))
                 return;
@@ -2055,9 +2277,22 @@ export class FleetManager {
                 if (existsSync(windowIdPath)) {
                     const windowId = readFileSync(windowIdPath, "utf-8").trim();
                     if (windowId) {
+                        // Async with an explicit timeout: this was execSync with NO timeout at
+                        // all, so a wedged tmux server blocked the whole fleet event loop
+                        // indefinitely — while we were here to diagnose a lost connection.
+                        // A timeout is also the correct signal: an unresponsive tmux server
+                        // means we cannot verify the pane, which is treated as dead (the same
+                        // conclusion the old code reached only by throwing).
                         try {
-                            const { execSync } = await import("node:child_process");
-                            execSync(`tmux list-panes -t "${windowId}"`, { stdio: "ignore" });
+                            const { execFile } = await import("node:child_process");
+                            const { promisify } = await import("node:util");
+                            const { getTmuxSocketName } = await import("./paths.js");
+                            // Honour socket isolation: without -L this queried the user's default
+                            // tmux server instead of the fleet's, so under a custom AGEND_HOME the
+                            // check was meaningless (it reported every pane dead).
+                            const socket = getTmuxSocketName();
+                            const args = socket ? ["-L", socket, "list-panes", "-t", windowId] : ["list-panes", "-t", windowId];
+                            await promisify(execFile)("tmux", args, { timeout: 5_000 });
                         }
                         catch {
                             // Pane dead — respawn
@@ -2078,6 +2313,12 @@ export class FleetManager {
         if (this.adapterRestarting.has(id))
             return;
         this.adapterRestarting.add(id);
+        // Reflect reality in adapterState throughout. This loop used to leave the state
+        // untouched, so getAdapterStates() — and therefore /health and the dashboard —
+        // kept reporting "connected" for an adapter that had been down for hours. An
+        // adapter's true status was simply not knowable from inside the process.
+        const previous = this.adapterState.get(id);
+        this.adapterState.set(id, { status: "retrying", retryCount: 0, lastError: previous?.lastError });
         try {
             for (let attempt = 1;; attempt++) {
                 if (this.ipcStoppingInstances.has("__fleet_stopping__"))
@@ -2090,9 +2331,16 @@ export class FleetManager {
                     await adapter.stop().catch(() => { });
                     await adapter.start();
                     this.logger.info({ id, attempt }, "Adapter restarted successfully");
+                    this.adapterState.set(id, { status: "connected", retryCount: 0 });
                     return;
                 }
-                catch { /* retry */ }
+                catch (err) {
+                    this.adapterState.set(id, {
+                        status: "retrying",
+                        retryCount: attempt,
+                        lastError: err?.message ?? String(err),
+                    });
+                }
                 if (attempt % 10 === 0) {
                     this.logger.warn({ id, attempt }, "Adapter restart still failing");
                 }
@@ -2128,6 +2376,45 @@ export class FleetManager {
         }
         return generals[0];
     }
+    /**
+     * A user reacted to one of the bot's messages (#408).
+     *
+     * A reaction is context, not a message (#432, reworking #413): it never triggers
+     * an agent turn and never wakes anything. It is queued in the event log and rides
+     * into the instance's NEXT real message as one compact leading line —
+     * `[Recent reactions: 👍×2 from hanhanv]` — after which it is marked consumed.
+     * No pending reactions → no line → zero context spent, which is the common case.
+     */
+    async handleInboundReaction(r) {
+        const instanceName = this.resolveSlashTarget(r.threadId ?? r.chatId, r.adapterId);
+        if (!instanceName) {
+            this.logger.debug({ emoji: r.emoji, chatId: r.chatId }, "Reaction in an unrouted channel — ignoring");
+            return;
+        }
+        this.eventLog?.logActivity("reaction", r.username, `${r.emoji} ${r.action}`, instanceName);
+        if (r.action === "add") {
+            this.eventLog?.addReaction(instanceName, r.messageId, r.username, r.emoji);
+        }
+        else {
+            // Withdrawn before anyone saw it → it never happened. See removeReaction.
+            this.eventLog?.removeReaction(instanceName, r.messageId, r.username, r.emoji);
+        }
+    }
+    /**
+     * The queued-reaction summary for an instance's next real message, or {} when
+     * nothing is pending (the common case must add zero context). The consume
+     * callback is separate from the fetch so reactions are only marked once the
+     * message actually went out — a failed delivery keeps them queued.
+     */
+    pendingReactionsMeta(instanceName) {
+        const pending = this.eventLog?.pendingReactions(instanceName);
+        if (!pending)
+            return { meta: {}, consume: () => { } };
+        return {
+            meta: { pending_reactions: pending.summary },
+            consume: () => this.eventLog?.markReactionsConsumed(instanceName, pending.maxId),
+        };
+    }
     async handleInboundMessage(msg) {
         const threadId = msg.threadId || undefined;
         this.logger.debug({ source: msg.source, chatId: msg.chatId, threadId, userId: msg.userId, isBotMessage: msg.isBotMessage, textLen: (msg.text ?? "").length, text: (msg.text ?? "").slice(0, 80) }, "handleInboundMessage entry");
@@ -2465,6 +2752,7 @@ export class FleetManager {
                 }
                 this.warnIfRateLimited(generalInstance, msg);
                 const { text, extraMeta } = await processAttachments(msg, inboundAdapter, this.logger, generalInstance);
+                const generalReactions = this.pendingReactionsMeta(generalInstance);
                 try {
                     await this.deliverToInstance(generalInstance, {
                         type: "fleet_inbound",
@@ -2480,9 +2768,11 @@ export class FleetManager {
                             adapter_id: msg.adapterId,
                             source: msg.source,
                             ...(msg.replyToText ? { reply_to_text: msg.replyToText } : {}),
+                            ...generalReactions.meta,
                             ...extraMeta,
                         },
                     });
+                    generalReactions.consume();
                     this.lastInboundUser.set(generalInstance, msg.username);
                     this.logger.info(`${msg.username} → ${generalInstance}: ${(text ?? "").slice(0, 100)}`);
                     this.eventLog?.logActivity("message", msg.username, (text ?? "").slice(0, 200), generalInstance);
@@ -2558,6 +2848,7 @@ export class FleetManager {
         this.setTopicIcon(instanceName, "blue");
         this.warnIfRateLimited(instanceName, msg);
         const { text, extraMeta } = await processAttachments(msg, inboundAdapter, this.logger, instanceName);
+        const reactions = this.pendingReactionsMeta(instanceName);
         try {
             await this.deliverToInstance(instanceName, {
                 type: "fleet_inbound",
@@ -2573,9 +2864,13 @@ export class FleetManager {
                     adapter_id: msg.adapterId,
                     source: msg.source,
                     ...(msg.replyToText ? { reply_to_text: msg.replyToText } : {}),
+                    ...reactions.meta,
                     ...extraMeta,
                 },
             });
+            // Only after the message actually went out. A failed delivery keeps the
+            // reactions queued for the retry / the next message.
+            reactions.consume();
         }
         catch (err) {
             this.logger.warn({ err: err.message, instanceName }, "Wake/delivery failed");
@@ -2667,8 +2962,18 @@ export class FleetManager {
         // Route standard channel tools (reply, react, edit_message, download_attachment)
         if (routeToolCall(outAdapter, tool, args, threadId, respond)) {
             if (tool === "reply") {
-                // Agent answered — retire its pending cancel button and mark ✅ done.
-                this.clearCancelButton(instanceName);
+                // A reply is NOT proof the turn is over: on multi-step work an agent
+                // replies ("starting…") and keeps going for many minutes. Retiring the
+                // button here left the channel looking idle with no way to cancel and no
+                // sign anything was happening (#410). Idle state owns retirement now; if the
+                // instance is still working, move the button below the new reply so it stays
+                // the last thing in the channel.
+                if (this.getInstanceIdle(instanceName)) {
+                    this.clearCancelButton(instanceName);
+                }
+                else {
+                    void this.sendCancelButton(instanceName);
+                }
                 this.reactDone(instanceName);
                 const replyTo = this.lastInboundUser.get(instanceName) ?? "user";
                 this.logger.info(`${instanceName} → ${replyTo}: ${(args.text ?? "").slice(0, 100)}`);
@@ -3299,6 +3604,9 @@ export class FleetManager {
         }
     }
     async removeInstance(name) {
+        // Drop cached pane context — the map is keyed by instance name and nothing
+        // else evicted deleted entries, so it grew for the life of the process.
+        forgetInstanceContext(name);
         // Clean up schedules (scheduler is fleet-level, not lifecycle-level)
         const config = this.fleetConfig?.instances[name];
         if (this.scheduler && config?.topic_id) {
@@ -3404,6 +3712,171 @@ export class FleetManager {
         this.collabInstances.add(instanceName);
         return true;
     }
+    /**
+     * Open the event log, tolerating a corrupt file.
+     *
+     * `events.db` holds history only — event rows and the activity feed. Nothing the
+     * fleet needs to run depends on it, and every consumer already uses
+     * `this.eventLog?.`. An unguarded `new EventLog(...)` here meant a corrupt or
+     * unreadable history file (a truncated WAL after a hard kill, a full disk)
+     * threw during startAll and the WHOLE FLEET FAILED TO BOOT — trading every
+     * running agent for a file whose only job is reporting.
+     *
+     * So: try, move a bad file aside and retry once with a fresh one, and if even
+     * that fails carry on without an event log.
+     */
+    /**
+     * Run `agend backend doctor` for the fleet's default backend and return its
+     * cleaned output.
+     *
+     * Async on purpose: this was `execSync` with a 30s timeout, reachable by any
+     * allowlisted user through `/doctor`. While it ran, the entire fleet event loop
+     * was frozen — no IPC, no adapter, no message delivery, no health responses,
+     * and critically no WATCHDOG ping, so a slow doctor could push past
+     * WatchdogSec and have systemd SIGABRT the fleet.
+     */
+    async runBackendDoctor() {
+        const stripAnsi = (s) => s.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
+        const backend = this.fleetConfig?.defaults?.backend || "claude-code";
+        try {
+            const { execFile } = await import("node:child_process");
+            const { promisify } = await import("node:util");
+            // execFile with an argv array — no shell, so the backend name cannot be
+            // interpreted as a command even if config is malformed.
+            const { stdout } = await promisify(execFile)("agend", ["backend", "doctor", backend], {
+                timeout: 30_000,
+                encoding: "utf-8",
+            });
+            return stripAnsi(stdout) || "No output";
+        }
+        catch (err) {
+            const e = err;
+            return stripAnsi(e.stdout ?? e.message ?? "Doctor failed");
+        }
+    }
+    /** Drop event/activity rows older than the retention window. Best-effort. */
+    /**
+     * Cap every instance's pipe-pane log, walking the instances **directory** rather
+     * than the config.
+     *
+     * A running instance rotates its own log on each health tick, so the ones that
+     * need this are the ones nothing else looks at:
+     *
+     *   - deleted instances, whose directory outlives the config entry. Nothing ever
+     *     touched these again. On the machine this was found on, one held 122 MB and
+     *     another 74 MB, out of 622 MB of pipe-pane logs in total.
+     *   - classic instances, which live in classicChannels, not fleetConfig.instances,
+     *     and so were never in the old config-driven loop at all.
+     *   - stopped instances, which have no health tick running.
+     *
+     * pipe-pane writes raw TUI output, so a wedged splash screen can emit ANSI frames
+     * at animation rate. Unbounded growth here fills the disk, which takes the whole
+     * fleet down rather than one instance.
+     */
+    rotateAllInstanceLogs() {
+        const root = join(this.dataDir, "instances");
+        let entries;
+        try {
+            entries = readdirSync(root, { withFileTypes: true });
+        }
+        catch {
+            return; // no instances directory yet
+        }
+        for (const entry of entries) {
+            if (!entry.isDirectory())
+                continue;
+            // rotateLogIfNeeded is already best-effort and returns early on a missing
+            // file, so a directory without a pipe-pane log costs one stat.
+            rotateLogIfNeeded(join(root, entry.name, "output.log"));
+        }
+    }
+    pruneEventLog() {
+        try {
+            this.eventLog?.prune(FleetManager.EVENT_LOG_RETENTION_DAYS);
+        }
+        catch (err) {
+            this.logger.warn({ err }, "Event log prune failed");
+        }
+    }
+    openEventLog() {
+        const dbPath = join(this.dataDir, "events.db");
+        try {
+            return new EventLog(dbPath);
+        }
+        catch (err) {
+            this.logger.error({ err, dbPath }, "events.db unusable — moving it aside and starting a fresh one");
+            const stamp = new Date().toISOString().replace(/[:.]/g, "-");
+            for (const suffix of ["", "-wal", "-shm"]) {
+                try {
+                    renameSync(`${dbPath}${suffix}`, `${dbPath}${suffix}.corrupt-${stamp}`);
+                }
+                catch { /* may not exist */ }
+            }
+            try {
+                return new EventLog(dbPath);
+            }
+            catch (retryErr) {
+                // History is worth losing; a fleet that won't start is not.
+                this.logger.error({ err: retryErr, dbPath }, "Could not open a fresh events.db — continuing without event logging");
+                return null;
+            }
+        }
+    }
+    /**
+     * Report a fleet-level fault (not attributable to one instance) to the General
+     * topic, so the operator learns about it without reading daemon.log.
+     *
+     * Throttled per distinct message: an unhandled rejection typically comes from a
+     * loop (a poller, a repeating timer), and one channel message per occurrence
+     * would bury the topic — which is worse than silence. First occurrence goes out
+     * immediately, repeats are suppressed for THROTTLE_MS and then re-sent with a
+     * count.
+     *
+     * The log line is written by the caller regardless: if every adapter is down,
+     * the only notification path is the one that is broken.
+     */
+    notifyFleetError(text) {
+        const now = Date.now();
+        const key = text.slice(0, 200);
+        const seen = this.fleetErrorNotices.get(key);
+        if (seen && now - seen.at < FleetManager.FLEET_ERROR_THROTTLE_MS) {
+            seen.suppressed++;
+            return;
+        }
+        const suppressed = seen?.suppressed ?? 0;
+        this.fleetErrorNotices.set(key, { at: now, suppressed: 0 });
+        // Bound the map: it is keyed by message text, and a message with a varying
+        // suffix (a path, an id) would otherwise grow it without limit.
+        if (this.fleetErrorNotices.size > 100) {
+            const oldest = this.fleetErrorNotices.keys().next().value;
+            if (oldest !== undefined)
+                this.fleetErrorNotices.delete(oldest);
+        }
+        const body = suppressed > 0
+            ? `${text}\n(plus ${suppressed} more in the last ${Math.round(FleetManager.FLEET_ERROR_THROTTLE_MS / 60_000)}m)`
+            : text;
+        // Resolved from config, NOT findGeneralInstance(): that requires a live daemon,
+        // and a fleet-level fault is exactly when the General may be down. The topic
+        // itself still exists, and notifyInstanceTopic only needs adapter + group +
+        // topic_id to post into it.
+        const general = Object.entries(this.fleetConfig?.instances ?? {})
+            .find(([, config]) => config.general_topic === true)?.[0];
+        if (general) {
+            this.notifyInstanceTopic(general, body);
+            return;
+        }
+        // No General instance — fall back to the primary channel's group.
+        const channelCfg = this.getChannelConfig();
+        const groupId = channelCfg?.group_id;
+        if (this.adapter && groupId) {
+            this.adapter.sendText(String(groupId), body)
+                .catch(err => this.logger.warn({ err }, "Failed to send fleet error notification"));
+            return;
+        }
+        this.logger.warn({ text: body }, "Fleet error had no notification target (no General instance, no adapter)");
+    }
+    static FLEET_ERROR_THROTTLE_MS = 10 * 60_000;
+    fleetErrorNotices = new Map();
     notifyInstanceTopic(instanceName, text, extraOpts) {
         const adapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
         if (!adapter)
@@ -3533,7 +4006,12 @@ export class FleetManager {
                 threadId: sent.threadId ?? threadId,
                 correlationId,
                 retryCount: 0,
+                // Elapsed time is measured from when this button was posted — i.e. from
+                // when the work was handed over — not from the pane's working transition,
+                // which resets if the CLI blips idle mid-turn.
+                startedAt: Date.now(),
             };
+            this.startProgressTicker(entry);
             // Idle-check backstop: every 5min, if the instance is idle, retire the
             // button. Covers turns that end without hitting a clear trigger (reply /
             // cancel / correlation). Cleared in discardButton when the entry is removed.
@@ -3554,6 +4032,97 @@ export class FleetManager {
             this.logger.warn({ err: e.message, instanceName }, "Failed to send cancel button");
         }
     }
+    /**
+     * The cancel button's text for a given elapsed time.
+     *
+     * Below the threshold it keeps the original wording, so a normal quick answer
+     * looks exactly as it did before. Past it, the button doubles as the live
+     * progress indicator (#409) — the channel showed nothing at all during long work,
+     * and once the agent had replied once there was no sign it was still going.
+     */
+    static progressText(elapsedMs, activity) {
+        if (elapsedMs < PROGRESS_MIN_ELAPSED_MS)
+            return "👀 處理中…";
+        const totalSeconds = Math.floor(elapsedMs / 1000);
+        const minutes = Math.floor(totalSeconds / 60);
+        const seconds = totalSeconds % 60;
+        const elapsed = minutes >= 60
+            ? `${Math.floor(minutes / 60)}h ${minutes % 60}m`
+            : `${minutes}m ${String(seconds).padStart(2, "0")}s`;
+        const detail = FleetManager.sanitizeActivity(activity);
+        return detail
+            ? `⏳ 處理中… (已進行 ${elapsed} · ${detail})`
+            : `⏳ 處理中… (已進行 ${elapsed})`;
+    }
+    /**
+     * Make a tool summary safe to paste into a channel message.
+     *
+     * The text is agent-controlled (it is built from tool inputs — file paths,
+     * shell commands), so it gets flattened to one line, capped, and stripped of
+     * the two Discord mass-mention triggers. Neither channel renders it with a
+     * parse mode, so no markup escaping is needed beyond that.
+     */
+    static sanitizeActivity(activity) {
+        if (!activity)
+            return null;
+        const flat = activity.replace(/\s+/g, " ").replace(/@(everyone|here)/g, "@​$1").trim();
+        if (!flat)
+            return null;
+        return flat.length > PROGRESS_ACTIVITY_MAX_CHARS
+            ? `${flat.slice(0, PROGRESS_ACTIVITY_MAX_CHARS - 1)}…`
+            : flat;
+    }
+    /**
+     * Remember what an instance is currently doing, for the progress line.
+     *
+     * Best-effort by design: only backends that expose a live activity feed report
+     * anything, and the progress line simply omits the detail for the rest. It is
+     * never used to decide anything — purely what the user is shown.
+     */
+    cacheInstanceActivity(name, activity) {
+        if (activity)
+            this.instanceActivity.set(name, activity);
+        else
+            this.instanceActivity.delete(name);
+    }
+    /**
+     * Refresh the button's text in place while the instance keeps working.
+     *
+     * Uses `editAlert`, NOT `editMessage`: on Telegram the latter omits reply_markup,
+     * and the Bot API treats that as "clear the keyboard" — so editing with it would
+     * delete the very cancel button this is trying to keep alive.
+     */
+    startProgressTicker(entry) {
+        entry.progressTimer = setInterval(() => {
+            if (!this.cancelButtons.has(entry.messageId)) {
+                clearInterval(entry.progressTimer);
+                return;
+            }
+            // Idle means the turn ended; the idle-edge handler retires the button.
+            if (this.getInstanceIdle(entry.instanceName))
+                return;
+            const text = FleetManager.progressText(Date.now() - (entry.startedAt ?? Date.now()), this.instanceActivity.get(entry.instanceName));
+            if (text === entry.lastProgressText)
+                return; // nothing changed — skip the API call
+            const adapter = this.getAdapterForInstance(entry.instanceName) ?? this.adapter;
+            if (!adapter?.editAlert)
+                return;
+            entry.lastProgressText = text;
+            adapter.editAlert(entry.chatId, entry.messageId, {
+                type: "cancel",
+                instanceName: entry.instanceName,
+                message: text,
+                choices: [{ id: `cancel:${entry.instanceName}`, label: t("cancel.button") }],
+            }, entry.threadId ? { threadId: entry.threadId } : undefined)
+                .catch(err => {
+                // A failed progress edit must never escalate: the button still works and
+                // the next tick retries. Common causes are a deleted message or a
+                // rate limit.
+                this.logger.debug({ err, instanceName: entry.instanceName }, "Progress edit failed");
+            });
+        }, PROGRESS_UPDATE_INTERVAL_MS);
+        entry.progressTimer.unref?.();
+    }
     /** Retire (delete) every cancel button belonging to an instance. */
     retireInstanceButtons(instanceName) {
         // Snapshot first — retireButton may delete entries from the map on success.
@@ -3585,6 +4154,8 @@ export class FleetManager {
             clearTimeout(entry.retryTimer);
         if (entry.idleCheckTimer)
             clearInterval(entry.idleCheckTimer);
+        if (entry.progressTimer)
+            clearInterval(entry.progressTimer);
         this.cancelButtons.delete(entry.messageId);
     }
     /** Re-attempt a failed button delete up to CANCEL_BTN_MAX_RETRIES times. */
@@ -4300,22 +4871,49 @@ When users create specialized instances, suggest these configurations:
         catch { /* missing / stale / corrupt */ }
         return null;
     }
-    /** Resolve the model currently configured for a fleet or ClassicBot instance. */
-    currentModelForInstance(instanceName) {
+    /**
+     * Resolve the effective model for a fleet or ClassicBot instance, plus where it
+     * came from. Single source of truth for `/model` and `/ctx` — precedence:
+     * per-instance → fleet defaults → classic channel → CLI's own default (from the
+     * cli-env probe cache) → unresolved.
+     */
+    resolveInstanceModel(instanceName) {
+        const done = (model, source, reason) => ({
+            model,
+            source,
+            reason,
+            // Make an inherited CLI default legible instead of the bare word "default".
+            display: source === "cli-default" ? `${model} (default)`
+                : source === "unresolved" ? `default (${reason ?? "unresolved"})`
+                    : model,
+        });
         const fleetInstance = this.fleetConfig?.instances[instanceName];
         if (fleetInstance) {
-            const fleetModel = fleetInstance.model ?? this.fleetConfig?.defaults?.model;
-            if (fleetModel?.trim())
-                return fleetModel.trim();
+            if (fleetInstance.model?.trim())
+                return done(fleetInstance.model.trim(), "instance");
+            const fleetDefault = this.fleetConfig?.defaults?.model;
+            if (fleetDefault?.trim())
+                return done(fleetDefault.trim(), "fleet-default");
         }
         const classic = this.classicChannels?.getAll().find(ch => ch.instanceName === instanceName);
         if (classic) {
             const classicModel = this.classicChannels?.getModel(classic.channelId, classic.adapterId, this.fleetConfig?.defaults?.model);
             if (classicModel?.trim())
-                return classicModel.trim();
-        }
-        const cachedModel = this.readCliEnv(this.backendNameForInstance(instanceName))?.currentModel;
-        return cachedModel?.trim() || "default";
+                return done(classicModel.trim(), "classic");
+        }
+        // Nothing configured → show what the CLI itself defaults to (kiro default_model,
+        // grok "Default model:", codex config.toml, agy settings.json), cached by the probe.
+        const cliEnv = this.readCliEnv(this.backendNameForInstance(instanceName));
+        const cachedModel = cliEnv?.currentModel;
+        if (cachedModel?.trim())
+            return done(cachedModel.trim(), "cli-default");
+        // Say WHY it's unresolved: no fresh probe yet vs. the CLI not exposing a default
+        // (e.g. claude-code's default is account-side, opencode's is provider-side).
+        return done("default", "unresolved", cliEnv ? "this CLI does not report a default" : "not probed yet");
+    }
+    /** Human-readable effective model, e.g. `auto (default)`. Used by /ctx. */
+    modelDisplayForInstance(instanceName) {
+        return this.resolveInstanceModel(instanceName).display;
     }
     modelChoiceLabel(option, currentModel) {
         const label = option.description ? `${option.label} — ${option.description}` : option.label;
@@ -4395,7 +4993,8 @@ When users create specialized instances, suggest these configurations:
             await data.respond(`No model list available for ${name}. Type \`/model <name>\` to set one directly.`);
             return;
         }
-        const currentModel = this.currentModelForInstance(name);
+        // Raw id for ✓-matching options; display resolves an inherited CLI default.
+        const { model: currentModel, display: currentDisplay } = this.resolveInstanceModel(name);
         const nonce = randomBytes(6).toString("hex");
         const choices = options.slice(0, 25).map(o => ({
             id: `${MODEL_SELECT_CALLBACK_PREFIX}${nonce}:${o.id}`,
@@ -4405,7 +5004,7 @@ When users create specialized instances, suggest these configurations:
         timer.unref?.();
         this.pendingModelSelects.set(nonce, { instanceName: name, model: "", userId: data.userId, channelId: data.channelId, timer, respond: data.respond });
         try {
-            await data.respondChoices(`Current model: **${currentModel}**\nSelect a new model:`, choices);
+            await data.respondChoices(`Current model: **${currentDisplay}**\nSelect a new model:`, choices);
         }
         catch (err) {
             this.pendingModelSelects.delete(nonce);
@@ -4424,7 +5023,7 @@ When users create specialized instances, suggest these configurations:
         if (options.length === 0) {
             return `No model list available for ${instanceName}. Use \`/model <name>\` to set one directly.`;
         }
-        const currentModel = this.currentModelForInstance(instanceName);
+        const { model: currentModel, display: currentDisplay } = this.resolveInstanceModel(instanceName);
         const nonce = randomBytes(6).toString("hex");
         const choices = options.slice(0, 25).map(o => ({
             id: `${MODEL_SELECT_CALLBACK_PREFIX}${nonce}:${o.id}`,
@@ -4444,7 +5043,7 @@ When users create specialized instances, suggest these configurations:
         timer.unref?.();
         this.pendingModelSelects.set(nonce, { instanceName, model: "", userId, channelId, timer, respond, adapter, adapterChatId: chatId, adapterThreadId: threadId });
         try {
-            const menuMessageId = await adapter.promptUser(chatId, `Current model: ${currentModel}\nSelect a new model:`, choices, { threadId });
+            const menuMessageId = await adapter.promptUser(chatId, `Current model: ${currentDisplay}\nSelect a new model:`, choices, { threadId });
             const pending = this.pendingModelSelects.get(nonce);
             if (pending)
                 pending.menuMessageId = menuMessageId;
@@ -4503,9 +5102,19 @@ When users create specialized instances, suggest these configurations:
             // DC path: respond immediately with progress text
             await pending.respond(progressText).catch(() => { });
         }
-        // Apply model in background — don't await here (keeps callback handler fast)
+        // Apply model in background — don't await here (keeps callback handler fast).
+        // Guarded: applyModel() restarts the instance, and an unguarded rejection here
+        // meant a user picking from the /model menu could take the whole fleet down.
+        // On failure the user gets told, rather than the click silently doing nothing.
         void (async () => {
-            const result = await this.applyModel(pending.instanceName, model);
+            let result;
+            try {
+                result = await this.applyModel(pending.instanceName, model);
+            }
+            catch (err) {
+                this.logger.error({ err, instance: pending.instanceName, model }, "Model switch failed");
+                result = `Model switch to \`${model}\` failed: ${err instanceof Error ? err.message : String(err)}`;
+            }
             if (pending.adapter && pending.adapterChatId) {
                 if (progressMsgId) {
                     pending.adapter.editMessage(pending.adapterChatId, progressMsgId, result, pending.adapterThreadId).catch(() => {
@@ -4743,7 +5352,7 @@ When users create specialized instances, suggest these configurations:
             ...(preTaskCommand ? { pre_task_command: preTaskCommand } : {}),
         };
         const topicMode = this.fleetConfig?.channel?.mode === "topic";
-        await this.startInstance(instanceName, config, topicMode);
+        await this.startInstance(instanceName, config, topicMode, "classic");
     }
     /** Handle /start slash command — register classic channel */
     async handleClassicStart(channelId, channelName, userId, guildId, adapterId, backend) {
@@ -4785,11 +5394,24 @@ When users create specialized instances, suggest these configurations:
         this.logger.info({ channelId, adapterId, instanceName: ch.instanceName }, "Classic channel stopped");
         return t("classic.stopped");
     }
-    async stopAll() {
+    /**
+     * Idempotent while in flight: SIGINT and SIGTERM share one handler and the
+     * uncaughtException path calls this too, so overlapping runs were possible —
+     * each snapshotting the daemon map and calling stop() on the same daemons
+     * concurrently. Deliberately NOT `async`, so callers receive the same promise
+     * object rather than a fresh wrapper around it. The latch clears when the run
+     * settles, so a later genuine stop (after a restart) still does the work.
+     */
+    stopAll() {
+        this.stopAllInFlight ??= this.doStopAll().finally(() => { this.stopAllInFlight = null; });
+        return this.stopAllInFlight;
+    }
+    stopAllInFlight = null;
+    async doStopAll() {
         this.startupComplete = false;
         this.reloadPending = false;
         this.ipcStoppingInstances.add("__fleet_stopping__");
-        sdNotify("STOPPING=1");
+        sdNotifyBlocking("STOPPING=1");
         if (this.watchdogTimer) {
             clearInterval(this.watchdogTimer);
             this.watchdogTimer = null;
@@ -4809,6 +5431,26 @@ When users create specialized instances, suggest these configurations:
             clearInterval(this.updateCheckTimer);
             this.updateCheckTimer = null;
         }
+        if (this.eventLogPruneTimer) {
+            clearInterval(this.eventLogPruneTimer);
+            this.eventLogPruneTimer = null;
+        }
+        if (this.logRotateTimer) {
+            clearInterval(this.logRotateTimer);
+            this.logRotateTimer = null;
+        }
+        // Cancel-button timers were never cleared here. The idle-check interval is not
+        // unref'd, so it held the event loop open past shutdown and kept retrying
+        // deletes against an adapter that was already gone.
+        for (const entry of [...this.cancelButtons.values()]) {
+            if (entry.retryTimer)
+                clearTimeout(entry.retryTimer);
+            if (entry.idleCheckTimer)
+                clearInterval(entry.idleCheckTimer);
+            if (entry.progressTimer)
+                clearInterval(entry.progressTimer);
+        }
+        this.cancelButtons.clear();
         if (this.topicCleanupTimer) {
             clearInterval(this.topicCleanupTimer);
             this.topicCleanupTimer = null;
@@ -5027,6 +5669,15 @@ When users create specialized instances, suggest these configurations:
                 removedRatio,
                 validationErrors: validation.errors,
             }, "Refusing unsafe fleet config reload; running configuration was kept");
+            // Tell the operator. A silently ignored config edit is the most confusing
+            // possible outcome: they change fleet.yaml, send SIGHUP, and nothing happens
+            // with no explanation anywhere they are looking.
+            const why = !validation.valid
+                ? `validation failed:\n${validation.errors.map(e => `• ${e.path}: ${e.message}`).join("\n")}`
+                : unsafeEmpty
+                    ? `it removed every instance (${oldCount} → 0)`
+                    : `it removed more than half the instances (${oldCount} → ${newCount})`;
+            this.notifyFleetError(`⚠️ fleet.yaml reload REJECTED — ${why}\nThe previous configuration is still running.`);
             return;
         }
         this.routing.rebuild(this.fleetConfig);
@@ -5226,10 +5877,20 @@ When users create specialized instances, suggest these configurations:
     // ── Update check ────────────────────────────────────────────────────
     async checkForUpdates() {
         try {
-            const { execSync } = await import("node:child_process");
+            // Both npm lookups are async: as execSync they froze the fleet event loop for
+            // up to 15s each, and on a beta build BOTH ran — 30s with no WATCHDOG ping,
+            // past WatchdogSec's half-interval and enough for systemd to SIGABRT the fleet
+            // for a background version check.
+            const { execFile } = await import("node:child_process");
+            const { promisify } = await import("node:util");
+            const execFileP = promisify(execFile);
+            const npmVersion = async (spec) => {
+                const { stdout } = await execFileP("npm", ["view", spec, "version"], { timeout: 15_000 });
+                return stdout.toString().trim();
+            };
             const pkgPath = join(dirname(fileURLToPath(import.meta.url)), "..", "package.json");
             const currentVersion = JSON.parse(readFileSync(pkgPath, "utf-8")).version ?? "0.0.0";
-            const latest = execSync("npm view @songsid/agend version", { stdio: "pipe", timeout: 15_000 }).toString().trim();
+            const latest = await npmVersion("@songsid/agend");
             let target = latest;
             if (currentVersion.includes("-beta")) {
                 // Beta users track the @beta channel (never fall back to @latest, which is
@@ -5237,7 +5898,7 @@ When users create specialized instances, suggest these configurations:
                 // of beta/latest is the newest.
                 let beta = "";
                 try {
-                    beta = execSync("npm view @songsid/agend@beta version", { stdio: "pipe", timeout: 15_000 }).toString().trim();
+                    beta = await npmVersion("@songsid/agend@beta");
                 }
                 catch { /* no beta tag */ }
                 target = beta || latest;
@@ -5344,6 +6005,10 @@ When users create specialized instances, suggest these configurations:
                 // /view routes accept the read-only view.token (or web.token) and do
                 // their own per-method auth in view-api.ts — skip the web-token gate.
             }
+            else if (isUsagePath(new URL(req.url ?? "/", `http://localhost:${port}`).pathname)) {
+                // /api/ai-usage is read-only GET data for the /view Usage panel — open
+                // like the other /view data routes (usage-api.ts rejects non-GET).
+            }
             else {
                 // All other endpoints require a valid token (query ?token= or X-Agend-Token header).
                 // /ui/* will also re-check in web-api.ts, which is harmless.
@@ -5358,34 +6023,34 @@ When users create specialized instances, suggest these configurations:
                 }
             }
             if (req.method === "GET" && req.url === "/health") {
-                const instanceCount = this.fleetConfig?.instances
-                    ? Object.keys(this.fleetConfig.instances).length
-                    : 0;
-                res.writeHead(200);
-                res.end(JSON.stringify({
-                    status: "ok",
-                    instances: instanceCount,
-                    uptime: Math.floor((Date.now() - this.startedAt) / 1000),
-                }));
+                const health = this.getFleetHealth();
+                // 503 when the fleet cannot do its job, so an external monitor sees it.
+                // This used to always answer 200 "ok" with a count of CONFIGURED instances,
+                // so every agent could be dead and every adapter down and it still looked
+                // green.
+                res.writeHead(health.status === "ok" ? 200 : 503);
+                res.end(JSON.stringify(health));
                 return;
             }
             if (req.method === "GET" && req.url === "/status") {
                 const instances = Object.keys(this.fleetConfig?.instances ?? {}).map(name => {
                     const statusFile = join(this.getInstanceDir(name), "statusline.json");
-                    let context_pct = 0;
                     let cost = 0;
                     try {
                         const data = JSON.parse(readFileSync(statusFile, "utf-8"));
-                        context_pct = data.context_window?.used_percentage ?? 0;
                         cost = data.cost?.total_cost_usd ?? 0;
                     }
                     catch (err) {
                         this.logger.debug({ err, name }, "statusline.json read failed (/status)");
                     }
+                    const backend = this.fleetConfig?.instances[name]?.backend
+                        ?? this.fleetConfig?.defaults?.backend
+                        ?? "claude-code";
+                    const { context } = resolveInstanceContext(this.dataDir, name, backend);
                     return {
                         name,
                         status: this.getInstanceStatus(name),
-                        context_pct,
+                        context_pct: context ?? 0,
                         cost,
                     };
                 });
@@ -5500,7 +6165,10 @@ When users create specialized instances, suggest these configurations:
                         res.writeHead(500);
                         res.end(JSON.stringify({ error: `Start failed: ${err.message}` }));
                     }
-                })();
+                    // The inner catch can itself throw (writeHead after a successful
+                    // writeHead is ERR_HTTP_HEADERS_SENT), and that rejection escapes the
+                    // IIFE. Same for the two handlers below.
+                })().catch(err => this.logger.error({ err, name }, "HTTP start handler failed"));
                 return;
             }
             // Instance restart (immediate, no idle wait)
@@ -5521,7 +6189,7 @@ When users create specialized instances, suggest these configurations:
                         res.writeHead(status);
                         res.end(JSON.stringify({ error: `Restart failed: ${err.message}` }));
                     }
-                })();
+                })().catch(err => this.logger.error({ err, name }, "HTTP restart handler failed"));
                 return;
             }
             if (req.method === "POST" && req.url?.startsWith("/stop/")) {
@@ -5544,7 +6212,7 @@ When users create specialized instances, suggest these configurations:
                         res.writeHead(500);
                         res.end(JSON.stringify({ error: `Stop failed: ${err.message}` }));
                     }
-                })();
+                })().catch(err => this.logger.error({ err, name }, "HTTP stop handler failed"));
                 return;
             }
             // ── Agent CLI endpoint ─────
@@ -5556,6 +6224,8 @@ When users create specialized instances, suggest these configurations:
             const url = new URL(req.url ?? "/", `http://localhost:${port}`);
             if (handleViewRequest(req, res, url, this))
                 return;
+            if (handleUsageRequest(req, res, url, this))
+                return;
             if (handleSettingsRequest(req, res, url, this))
                 return;
             if (handleWebRequest(req, res, url, this))
@@ -5602,20 +6272,35 @@ When users create specialized instances, suggest these configurations:
         this.logger.info({ url: `http://localhost:${port}/view?token=${this.viewToken}` }, "Web View available");
     }
     getUiStatus() {
-        const instances = Object.keys(this.fleetConfig?.instances ?? {}).map(name => {
+        const fleetNames = Object.keys(this.fleetConfig?.instances ?? {});
+        // Classic rooms live only in classicBot.yaml — /api/profiles merges them into
+        // the View roster, but previously getUiStatus skipped them so context_pct was
+        // always 0 (live map miss → l?.context_pct ?? 0).
+        const classicOnly = (this.classicChannels?.getAll() ?? [])
+            .map(ch => ch.instanceName)
+            .filter(name => !fleetNames.includes(name));
+        const names = [...fleetNames, ...classicOnly];
+        const instances = names.map(name => {
             const statusFile = join(this.getInstanceDir(name), "statusline.json");
-            let context_pct = 0;
             let cost = 0;
             let model = "";
             try {
                 const data = JSON.parse(readFileSync(statusFile, "utf-8"));
-                context_pct = data.context_window?.used_percentage ?? 0;
                 cost = data.cost?.total_cost_usd ?? 0;
                 model = data.model?.display_name ?? "";
             }
             catch (err) {
                 this.logger.debug({ err, name }, "statusline.json read failed (getUiStatus)");
             }
+            // Align with /ctx: statusline for claude-code, pane scrape for kiro/grok/codex.
+            const classic = classicOnly.includes(name);
+            const backend = classic
+                ? this.classicChannels.getBackendByInstance(name, this.fleetConfig?.defaults?.backend)
+                : (this.fleetConfig?.instances[name]?.backend
+                    ?? this.fleetConfig?.defaults?.backend
+                    ?? "claude-code");
+            const { context } = resolveInstanceContext(this.dataDir, name, backend);
+            const context_pct = context ?? 0;
             return { name, status: this.getInstanceStatus(name), context_pct, cost, model };
         });
         return {