@songsid/agend 2.1.2-beta.4 → 2.1.2-beta.40

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (166) hide show
  1. package/dist/access-path.js +15 -6
  2. package/dist/access-path.js.map +1 -1
  3. package/dist/agent-cli-instructions.md +1 -0
  4. package/dist/agent-cli.js +14 -0
  5. package/dist/agent-cli.js.map +1 -1
  6. package/dist/agent-endpoint.d.ts +2 -0
  7. package/dist/agent-endpoint.js +25 -1
  8. package/dist/agent-endpoint.js.map +1 -1
  9. package/dist/backend/antigravity.d.ts +20 -0
  10. package/dist/backend/antigravity.js +51 -4
  11. package/dist/backend/antigravity.js.map +1 -1
  12. package/dist/backend/claude-code.d.ts +59 -0
  13. package/dist/backend/claude-code.js +67 -1
  14. package/dist/backend/claude-code.js.map +1 -1
  15. package/dist/backend/codex.d.ts +27 -0
  16. package/dist/backend/codex.js +93 -0
  17. package/dist/backend/codex.js.map +1 -1
  18. package/dist/backend/grok.d.ts +25 -0
  19. package/dist/backend/grok.js +51 -0
  20. package/dist/backend/grok.js.map +1 -1
  21. package/dist/backend/kiro.d.ts +22 -0
  22. package/dist/backend/kiro.js +79 -1
  23. package/dist/backend/kiro.js.map +1 -1
  24. package/dist/backend/types.d.ts +58 -0
  25. package/dist/backend/types.js +13 -1
  26. package/dist/backend/types.js.map +1 -1
  27. package/dist/channel/adapters/discord.d.ts +15 -0
  28. package/dist/channel/adapters/discord.js +100 -2
  29. package/dist/channel/adapters/discord.js.map +1 -1
  30. package/dist/channel/adapters/telegram.d.ts +21 -0
  31. package/dist/channel/adapters/telegram.js +86 -0
  32. package/dist/channel/adapters/telegram.js.map +1 -1
  33. package/dist/channel/ipc-bridge.d.ts +9 -1
  34. package/dist/channel/ipc-bridge.js +12 -3
  35. package/dist/channel/ipc-bridge.js.map +1 -1
  36. package/dist/channel/ipc-timeouts.d.ts +46 -0
  37. package/dist/channel/ipc-timeouts.js +65 -0
  38. package/dist/channel/ipc-timeouts.js.map +1 -0
  39. package/dist/channel/mcp-server.js +30 -14
  40. package/dist/channel/mcp-server.js.map +1 -1
  41. package/dist/channel/mcp-tools.js +30 -1
  42. package/dist/channel/mcp-tools.js.map +1 -1
  43. package/dist/channel/reconnect-backoff.d.ts +17 -0
  44. package/dist/channel/reconnect-backoff.js +21 -0
  45. package/dist/channel/reconnect-backoff.js.map +1 -0
  46. package/dist/channel/types.d.ts +29 -0
  47. package/dist/classic-channel-manager.js +1 -5
  48. package/dist/classic-channel-manager.js.map +1 -1
  49. package/dist/cli.js +204 -82
  50. package/dist/cli.js.map +1 -1
  51. package/dist/completion.d.ts +27 -0
  52. package/dist/completion.js +121 -0
  53. package/dist/completion.js.map +1 -0
  54. package/dist/config-validator.js +21 -0
  55. package/dist/config-validator.js.map +1 -1
  56. package/dist/config.js +1 -0
  57. package/dist/config.js.map +1 -1
  58. package/dist/cost-guard.d.ts +3 -1
  59. package/dist/cost-guard.js +3 -1
  60. package/dist/cost-guard.js.map +1 -1
  61. package/dist/daemon.d.ts +260 -14
  62. package/dist/daemon.js +991 -309
  63. package/dist/daemon.js.map +1 -1
  64. package/dist/event-log.d.ts +31 -0
  65. package/dist/event-log.js +96 -0
  66. package/dist/event-log.js.map +1 -1
  67. package/dist/fleet-context.d.ts +9 -0
  68. package/dist/fleet-lock.d.ts +28 -0
  69. package/dist/fleet-lock.js +130 -0
  70. package/dist/fleet-lock.js.map +1 -0
  71. package/dist/fleet-manager.d.ts +315 -2
  72. package/dist/fleet-manager.js +1410 -108
  73. package/dist/fleet-manager.js.map +1 -1
  74. package/dist/general-knowledge/skills/cross-instance-messaging/SKILL.md +22 -0
  75. package/dist/general-knowledge/skills/model-discovery/SKILL.md +14 -23
  76. package/dist/general-knowledge/skills/session-management/SKILL.md +15 -34
  77. package/dist/hang-detector.d.ts +19 -13
  78. package/dist/hang-detector.js +19 -49
  79. package/dist/hang-detector.js.map +1 -1
  80. package/dist/instance-lifecycle.d.ts +44 -3
  81. package/dist/instance-lifecycle.js +165 -48
  82. package/dist/instance-lifecycle.js.map +1 -1
  83. package/dist/instructions.d.ts +5 -0
  84. package/dist/instructions.js +9 -11
  85. package/dist/instructions.js.map +1 -1
  86. package/dist/locale.js +5 -1
  87. package/dist/locale.js.map +1 -1
  88. package/dist/logger.js +14 -0
  89. package/dist/logger.js.map +1 -1
  90. package/dist/mcp-liveness.d.ts +21 -0
  91. package/dist/mcp-liveness.js +27 -0
  92. package/dist/mcp-liveness.js.map +1 -0
  93. package/dist/outbound-handlers.d.ts +16 -0
  94. package/dist/outbound-handlers.js +151 -27
  95. package/dist/outbound-handlers.js.map +1 -1
  96. package/dist/outbound-schemas.d.ts +11 -3
  97. package/dist/outbound-schemas.js +16 -2
  98. package/dist/outbound-schemas.js.map +1 -1
  99. package/dist/pane-write-lock.d.ts +48 -0
  100. package/dist/pane-write-lock.js +73 -0
  101. package/dist/pane-write-lock.js.map +1 -0
  102. package/dist/process-memory.d.ts +31 -0
  103. package/dist/process-memory.js +79 -0
  104. package/dist/process-memory.js.map +1 -0
  105. package/dist/quickstart.js +17 -16
  106. package/dist/quickstart.js.map +1 -1
  107. package/dist/reply-dedup.d.ts +41 -0
  108. package/dist/reply-dedup.js +0 -0
  109. package/dist/reply-dedup.js.map +1 -0
  110. package/dist/scheduler/db.js +3 -0
  111. package/dist/scheduler/db.js.map +1 -1
  112. package/dist/sd-notify.d.ts +27 -0
  113. package/dist/sd-notify.js +33 -1
  114. package/dist/sd-notify.js.map +1 -1
  115. package/dist/secret-file.d.ts +33 -0
  116. package/dist/secret-file.js +36 -0
  117. package/dist/secret-file.js.map +1 -0
  118. package/dist/service-installer.d.ts +16 -0
  119. package/dist/service-installer.js +56 -1
  120. package/dist/service-installer.js.map +1 -1
  121. package/dist/setup-wizard.js +9 -7
  122. package/dist/setup-wizard.js.map +1 -1
  123. package/dist/tmux-control.d.ts +58 -5
  124. package/dist/tmux-control.js +102 -14
  125. package/dist/tmux-control.js.map +1 -1
  126. package/dist/tmux-manager.d.ts +31 -1
  127. package/dist/tmux-manager.js +40 -11
  128. package/dist/tmux-manager.js.map +1 -1
  129. package/dist/topic-commands.d.ts +49 -9
  130. package/dist/topic-commands.js +267 -105
  131. package/dist/topic-commands.js.map +1 -1
  132. package/dist/tui-glyphs.d.ts +5 -1
  133. package/dist/tui-glyphs.js +6 -2
  134. package/dist/tui-glyphs.js.map +1 -1
  135. package/dist/types.d.ts +12 -2
  136. package/dist/ui/view.html +123 -2
  137. package/dist/update-marker.d.ts +15 -0
  138. package/dist/update-marker.js +69 -0
  139. package/dist/update-marker.js.map +1 -0
  140. package/dist/usage/format-rich.d.ts +11 -0
  141. package/dist/usage/format-rich.js +121 -0
  142. package/dist/usage/format-rich.js.map +1 -0
  143. package/dist/usage/provider-alerts.d.ts +33 -0
  144. package/dist/usage/provider-alerts.js +57 -0
  145. package/dist/usage/provider-alerts.js.map +1 -0
  146. package/dist/usage/providers.d.ts +130 -0
  147. package/dist/usage/providers.js +1063 -0
  148. package/dist/usage/providers.js.map +1 -0
  149. package/dist/usage/statusline-usage.d.ts +20 -0
  150. package/dist/usage/statusline-usage.js +120 -0
  151. package/dist/usage/statusline-usage.js.map +1 -0
  152. package/dist/usage/usage-api.d.ts +50 -0
  153. package/dist/usage/usage-api.js +197 -0
  154. package/dist/usage/usage-api.js.map +1 -0
  155. package/dist/view-api.d.ts +3 -0
  156. package/dist/view-api.js +11 -2
  157. package/dist/view-api.js.map +1 -1
  158. package/dist/web-api.js +5 -2
  159. package/dist/web-api.js.map +1 -1
  160. package/package.json +4 -1
  161. package/dist/channel/tool-tracker.d.ts +0 -13
  162. package/dist/channel/tool-tracker.js +0 -58
  163. package/dist/channel/tool-tracker.js.map +0 -1
  164. package/dist/daemon-entry.d.ts +0 -1
  165. package/dist/daemon-entry.js +0 -30
  166. package/dist/daemon-entry.js.map +0 -1
@@ -4,7 +4,10 @@ import { createServer } from "node:http";
4
4
  import { join, dirname, basename } from "node:path";
5
5
  import { fileURLToPath } from "node:url";
6
6
  import { getAgendHome, ensureWorkspaceGit } from "./paths.js";
7
- import { sdNotify } from "./sd-notify.js";
7
+ import { clearUpdateMarker, isUpdateInProgress } from "./update-marker.js";
8
+ import { sdNotify, sdNotifyBlocking } from "./sd-notify.js";
9
+ import { readFleetMemory } from "./process-memory.js";
10
+ import { ReplyDeduper } from "./reply-dedup.js";
8
11
  import { isScalar, parseDocument } from "yaml";
9
12
  const __filename = fileURLToPath(import.meta.url);
10
13
  const __dirname = dirname(__filename);
@@ -21,12 +24,12 @@ import { IpcClient } from "./channel/ipc-bridge.js";
21
24
  import { createAdapter } from "./channel/factory.js";
22
25
  import { createBackend } from "./backend/factory.js";
23
26
  import { isModelCompatible } from "./backend/types.js";
24
- import { createLogger } from "./logger.js";
27
+ import { createLogger, rotateLogIfNeeded } from "./logger.js";
25
28
  import { processAttachments } from "./channel/attachment-handler.js";
26
29
  import { routeToolCall } from "./channel/tool-router.js";
27
30
  import { Scheduler } from "./scheduler/index.js";
28
31
  import { DEFAULT_SCHEDULER_CONFIG } from "./scheduler/index.js";
29
- import { TopicCommands, saveCommandForBackend, parseSaveFilename, parsePauseWakeCommand, SAVE_FILENAME_RE, SAVE_UNSUPPORTED_MSG } from "./topic-commands.js";
32
+ import { TopicCommands, saveCommandForBackend, parseSaveFilename, parsePauseWakeCommand, SAVE_FILENAME_RE, SAVE_UNSUPPORTED_MSG, resolveInstanceContext, forgetInstanceContext } from "./topic-commands.js";
30
33
  import { DailySummary } from "./daily-summary.js";
31
34
  import { WebhookEmitter } from "./webhook-emitter.js";
32
35
  import { TmuxControlClient } from "./tmux-control.js";
@@ -38,6 +41,7 @@ import { StatuslineWatcher } from "./statusline-watcher.js";
38
41
  import { outboundHandlers } from "./outbound-handlers.js";
39
42
  import { handleWebRequest, broadcastSseEvent } from "./web-api.js";
40
43
  import { handleViewRequest, isViewPath } from "./view-api.js";
44
+ import { handleUsageRequest, isUsagePath } from "./usage/usage-api.js";
41
45
  import { handleSettingsRequest } from "./settings-api.js";
42
46
  import { setLocale, detectLocale, t } from "./locale.js";
43
47
  import { handleAgentRequest } from "./agent-endpoint.js";
@@ -45,6 +49,7 @@ import { ClassicChannelManager, getClassicBackendChoices, isSelectableClassicBac
45
49
  import { validateFleetConfig } from "./config-validator.js";
46
50
  import { readLastInboundAt } from "./daemon.js";
47
51
  import { clearPausedMarker } from "./pause-marker.js";
52
+ import { releaseProcessFleetLock } from "./fleet-lock.js";
48
53
  import { getTmuxSession } from "./config.js";
49
54
  export function resolveReplyThreadId(argsThreadId, instanceConfig) {
50
55
  if (typeof argsThreadId === "string" && argsThreadId.length > 0) {
@@ -83,9 +88,80 @@ const CANCEL_BTN_MAX_RETRIES = 3;
83
88
  * buttons no clear trigger reached (e.g. a scheduled/HTTP turn that never called
84
89
  * reply). 5min (not the old 2s idle-watch) so Thinking isn't misread as idle. */
85
90
  const CANCEL_BTN_IDLE_CHECK_INTERVAL_MS = 5 * 60_000;
91
+ /**
92
+ * A queued turn can produce a very short idle edge while the CLI hands off to
93
+ * the next message. Do not retire the cancel button until that edge remains
94
+ * idle for this long; a working/stuck report during the grace cancels it.
95
+ */
96
+ const CANCEL_BTN_IDLE_RETIRE_GRACE_MS = 2_000;
97
+ /**
98
+ * How long after a reply an instance gets to resume working before its cancel
99
+ * button is retired. A short turn ends with a reply and never works again → the
100
+ * button disappears ~2 minutes after the answer. A multi-step run replies
101
+ * mid-flight and keeps going → the grace check sees "working" and leaves the
102
+ * button alone (the idle edge retires it when the run really ends).
103
+ */
104
+ const REPLY_RETIRE_GRACE_MS = 2 * 60_000;
105
+ /**
106
+ * The daemon only broadcasts execution state on TRANSITIONS, so a long
107
+ * single-state run sends nothing for hours. The idle backstop therefore pokes a
108
+ * query each tick; a live daemon answers within milliseconds and refreshes the
109
+ * cache. When nothing has refreshed it for this long despite those pokes, the
110
+ * reporting chain (daemon, IPC, or state monitor) is dead and a "working" state
111
+ * from 30 minutes ago proves nothing — the button may be retired.
112
+ */
113
+ const STATE_REPORT_STALE_MS = 30 * 60_000;
114
+ /**
115
+ * Unconditional ceiling on a cancel button's life. Deliberately far beyond any
116
+ * legitimate run (multi-hour tasks are normal on this fleet): everything below
117
+ * this is decided by real state; a button that somehow survives a full day is
118
+ * wreckage, stuck or not.
119
+ */
120
+ const CANCEL_BTN_MAX_LIFETIME_MS = 24 * 60 * 60_000;
121
+ /** A click on a button the fleet no longer tracks may fire at most this often. */
122
+ const STALE_CANCEL_CLICK_COOLDOWN_MS = 10_000;
123
+ /** Orphaned-button ledger, swept at startup. Lives in the fleet data dir. */
124
+ const CANCEL_BTN_LEDGER_FILE = "cancel-buttons.json";
125
+ /**
126
+ * How often the cancel button's text is refreshed with elapsed working time.
127
+ *
128
+ * One edit per working instance per interval — at 60s that is trivial for both
129
+ * platforms' rate limits, and it reads as a live counter rather than a stale
130
+ * snapshot. Nothing new is posted, so the channel is never spammed: there is
131
+ * exactly one progress message per turn, and it is the cancel button itself.
132
+ */
133
+ const PROGRESS_UPDATE_INTERVAL_MS = 60_000;
134
+ /** Elapsed time is only shown once work has clearly outlasted a quick answer. */
135
+ /**
136
+ * Default delay before the button starts showing elapsed time. Configurable via
137
+ * `defaults.progress_min_elapsed` (seconds) in fleet.yaml. 30s is the balance
138
+ * point: most quick answers finish inside it (no churn for ordinary turns),
139
+ * while anything real shows signs of life well before the old two minutes.
140
+ */
141
+ const PROGRESS_MIN_ELAPSED_MS = 30_000;
142
+ /** How much of a tool summary the progress line will show before eliding. */
143
+ const PROGRESS_ACTIVITY_MAX_CHARS = 48;
144
+ /**
145
+ * Emoji AgEnD itself stamps on messages as the delivery-status ladder
146
+ * (⏳ queued, 👀 delivered, ✅ confirmed, ❌ failed). These are machine
147
+ * indicators, not opinions, so they never enter the reactions queue — from
148
+ * anyone. This exact-emoji filter is the ONLY bot filtering left: bot-to-bot
149
+ * reactions are otherwise delivered on purpose (agents signal each other), and
150
+ * 🫡 passes too — it reads as a deliberate acknowledgement, not plumbing.
151
+ */
152
+ const DELIVERY_STATUS_EMOJIS = new Set(["👀", "⏳", "✅", "❌"]);
153
+ /**
154
+ * How long a delivery waits out a disconnected instance IPC before giving up.
155
+ *
156
+ * Sized for a daemon restart (socket close → respawn → CLI ready), which is the
157
+ * event this exists for. Past it the delivery fails loudly as it always did.
158
+ */
159
+ const IPC_RECONNECT_GRACE_MS = 30_000;
160
+ const IPC_RECONNECT_POLL_MS = 250;
86
161
  const CLASSIC_BACKEND_SELECTION_TIMEOUT_MS = 60_000;
87
162
  const CLASSIC_BACKEND_CALLBACK_PREFIX = "classic-backend:";
88
163
  const MODEL_SELECT_CALLBACK_PREFIX = "model-select:";
164
+ const EFFORT_SELECT_CALLBACK_PREFIX = "effort-select:";
89
165
  const CLI_ENV_TTL_MS = 24 * 60 * 60 * 1000; // /model reads cached CLI env within 24h
90
166
  export class FleetManager {
91
167
  dataDir;
@@ -93,7 +169,7 @@ export class FleetManager {
93
169
  static sighupHandlerInstalled = false;
94
170
  children = new Map();
95
171
  lifecycle;
96
- /** @deprecated Use lifecycle.daemons — kept for backward compat */
172
+ /** Live view of lifecycle.daemons — used throughout; not deprecated. */
97
173
  get daemons() { return this.lifecycle.daemons; }
98
174
  fleetConfig = null;
99
175
  rawFleetConfig = {};
@@ -151,6 +227,14 @@ export class FleetManager {
151
227
  // reply, on cancel, or when a newer button supersedes it for the same
152
228
  // instance. Per-button tracking means a failed delete never strands a button.
153
229
  cancelButtons = new Map();
230
+ /** Pending idle-edge retirement, one timer per instance. */
231
+ cancelButtonIdleRetireTimers = new Map();
232
+ /** Duplicate-reply suppression across both the MCP and HTTP reply paths. */
233
+ replyDeduper = new ReplyDeduper();
234
+ /** instanceName → what it is doing right now, when the backend can tell us. */
235
+ instanceActivity = new Map();
236
+ /** instanceName → tail of deliveries waiting for its IPC to come back. */
237
+ ipcWaitTails = new Map();
154
238
  // Last user message delivered to each instance — used to react ✅ on completion.
155
239
  lastInboundMsg = new Map();
156
240
  topicArchiver;
@@ -158,11 +242,17 @@ export class FleetManager {
158
242
  classicChannels = null;
159
243
  pendingClassicStarts = new Map();
160
244
  /** In-flight /model selections, keyed by nonce (see handleModelSelection). */
245
+ /** In-flight /effort selections, same coordinator shape as pendingModelSelects. */
246
+ pendingEffortSelects = new Map();
161
247
  pendingModelSelects = new Map();
162
248
  // Model failover state
163
249
  failoverActive = new Map(); // instance → current failover model
164
250
  // IPC reconnect: tracks instances being intentionally stopped (skip reconnect)
251
+ /** instance → when a click with no live button entry last fired a cancel. */
252
+ staleCancelClickAt = new Map();
165
253
  ipcStoppingInstances = new Set();
254
+ /** Set the moment a graceful stop begins — see isPlannedRestart(). */
255
+ shuttingDown = false;
166
256
  /** Coalesce concurrent connection attempts for the same daemon socket. */
167
257
  ipcConnectInFlight = new Map();
168
258
  /** At most one reconnect/backoff loop may exist per instance. */
@@ -176,6 +266,10 @@ export class FleetManager {
176
266
  healthServer = null;
177
267
  healthPortRetried = false;
178
268
  updateCheckTimer = null;
269
+ eventLogPruneTimer = null;
270
+ logRotateTimer = null;
271
+ /** Days of event/activity history to keep. */
272
+ static EVENT_LOG_RETENTION_DAYS = 30;
179
273
  watchdogTimer = null;
180
274
  startedAt = 0;
181
275
  // Mirror topic: buffer cross-instance messages, flush every 3s
@@ -214,7 +308,13 @@ export class FleetManager {
214
308
  }
215
309
  this.reloadPending = false;
216
310
  this.reconcileInFlight = this.reconcileInstances()
217
- .catch(err => this.logger.error({ err }, "SIGHUP config reload failed"))
311
+ .catch(err => {
312
+ // Almost always a YAML parse error. Log-only meant the user edited
313
+ // fleet.yaml, sent SIGHUP, and got no reaction and no explanation.
314
+ this.logger.error({ err }, "SIGHUP config reload failed");
315
+ const message = err instanceof Error ? err.message : String(err);
316
+ this.notifyFleetError(`⚠️ fleet.yaml reload FAILED — ${message}\nThe previous configuration is still running.`);
317
+ })
218
318
  .finally(() => {
219
319
  this.reconcileInFlight = null;
220
320
  if (this.reloadPending && this.startupComplete) {
@@ -222,16 +322,55 @@ export class FleetManager {
222
322
  }
223
323
  });
224
324
  }
325
+ /**
326
+ * Is the fleet going down (or coming back up) on purpose?
327
+ *
328
+ * Instances dying during a planned restart is the restart working, not an
329
+ * incident — but the code that notices a dead pane or a dead MCP server
330
+ * cannot tell the difference on its own. Two sources, because the noise
331
+ * starts before this process is even told to stop: `agend update` replaces
332
+ * the package on disk while this daemon is still running and still watching.
333
+ */
334
+ isPlannedRestart() {
335
+ return this.shuttingDown || isUpdateInProgress(this.dataDir);
336
+ }
225
337
  finishStartup() {
226
338
  this.startupComplete = true;
339
+ // We are the post-update fleet: the update is over by definition. Clearing
340
+ // it here (rather than in the update command, which exits before the new
341
+ // fleet is up) is what keeps the quiet window from outliving the restart.
342
+ clearUpdateMarker(this.dataDir);
227
343
  if (this.reloadPending)
228
344
  this.scheduleReconcile();
345
+ void this.sweepOrphanedCancelButtons();
229
346
  }
230
347
  // ── ArchiverContext bridge ────────────────────────────────────────────
231
348
  lastActivityMs(name) {
232
349
  return this.lastActivity.get(name) ?? 0;
233
350
  }
351
+ /**
352
+ * Is the instance between turns?
353
+ *
354
+ * Prefers the daemon's pane state machine (debounced, busy-pattern aware) over
355
+ * the control client's raw 2-second output-silence heuristic. The raw heuristic
356
+ * reads every >2s output lull as idle — and long silent tools (a build, a test
357
+ * run) or an LLM pause produce those constantly mid-turn. That misreading is
358
+ * what retired cancel buttons in the middle of long work (the 5-minute backstop
359
+ * fired during a lull) and froze their progress text (ticker skipped "idle"
360
+ * ticks). The silence heuristic remains only as the fallback for instances
361
+ * whose daemon has not reported a state yet.
362
+ */
234
363
  getInstanceIdle(name) {
364
+ // A daemon that is not running cannot be mid-turn. This is what a stale
365
+ // "working" cache after a hard daemon kill (SIGKILL/OOM — no IPC crash
366
+ // report ever arrives) must not override.
367
+ if (this.getInstanceStatus(name) !== "running")
368
+ return true;
369
+ const state = this.getInstanceExecutionState(name);
370
+ if (state === "working" || state === "stuck")
371
+ return false;
372
+ if (state === "idle")
373
+ return true;
235
374
  try {
236
375
  const widFile = join(this.getInstanceDir(name), "window-id");
237
376
  if (!existsSync(widFile))
@@ -243,6 +382,18 @@ export class FleetManager {
243
382
  return true;
244
383
  }
245
384
  }
385
+ /**
386
+ * True when the instance claims working/stuck but nothing has refreshed that
387
+ * claim for STATE_REPORT_STALE_MS despite the backstop's per-tick queries.
388
+ * Measures the CACHE's age, not the button's — a healthy multi-hour run
389
+ * answers every query and never trips this.
390
+ */
391
+ stateReportDead(name) {
392
+ const cached = this.instanceStateCache.get(name);
393
+ if (!cached)
394
+ return false; // no claim to distrust — getInstanceIdle owns this case
395
+ return Date.now() - cached.receivedAt > STATE_REPORT_STALE_MS;
396
+ }
246
397
  // ── LifecycleContext bridge methods ──────────────────────────────────────
247
398
  webhookEmit(event, name, data) {
248
399
  this.webhookEmitter?.emit(event, name, data);
@@ -471,14 +622,45 @@ export class FleetManager {
471
622
  unchangedForMs: numberOr(msg.unchangedForMs, previous?.unchangedForMs ?? 0),
472
623
  observedAt: numberOr(msg.observedAt, now),
473
624
  stateChangedAt: numberOr(msg.stateChangedAt, previous?.state === state ? previous.stateChangedAt : now),
625
+ // Fleet-manager receipt time, NOT the daemon's observation time: staleness
626
+ // asks "is anyone still reporting", which only the receiver can date.
627
+ receivedAt: now,
474
628
  });
475
629
  for (const check of this.instanceIdleWaiters.get(name) ?? [])
476
630
  check();
477
631
  // warm_cap: a fresh transition into idle may free this instance for eviction,
478
632
  // or (more usefully) reveal that the fleet is now over cap. Only fire on the
479
633
  // edge into idle, not on every idle heartbeat.
480
- if (state === "idle" && previous?.state !== "idle")
634
+ if (state === "idle" && previous?.state !== "idle") {
481
635
  this.enforceWarmCap();
636
+ // A queued message may turn this edge back into working almost
637
+ // immediately. Give that handoff a short grace before retiring the button.
638
+ this.scheduleIdleButtonRetirement(name);
639
+ }
640
+ else if (state !== "idle") {
641
+ this.cancelIdleButtonRetirement(name);
642
+ }
643
+ }
644
+ cancelIdleButtonRetirement(name) {
645
+ const timer = this.cancelButtonIdleRetireTimers.get(name);
646
+ if (!timer)
647
+ return;
648
+ clearTimeout(timer);
649
+ this.cancelButtonIdleRetireTimers.delete(name);
650
+ }
651
+ scheduleIdleButtonRetirement(name) {
652
+ this.cancelIdleButtonRetirement(name);
653
+ const timer = setTimeout(() => {
654
+ // Ignore a superseded timer even if it was already queued to run.
655
+ if (this.cancelButtonIdleRetireTimers.get(name) !== timer)
656
+ return;
657
+ this.cancelButtonIdleRetireTimers.delete(name);
658
+ if (this.getInstanceExecutionState(name) === "idle") {
659
+ this.retireInstanceButtons(name);
660
+ }
661
+ }, CANCEL_BTN_IDLE_RETIRE_GRACE_MS);
662
+ timer.unref?.();
663
+ this.cancelButtonIdleRetireTimers.set(name, timer);
482
664
  }
483
665
  cacheInstanceProcessStatus(name, status) {
484
666
  if (status === "running") {
@@ -487,6 +669,7 @@ export class FleetManager {
487
669
  }
488
670
  if (status !== "crashed" && status !== "stopped")
489
671
  return;
672
+ this.cancelIdleButtonRetirement(name);
490
673
  this.instanceProcessStatus.set(name, status);
491
674
  // Never display the last ready prompt as current execution state after its
492
675
  // owning CLI process has exited.
@@ -583,12 +766,70 @@ export class FleetManager {
583
766
  if (!idle) {
584
767
  this.logger.warn({ instanceName, timeoutMs }, "Idle gate timed out; forcing delivery");
585
768
  }
586
- const ipc = this.instanceIpcClients.get(instanceName);
587
- if (!ipc?.connected)
588
- throw new Error(`Instance '${instanceName}' IPC is unavailable`);
589
- ipc.send(payload);
769
+ await this.sendWhenConnected(instanceName, payload);
590
770
  this.lastDeliveryAt.set(instanceName, Date.now());
591
771
  }
772
+ /**
773
+ * Hand a payload to an instance's IPC, waiting out a *transient* disconnect.
774
+ *
775
+ * A daemon that is restarting — `/restart`, crash recovery, a model switch —
776
+ * drops its socket for a few seconds. Any message arriving in that window used
777
+ * to fail instantly: the caller logged a warning, put ❌ on the user's message,
778
+ * and the message was gone. The user had to notice the ❌ and retype it. That is
779
+ * the "instance 訊息不容易掉" goal failing on the most predictable event there is.
780
+ *
781
+ * The wait is bounded. If the instance is genuinely down, this still throws and
782
+ * the ❌ still appears — just for a real failure rather than a restart.
783
+ *
784
+ * Ordering is preserved by serialising behind any waiter already queued for this
785
+ * instance, *including* when the socket happens to be up: otherwise a message
786
+ * arriving after the reconnect could overtake one that has been waiting for it.
787
+ */
788
+ async sendWhenConnected(instanceName, payload) {
789
+ const queued = this.ipcWaitTails.get(instanceName);
790
+ if (!queued) {
791
+ const ipc = this.instanceIpcClients.get(instanceName);
792
+ if (ipc?.connected && ipc.send(payload))
793
+ return;
794
+ }
795
+ const attempt = (queued ?? Promise.resolve())
796
+ .catch(() => { })
797
+ .then(() => this.sendAfterIpcReturns(instanceName, payload));
798
+ // The chain stores a settled-either-way promise so one failed delivery cannot
799
+ // wedge every later one, and so `queued` above is safe to await unguarded.
800
+ const tail = attempt.catch(() => { });
801
+ this.ipcWaitTails.set(instanceName, tail);
802
+ try {
803
+ await attempt;
804
+ }
805
+ finally {
806
+ // Only the last waiter clears the chain; while a queue is still draining the
807
+ // map must keep pointing at it or ordering is lost.
808
+ if (this.ipcWaitTails.get(instanceName) === tail) {
809
+ this.ipcWaitTails.delete(instanceName);
810
+ }
811
+ }
812
+ }
813
+ /** Poll for the instance's IPC to come back, then send. Throws if it does not. */
814
+ async sendAfterIpcReturns(instanceName, payload) {
815
+ const deadline = Date.now() + IPC_RECONNECT_GRACE_MS;
816
+ let warned = false;
817
+ for (;;) {
818
+ // Re-read every round: a reconnect replaces the IpcClient object entirely,
819
+ // so a cached reference would stay dead forever.
820
+ const ipc = this.instanceIpcClients.get(instanceName);
821
+ if (ipc?.connected && ipc.send(payload))
822
+ return;
823
+ if (Date.now() >= deadline) {
824
+ throw new Error(`Instance '${instanceName}' IPC is unavailable`);
825
+ }
826
+ if (!warned) {
827
+ warned = true;
828
+ this.logger.info({ instanceName }, "Instance IPC is down — holding delivery until it reconnects");
829
+ }
830
+ await new Promise(resolve => setTimeout(resolve, IPC_RECONNECT_POLL_MS));
831
+ }
832
+ }
592
833
  /** Single delivery facade: wake paused CLIs and serialize non-user work behind idle. */
593
834
  async deliverToInstance(instanceName, payload, options = {}) {
594
835
  const meta = payload.meta && typeof payload.meta === "object"
@@ -604,10 +845,7 @@ export class FleetManager {
604
845
  await this.lifecycle.wake(instanceName, 30_000);
605
846
  this.enforceWarmCap(instanceName); // woke one → evict a different LRU idle if over cap
606
847
  }
607
- const ipc = this.instanceIpcClients.get(instanceName);
608
- if (!ipc?.connected)
609
- throw new Error(`Instance '${instanceName}' IPC is unavailable`);
610
- ipc.send(payload);
848
+ await this.sendWhenConnected(instanceName, payload);
611
849
  // A cross-instance item arriving before the daemon observes this turn as
612
850
  // working must not trust the stale idle snapshot from before the send.
613
851
  this.lastDeliveryAt.set(instanceName, Date.now());
@@ -655,7 +893,7 @@ export class FleetManager {
655
893
  await new Promise(resolve => setTimeout(resolve, 250));
656
894
  await this.startClassicInstance(instanceName, this.classicChannels.getBackendByInstance(instanceName, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(channel.channelId, channel.adapterId), this.classicChannels.getModel(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
657
895
  }
658
- async startInstance(name, config, topicMode) {
896
+ async startInstance(name, config, topicMode, kind = "fleet-topic") {
659
897
  if (this.lifecycle.isPaused(name)) {
660
898
  this.logger.info({ name }, "Persisted paused instance — skipping startup");
661
899
  return;
@@ -671,7 +909,11 @@ export class FleetManager {
671
909
  this.ensureGeneralInstructions(config.working_directory, config.backend);
672
910
  }
673
911
  this.instanceProcessStatus.delete(name);
674
- await this.lifecycle.start(name, config, topicMode);
912
+ await this.lifecycle.start(name, config, topicMode, {
913
+ kind,
914
+ backend: config.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code",
915
+ model: this.resolveInstanceModel(name).display,
916
+ });
675
917
  // Auto-connect IPC — daemon.start() ensures socket is ready before resolving
676
918
  await this.connectIpcToInstance(name);
677
919
  }
@@ -772,6 +1014,7 @@ export class FleetManager {
772
1014
  }
773
1015
  async stopInstance(name) {
774
1016
  this.failoverActive.delete(name);
1017
+ this.cancelIdleButtonRetirement(name);
775
1018
  this.instanceStateCache.delete(name);
776
1019
  this.instanceProcessStatus.delete(name);
777
1020
  this.lastDeliveryAt.delete(name);
@@ -834,7 +1077,10 @@ export class FleetManager {
834
1077
  const eqIdx = trimmed.indexOf("=");
835
1078
  if (eqIdx < 0)
836
1079
  continue;
837
- const key = trimmed.slice(0, eqIdx);
1080
+ // Accept `export KEY=value` the shell-style form people paste from their
1081
+ // .bashrc. Without this the variable landed in process.env under the key
1082
+ // "export KEY" and silently did nothing.
1083
+ const key = trimmed.slice(0, eqIdx).replace(/^export\s+/, "").trim();
838
1084
  const raw = trimmed.slice(eqIdx + 1);
839
1085
  const value = raw.replace(/^["'](.*)["']$/, '$1');
840
1086
  // .env file always wins over inherited shell env vars, so that
@@ -846,10 +1092,12 @@ export class FleetManager {
846
1092
  async startAll(configPath) {
847
1093
  FleetManager.signalTarget = this;
848
1094
  this.startupComplete = false;
1095
+ // Cleared here, not at the end of doStopAll: a stop has an async tail, and
1096
+ // anything arriving during it is still part of the stop.
1097
+ this.shuttingDown = false;
849
1098
  this.configPath = configPath;
850
1099
  this.loadEnvFile();
851
1100
  // Rotate fleet.log if oversized (before any logging)
852
- const { rotateLogIfNeeded } = await import("./logger.js");
853
1101
  rotateLogIfNeeded(join(this.dataDir, "fleet.log"));
854
1102
  const fleet = this.loadConfig(configPath);
855
1103
  setLocale(detectLocale(fleet)); // user-facing text language (fleet.yaml defaults.locale / timezone)
@@ -893,7 +1141,7 @@ export class FleetManager {
893
1141
  }
894
1142
  const pidPath = join(this.dataDir, "fleet.pid");
895
1143
  writeFileSync(pidPath, String(process.pid), "utf-8");
896
- this.eventLog = new EventLog(join(this.dataDir, "events.db"));
1144
+ this.eventLog = this.openEventLog();
897
1145
  // Initialize classic channel manager. The primary adapter (channels[0])
898
1146
  // migrates legacy single-bot entries and names without a suffix. Classic
899
1147
  // routing does NOT go through the routing engine (single-key, can't hold two
@@ -982,6 +1230,10 @@ export class FleetManager {
982
1230
  // Rotate fleet.log daily too (besides the startup size check above), so a
983
1231
  // long-running fleet doesn't accumulate an unbounded log.
984
1232
  rotateLogIfNeeded(join(this.dataDir, "fleet.log"));
1233
+ // Instance output.log is pipe-pane (TUI ANSI). Daemon health ticks rotate a
1234
+ // running instance's own log; this sweep is the safety net for every other
1235
+ // kind. One implementation, so the two cannot cover different sets.
1236
+ this.rotateAllInstanceLogs();
985
1237
  }, () => {
986
1238
  const instances = Object.keys(this.fleetConfig?.instances ?? {});
987
1239
  const costMap = new Map();
@@ -1121,9 +1373,29 @@ export class FleetManager {
1121
1373
  }
1122
1374
  }
1123
1375
  }
1124
- // Signal systemd: generals ready
1125
- sdNotify("READY=1");
1376
+ // The systemd watchdog answers exactly one question: is this process still
1377
+ // turning its event loop? Pinging from a timer proves that, and after the
1378
+ // blocking child-process calls were made async it is a meaningful signal —
1379
+ // a deadlocked or frozen fleet stops pinging and systemd restarts it.
1380
+ //
1381
+ // It deliberately does NOT gate on fleet health. "No adapter connected" or
1382
+ // "an instance crashed" must not kill the process: the fleet would be restarted
1383
+ // into the same broken state, and a user who has legitimately stopped every
1384
+ // instance would get a restart loop. Those conditions surface through /health
1385
+ // (which now returns 503) and through the General-topic notifications instead.
1126
1386
  this.watchdogTimer = setInterval(() => sdNotify("WATCHDOG=1"), 30_000);
1387
+ // EventLog.prune() existed but was never called, so `events` and `activity`
1388
+ // grew without bound for the life of the install. Prune once at startup and
1389
+ // daily after that; the timer is unref'd so it never holds the loop open.
1390
+ this.pruneEventLog();
1391
+ this.eventLogPruneTimer = setInterval(() => this.pruneEventLog(), 24 * 60 * 60_000);
1392
+ this.eventLogPruneTimer.unref?.();
1393
+ // Same shape for pipe-pane logs, and for the same reason: the only sweep that
1394
+ // covered them lived inside the daily-summary callback, so it did not run at
1395
+ // all when summaries were off.
1396
+ this.rotateAllInstanceLogs();
1397
+ this.logRotateTimer = setInterval(() => this.rotateAllInstanceLogs(), 24 * 60 * 60_000);
1398
+ this.logRotateTimer.unref?.();
1127
1399
  // Phase 2: Start remaining instances with staggered concurrency
1128
1400
  if (others.length > 0) {
1129
1401
  await this.startInstancesWithConcurrency(others, topicMode);
@@ -1259,6 +1531,15 @@ export class FleetManager {
1259
1531
  // rest of startup finishes. Replay one coalesced reload only after all
1260
1532
  // startup-owned lifecycle work and signal handlers are in place.
1261
1533
  this.finishStartup();
1534
+ // Tell systemd we are ready only now. This used to fire right after the
1535
+ // generals started — before adapters, classic instances, topic creation and the
1536
+ // health server — so `systemctl start` returned success while the fleet was
1537
+ // still deaf: no path existed for a user message to arrive.
1538
+ sdNotify("READY=1");
1539
+ const health = this.getFleetHealth();
1540
+ if (health.status !== "ok") {
1541
+ this.logger.warn({ health }, "Fleet started with problems — see /health");
1542
+ }
1262
1543
  }
1263
1544
  /**
1264
1545
  * Delete inbox files older than retentionDays (by mtime). Cleans the shared
@@ -1375,6 +1656,67 @@ export class FleetManager {
1375
1656
  getAdapterStates() {
1376
1657
  return this.adapterState;
1377
1658
  }
1659
+ /**
1660
+ * Real, checkable fleet health for `/health` and the operator.
1661
+ *
1662
+ * `status` is:
1663
+ * - `ok` — at least one adapter connected and every configured instance
1664
+ * that should be running is running
1665
+ * - `degraded` — reachable, but something the operator should look at (an
1666
+ * adapter retrying, an instance crashed or stopped)
1667
+ * - `down` — the fleet cannot do its job: no adapter is connected, so no
1668
+ * message can arrive or be answered
1669
+ *
1670
+ * Deliberately does NOT gate the systemd watchdog — see the comment at the
1671
+ * WATCHDOG timer for why.
1672
+ */
1673
+ getFleetHealth() {
1674
+ const names = Object.keys(this.fleetConfig?.instances ?? {});
1675
+ const counts = { configured: names.length, running: 0, crashed: 0, paused: 0, stopped: 0 };
1676
+ for (const name of names) {
1677
+ const state = this.getInstanceStatus(name);
1678
+ if (state === "running")
1679
+ counts.running++;
1680
+ else if (state === "crashed")
1681
+ counts.crashed++;
1682
+ else if (state === "paused")
1683
+ counts.paused++;
1684
+ else
1685
+ counts.stopped++;
1686
+ }
1687
+ const states = {};
1688
+ let connected = 0;
1689
+ for (const [id, state] of this.adapterState) {
1690
+ states[id] = state.status;
1691
+ if (state.status === "connected")
1692
+ connected++;
1693
+ }
1694
+ const problems = [];
1695
+ if (this.adapterState.size > 0 && connected === 0)
1696
+ problems.push("no channel adapter is connected");
1697
+ if (counts.crashed > 0)
1698
+ problems.push(`${counts.crashed} instance(s) crashed`);
1699
+ for (const [id, state] of this.adapterState) {
1700
+ if (state.status !== "connected")
1701
+ problems.push(`adapter ${id} is ${state.status}`);
1702
+ }
1703
+ if (!this.startupComplete)
1704
+ problems.push("startup has not completed");
1705
+ // "down" is reserved for "cannot receive or answer a message at all". A fleet
1706
+ // with adapters configured but none connected is exactly that.
1707
+ const status = this.adapterState.size > 0 && connected === 0
1708
+ ? "down"
1709
+ : problems.length > 0 ? "degraded" : "ok";
1710
+ return {
1711
+ status,
1712
+ uptime: Math.floor((Date.now() - this.startedAt) / 1000),
1713
+ instances: counts,
1714
+ adapters: { total: this.adapterState.size, connected, states },
1715
+ startupComplete: this.startupComplete,
1716
+ memory: readFleetMemory(),
1717
+ problems,
1718
+ };
1719
+ }
1378
1720
  /** Start the primary adapter (backward-compatible, sets this.adapter) */
1379
1721
  async startSingleAdapter(fleet, channelConfig) {
1380
1722
  const botToken = process.env[channelConfig.bot_token_env];
@@ -1401,11 +1743,16 @@ export class FleetManager {
1401
1743
  this.adapter.on("message", safeHandler(async (msg) => {
1402
1744
  await this.handleInboundMessage(msg);
1403
1745
  }, this.logger, "adapter.message"));
1746
+ this.adapter.on("reaction", safeHandler(async (r) => {
1747
+ await this.handleInboundReaction(r);
1748
+ }, this.logger, "adapter.reaction"));
1404
1749
  this.adapter.on("callback_query", safeHandler(async (data) => {
1405
1750
  if (await this.handleClassicBackendSelection(data))
1406
1751
  return;
1407
1752
  if (await this.handleModelSelection(data))
1408
1753
  return;
1754
+ if (await this.handleEffortSelection(data))
1755
+ return;
1409
1756
  if (data.callbackData.startsWith("hang:")) {
1410
1757
  const parts = data.callbackData.split(":");
1411
1758
  const action = parts[1];
@@ -1426,12 +1773,7 @@ export class FleetManager {
1426
1773
  return;
1427
1774
  }
1428
1775
  if (data.callbackData.startsWith("cancel:")) {
1429
- const instanceName = data.callbackData.slice("cancel:".length);
1430
- // Idempotent: a button click only acts while the button is live. A
1431
- // second click (entry already cleared) is a no-op — don't re-send the
1432
- // interrupt key. (The /cancel command path calls cancelInstance directly.)
1433
- if (this.hasCancelButton(instanceName))
1434
- this.cancelInstance(instanceName);
1776
+ this.handleCancelClick(data.callbackData.slice("cancel:".length), this.adapter, data);
1435
1777
  return;
1436
1778
  }
1437
1779
  }, this.logger, "adapter.callback_query"));
@@ -1511,6 +1853,9 @@ export class FleetManager {
1511
1853
  else if (data.command === "model") {
1512
1854
  await this.handleModelSlash(data, adapterId);
1513
1855
  }
1856
+ else if (data.command === "effort") {
1857
+ await this.handleEffortSlash(data, adapterId);
1858
+ }
1514
1859
  else if (data.command === "cancel") {
1515
1860
  const name = this.resolveSlashTarget(data.channelId, adapterId);
1516
1861
  if (!name) {
@@ -1575,19 +1920,29 @@ export class FleetManager {
1575
1920
  await data.respond(t("not_authorized"));
1576
1921
  return;
1577
1922
  }
1923
+ await data.respond(await this.runBackendDoctor());
1924
+ }
1925
+ else if (data.command === "usage") {
1926
+ // Same permission level as /ctx (none). The reply is still ephemeral —
1927
+ // the adapter defers non-chat commands that way — so it never spams the
1928
+ // channel either way.
1578
1929
  try {
1579
- const { execSync } = await import("node:child_process");
1580
- const backend = this.fleetConfig?.defaults?.backend || "claude-code";
1581
- const result = execSync(`agend backend doctor ${backend}`, { timeout: 30_000, encoding: "utf-8" });
1582
- const clean = result.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
1583
- await data.respond(clean || "No output");
1930
+ const { getUsageSnapshot } = await import("./usage/usage-api.js");
1931
+ const { renderUsageMarkdown } = await import("./usage/format-rich.js");
1932
+ // slash_command is Discord-only; editReply renders Markdown natively.
1933
+ await data.respond(renderUsageMarkdown(await getUsageSnapshot()));
1584
1934
  }
1585
1935
  catch (err) {
1586
- const output = (err.stdout ?? err.message ?? "Doctor failed").replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
1587
- await data.respond(output);
1936
+ await data.respond(`⚠️ Usage fetch failed: ${err.message}`);
1588
1937
  }
1589
1938
  }
1590
1939
  else if (data.command === "status") {
1940
+ // Admin-gated (like the topic path): the merged table shows every
1941
+ // instance's cost and IPC health.
1942
+ if (!this.isFleetAdmin(data.userId, adapterId)) {
1943
+ await data.respond(t("cmd.admin_required", "/status"));
1944
+ return;
1945
+ }
1591
1946
  const text = await this.topicCommands.getStatusText();
1592
1947
  await data.respond(text);
1593
1948
  }
@@ -1696,11 +2051,16 @@ export class FleetManager {
1696
2051
  adapter.on("message", safeHandler(async (msg) => {
1697
2052
  await this.handleInboundMessage(msg);
1698
2053
  }, this.logger, `adapter[${adapterId}].message`));
2054
+ adapter.on("reaction", safeHandler(async (r) => {
2055
+ await this.handleInboundReaction(r);
2056
+ }, this.logger, `adapter[${adapterId}].reaction`));
1699
2057
  adapter.on("callback_query", safeHandler(async (data) => {
1700
2058
  if (await this.handleClassicBackendSelection(data))
1701
2059
  return;
1702
2060
  if (await this.handleModelSelection(data))
1703
2061
  return;
2062
+ if (await this.handleEffortSelection(data))
2063
+ return;
1704
2064
  if (data.callbackData.startsWith("hang:")) {
1705
2065
  const parts = data.callbackData.split(":");
1706
2066
  const action = parts[1];
@@ -1720,10 +2080,7 @@ export class FleetManager {
1720
2080
  return;
1721
2081
  }
1722
2082
  if (data.callbackData.startsWith("cancel:")) {
1723
- const instanceName = data.callbackData.slice("cancel:".length);
1724
- // Idempotent: only the first click (while the button is live) acts.
1725
- if (this.hasCancelButton(instanceName))
1726
- this.cancelInstance(instanceName);
2083
+ this.handleCancelClick(data.callbackData.slice("cancel:".length), adapter, data);
1727
2084
  return;
1728
2085
  }
1729
2086
  }, this.logger, `adapter[${adapterId}].callback_query`));
@@ -1793,6 +2150,9 @@ export class FleetManager {
1793
2150
  else if (data.command === "model") {
1794
2151
  await this.handleModelSlash(data, adapterId);
1795
2152
  }
2153
+ else if (data.command === "effort") {
2154
+ await this.handleEffortSlash(data, adapterId);
2155
+ }
1796
2156
  else if (data.command === "cancel") {
1797
2157
  const name = this.resolveSlashTarget(data.channelId, adapterId);
1798
2158
  if (!name) {
@@ -1857,19 +2217,29 @@ export class FleetManager {
1857
2217
  await data.respond(t("not_authorized"));
1858
2218
  return;
1859
2219
  }
2220
+ await data.respond(await this.runBackendDoctor());
2221
+ }
2222
+ else if (data.command === "usage") {
2223
+ // Same permission level as /ctx (none). The reply is still ephemeral —
2224
+ // the adapter defers non-chat commands that way — so it never spams the
2225
+ // channel either way.
1860
2226
  try {
1861
- const { execSync } = await import("node:child_process");
1862
- const backend = this.fleetConfig?.defaults?.backend || "claude-code";
1863
- const result = execSync(`agend backend doctor ${backend}`, { timeout: 30_000, encoding: "utf-8" });
1864
- const clean = result.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
1865
- await data.respond(clean || "No output");
2227
+ const { getUsageSnapshot } = await import("./usage/usage-api.js");
2228
+ const { renderUsageMarkdown } = await import("./usage/format-rich.js");
2229
+ // slash_command is Discord-only; editReply renders Markdown natively.
2230
+ await data.respond(renderUsageMarkdown(await getUsageSnapshot()));
1866
2231
  }
1867
2232
  catch (err) {
1868
- const output = (err.stdout ?? err.message ?? "Doctor failed").replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
1869
- await data.respond(output);
2233
+ await data.respond(`⚠️ Usage fetch failed: ${err.message}`);
1870
2234
  }
1871
2235
  }
1872
2236
  else if (data.command === "status") {
2237
+ // Admin-gated (like the topic path): the merged table shows every
2238
+ // instance's cost and IPC health.
2239
+ if (!this.isFleetAdmin(data.userId, adapterId)) {
2240
+ await data.respond(t("cmd.admin_required", "/status"));
2241
+ return;
2242
+ }
1873
2243
  const text = await this.topicCommands.getStatusText();
1874
2244
  await data.respond(text);
1875
2245
  }
@@ -2025,6 +2395,9 @@ export class FleetManager {
2025
2395
  else if (msg.type === "instance_process_state") {
2026
2396
  this.cacheInstanceProcessStatus(name, msg.status);
2027
2397
  }
2398
+ else if (msg.type === "instance_activity") {
2399
+ this.cacheInstanceActivity(name, msg.activity);
2400
+ }
2028
2401
  else if (msg.type === "instance_state" || msg.type === "instance_state_response") {
2029
2402
  this.cacheInstanceExecutionState(name, msg);
2030
2403
  if (msg.type === "instance_state_response") {
@@ -2095,9 +2468,22 @@ export class FleetManager {
2095
2468
  if (existsSync(windowIdPath)) {
2096
2469
  const windowId = readFileSync(windowIdPath, "utf-8").trim();
2097
2470
  if (windowId) {
2471
+ // Async with an explicit timeout: this was execSync with NO timeout at
2472
+ // all, so a wedged tmux server blocked the whole fleet event loop
2473
+ // indefinitely — while we were here to diagnose a lost connection.
2474
+ // A timeout is also the correct signal: an unresponsive tmux server
2475
+ // means we cannot verify the pane, which is treated as dead (the same
2476
+ // conclusion the old code reached only by throwing).
2098
2477
  try {
2099
- const { execSync } = await import("node:child_process");
2100
- execSync(`tmux list-panes -t "${windowId}"`, { stdio: "ignore" });
2478
+ const { execFile } = await import("node:child_process");
2479
+ const { promisify } = await import("node:util");
2480
+ const { getTmuxSocketName } = await import("./paths.js");
2481
+ // Honour socket isolation: without -L this queried the user's default
2482
+ // tmux server instead of the fleet's, so under a custom AGEND_HOME the
2483
+ // check was meaningless (it reported every pane dead).
2484
+ const socket = getTmuxSocketName();
2485
+ const args = socket ? ["-L", socket, "list-panes", "-t", windowId] : ["list-panes", "-t", windowId];
2486
+ await promisify(execFile)("tmux", args, { timeout: 5_000 });
2101
2487
  }
2102
2488
  catch {
2103
2489
  // Pane dead — respawn
@@ -2118,6 +2504,12 @@ export class FleetManager {
2118
2504
  if (this.adapterRestarting.has(id))
2119
2505
  return;
2120
2506
  this.adapterRestarting.add(id);
2507
+ // Reflect reality in adapterState throughout. This loop used to leave the state
2508
+ // untouched, so getAdapterStates() — and therefore /health and the dashboard —
2509
+ // kept reporting "connected" for an adapter that had been down for hours. An
2510
+ // adapter's true status was simply not knowable from inside the process.
2511
+ const previous = this.adapterState.get(id);
2512
+ this.adapterState.set(id, { status: "retrying", retryCount: 0, lastError: previous?.lastError });
2121
2513
  try {
2122
2514
  for (let attempt = 1;; attempt++) {
2123
2515
  if (this.ipcStoppingInstances.has("__fleet_stopping__"))
@@ -2130,9 +2522,16 @@ export class FleetManager {
2130
2522
  await adapter.stop().catch(() => { });
2131
2523
  await adapter.start();
2132
2524
  this.logger.info({ id, attempt }, "Adapter restarted successfully");
2525
+ this.adapterState.set(id, { status: "connected", retryCount: 0 });
2133
2526
  return;
2134
2527
  }
2135
- catch { /* retry */ }
2528
+ catch (err) {
2529
+ this.adapterState.set(id, {
2530
+ status: "retrying",
2531
+ retryCount: attempt,
2532
+ lastError: err?.message ?? String(err),
2533
+ });
2534
+ }
2136
2535
  if (attempt % 10 === 0) {
2137
2536
  this.logger.warn({ id, attempt }, "Adapter restart still failing");
2138
2537
  }
@@ -2168,6 +2567,49 @@ export class FleetManager {
2168
2567
  }
2169
2568
  return generals[0];
2170
2569
  }
2570
+ /**
2571
+ * A user reacted to one of the bot's messages (#408).
2572
+ *
2573
+ * A reaction is context, not a message (#432, reworking #413): it never triggers
2574
+ * an agent turn and never wakes anything. It is queued in the event log and rides
2575
+ * into the instance's NEXT real message as one compact leading line —
2576
+ * `[Recent reactions: 👍×2 from hanhanv]` — after which it is marked consumed.
2577
+ * No pending reactions → no line → zero context spent, which is the common case.
2578
+ */
2579
+ async handleInboundReaction(r) {
2580
+ const instanceName = this.resolveSlashTarget(r.threadId ?? r.chatId, r.adapterId);
2581
+ if (!instanceName) {
2582
+ this.logger.debug({ emoji: r.emoji, chatId: r.chatId }, "Reaction in an unrouted channel — ignoring");
2583
+ return;
2584
+ }
2585
+ if (DELIVERY_STATUS_EMOJIS.has(r.emoji)) {
2586
+ this.logger.debug({ emoji: r.emoji, user: r.username }, "Ignoring delivery-status emoji as a reaction");
2587
+ return;
2588
+ }
2589
+ this.eventLog?.logActivity("reaction", r.username, `${r.emoji} ${r.action}`, instanceName);
2590
+ if (r.action === "add") {
2591
+ this.eventLog?.addReaction(instanceName, r.messageId, r.username, r.emoji);
2592
+ }
2593
+ else {
2594
+ // Withdrawn before anyone saw it → it never happened. See removeReaction.
2595
+ this.eventLog?.removeReaction(instanceName, r.messageId, r.username, r.emoji);
2596
+ }
2597
+ }
2598
+ /**
2599
+ * The queued-reaction summary for an instance's next real message, or {} when
2600
+ * nothing is pending (the common case must add zero context). The consume
2601
+ * callback is separate from the fetch so reactions are only marked once the
2602
+ * message actually went out — a failed delivery keeps them queued.
2603
+ */
2604
+ pendingReactionsMeta(instanceName) {
2605
+ const pending = this.eventLog?.pendingReactions(instanceName);
2606
+ if (!pending)
2607
+ return { meta: {}, consume: () => { } };
2608
+ return {
2609
+ meta: { pending_reactions: pending.summary },
2610
+ consume: () => this.eventLog?.markReactionsConsumed(instanceName, pending.maxId),
2611
+ };
2612
+ }
2171
2613
  async handleInboundMessage(msg) {
2172
2614
  const threadId = msg.threadId || undefined;
2173
2615
  this.logger.debug({ source: msg.source, chatId: msg.chatId, threadId, userId: msg.userId, isBotMessage: msg.isBotMessage, textLen: (msg.text ?? "").length, text: (msg.text ?? "").slice(0, 80) }, "handleInboundMessage entry");
@@ -2505,6 +2947,7 @@ export class FleetManager {
2505
2947
  }
2506
2948
  this.warnIfRateLimited(generalInstance, msg);
2507
2949
  const { text, extraMeta } = await processAttachments(msg, inboundAdapter, this.logger, generalInstance);
2950
+ const generalReactions = this.pendingReactionsMeta(generalInstance);
2508
2951
  try {
2509
2952
  await this.deliverToInstance(generalInstance, {
2510
2953
  type: "fleet_inbound",
@@ -2520,9 +2963,11 @@ export class FleetManager {
2520
2963
  adapter_id: msg.adapterId,
2521
2964
  source: msg.source,
2522
2965
  ...(msg.replyToText ? { reply_to_text: msg.replyToText } : {}),
2966
+ ...generalReactions.meta,
2523
2967
  ...extraMeta,
2524
2968
  },
2525
2969
  });
2970
+ generalReactions.consume();
2526
2971
  this.lastInboundUser.set(generalInstance, msg.username);
2527
2972
  this.logger.info(`${msg.username} → ${generalInstance}: ${(text ?? "").slice(0, 100)}`);
2528
2973
  this.eventLog?.logActivity("message", msg.username, (text ?? "").slice(0, 200), generalInstance);
@@ -2598,6 +3043,7 @@ export class FleetManager {
2598
3043
  this.setTopicIcon(instanceName, "blue");
2599
3044
  this.warnIfRateLimited(instanceName, msg);
2600
3045
  const { text, extraMeta } = await processAttachments(msg, inboundAdapter, this.logger, instanceName);
3046
+ const reactions = this.pendingReactionsMeta(instanceName);
2601
3047
  try {
2602
3048
  await this.deliverToInstance(instanceName, {
2603
3049
  type: "fleet_inbound",
@@ -2613,9 +3059,13 @@ export class FleetManager {
2613
3059
  adapter_id: msg.adapterId,
2614
3060
  source: msg.source,
2615
3061
  ...(msg.replyToText ? { reply_to_text: msg.replyToText } : {}),
3062
+ ...reactions.meta,
2616
3063
  ...extraMeta,
2617
3064
  },
2618
3065
  });
3066
+ // Only after the message actually went out. A failed delivery keeps the
3067
+ // reactions queued for the retry / the next message.
3068
+ reactions.consume();
2619
3069
  }
2620
3070
  catch (err) {
2621
3071
  this.logger.warn({ err: err.message, instanceName }, "Wake/delivery failed");
@@ -2704,25 +3154,33 @@ export class FleetManager {
2704
3154
  delete args.thread_id;
2705
3155
  threadId = undefined;
2706
3156
  }
3157
+ // Reply dedup: retries land here when the agent was told a send failed
3158
+ // (daemon budget elapsed, shell tool killed) while the adapter send was
3159
+ // still in flight and about to succeed. One real send, everyone gets its
3160
+ // outcome; a genuinely failed send clears the entry so a retry passes.
3161
+ if (tool === "reply") {
3162
+ const ticket = this.replyDeduper.begin(instanceName, String(args.text ?? ""), Array.isArray(args.files) ? args.files : []);
3163
+ if (ticket.duplicate) {
3164
+ this.logger.info({ instanceName }, "Duplicate reply suppressed — replaying the original send's outcome");
3165
+ ticket.subscribe(respond);
3166
+ return;
3167
+ }
3168
+ const original = respond;
3169
+ const respondAndRecord = (result, error) => {
3170
+ ticket.complete(result, error);
3171
+ original(result, error);
3172
+ };
3173
+ if (routeToolCall(outAdapter, tool, args, threadId, respondAndRecord)) {
3174
+ this.afterReplyRouted(instanceName, args, senderSessionName);
3175
+ return;
3176
+ }
3177
+ // routeToolCall knows "reply"; not handling it means the world changed.
3178
+ ticket.complete(null, "reply not handled");
3179
+ original(null, "reply not handled");
3180
+ return;
3181
+ }
2707
3182
  // Route standard channel tools (reply, react, edit_message, download_attachment)
2708
3183
  if (routeToolCall(outAdapter, tool, args, threadId, respond)) {
2709
- if (tool === "reply") {
2710
- // Agent answered — retire its pending cancel button and mark ✅ done.
2711
- this.clearCancelButton(instanceName);
2712
- this.reactDone(instanceName);
2713
- const replyTo = this.lastInboundUser.get(instanceName) ?? "user";
2714
- this.logger.info(`${instanceName} → ${replyTo}: ${(args.text ?? "").slice(0, 100)}`);
2715
- this.emitSseEvent("message", {
2716
- instance: instanceName, sender: senderSessionName ?? instanceName,
2717
- text: (args.text ?? "").slice(0, 2000),
2718
- ts: new Date().toISOString(),
2719
- });
2720
- // Log bot reply to classic instance chat-log
2721
- const isClassic = this.classicChannels?.getChannelIdByInstance(instanceName) !== undefined;
2722
- if (isClassic) {
2723
- ClassicChannelManager.logMessage(instanceName, "bot", args.text ?? "", new Date());
2724
- }
2725
- }
2726
3184
  return;
2727
3185
  }
2728
3186
  // Log tool calls for activity visualization
@@ -2737,6 +3195,35 @@ export class FleetManager {
2737
3195
  respond(null, `Unknown tool: ${tool}`);
2738
3196
  }
2739
3197
  }
3198
+ /** Side effects of a routed reply: cancel-button lifecycle, logs, SSE, chat log. */
3199
+ afterReplyRouted(instanceName, args, senderSessionName) {
3200
+ // A reply is NOT proof the turn is over (#410) — but it is not proof of
3201
+ // more work either. Split the difference: an instance that is clearly
3202
+ // idle loses the button now; one that looks busy keeps it (re-posted
3203
+ // below the reply so it stays last in the channel), with a 2-minute
3204
+ // grace check — if it has NOT resumed working by then, the reply was the
3205
+ // end of the turn and the button goes. A multi-step run that keeps
3206
+ // working sails through the check and keeps its button.
3207
+ if (this.getInstanceIdle(instanceName)) {
3208
+ this.clearCancelButton(instanceName);
3209
+ }
3210
+ else {
3211
+ void this.sendCancelButton(instanceName).then(() => this.armReplyGrace(instanceName));
3212
+ }
3213
+ this.reactDone(instanceName);
3214
+ const replyTo = this.lastInboundUser.get(instanceName) ?? "user";
3215
+ this.logger.info(`${instanceName} → ${replyTo}: ${(args.text ?? "").slice(0, 100)}`);
3216
+ this.emitSseEvent("message", {
3217
+ instance: instanceName, sender: senderSessionName ?? instanceName,
3218
+ text: (args.text ?? "").slice(0, 2000),
3219
+ ts: new Date().toISOString(),
3220
+ });
3221
+ // Log bot reply to classic instance chat-log
3222
+ const isClassic = this.classicChannels?.getChannelIdByInstance(instanceName) !== undefined;
3223
+ if (isClassic) {
3224
+ ClassicChannelManager.logMessage(instanceName, "bot", args.text ?? "", new Date());
3225
+ }
3226
+ }
2740
3227
  /** Handle tool status update from a daemon instance */
2741
3228
  handleToolStatusFromInstance(instanceName, msg) {
2742
3229
  const statusAdapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
@@ -3339,6 +3826,9 @@ export class FleetManager {
3339
3826
  }
3340
3827
  }
3341
3828
  async removeInstance(name) {
3829
+ // Drop cached pane context — the map is keyed by instance name and nothing
3830
+ // else evicted deleted entries, so it grew for the life of the process.
3831
+ forgetInstanceContext(name);
3342
3832
  // Clean up schedules (scheduler is fleet-level, not lifecycle-level)
3343
3833
  const config = this.fleetConfig?.instances[name];
3344
3834
  if (this.scheduler && config?.topic_id) {
@@ -3444,6 +3934,171 @@ export class FleetManager {
3444
3934
  this.collabInstances.add(instanceName);
3445
3935
  return true;
3446
3936
  }
3937
+ /**
3938
+ * Open the event log, tolerating a corrupt file.
3939
+ *
3940
+ * `events.db` holds history only — event rows and the activity feed. Nothing the
3941
+ * fleet needs to run depends on it, and every consumer already uses
3942
+ * `this.eventLog?.`. An unguarded `new EventLog(...)` here meant a corrupt or
3943
+ * unreadable history file (a truncated WAL after a hard kill, a full disk)
3944
+ * threw during startAll and the WHOLE FLEET FAILED TO BOOT — trading every
3945
+ * running agent for a file whose only job is reporting.
3946
+ *
3947
+ * So: try, move a bad file aside and retry once with a fresh one, and if even
3948
+ * that fails carry on without an event log.
3949
+ */
3950
+ /**
3951
+ * Run `agend backend doctor` for the fleet's default backend and return its
3952
+ * cleaned output.
3953
+ *
3954
+ * Async on purpose: this was `execSync` with a 30s timeout, reachable by any
3955
+ * allowlisted user through `/doctor`. While it ran, the entire fleet event loop
3956
+ * was frozen — no IPC, no adapter, no message delivery, no health responses,
3957
+ * and critically no WATCHDOG ping, so a slow doctor could push past
3958
+ * WatchdogSec and have systemd SIGABRT the fleet.
3959
+ */
3960
+ async runBackendDoctor() {
3961
+ const stripAnsi = (s) => s.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
3962
+ const backend = this.fleetConfig?.defaults?.backend || "claude-code";
3963
+ try {
3964
+ const { execFile } = await import("node:child_process");
3965
+ const { promisify } = await import("node:util");
3966
+ // execFile with an argv array — no shell, so the backend name cannot be
3967
+ // interpreted as a command even if config is malformed.
3968
+ const { stdout } = await promisify(execFile)("agend", ["backend", "doctor", backend], {
3969
+ timeout: 30_000,
3970
+ encoding: "utf-8",
3971
+ });
3972
+ return stripAnsi(stdout) || "No output";
3973
+ }
3974
+ catch (err) {
3975
+ const e = err;
3976
+ return stripAnsi(e.stdout ?? e.message ?? "Doctor failed");
3977
+ }
3978
+ }
3979
+ /** Drop event/activity rows older than the retention window. Best-effort. */
3980
+ /**
3981
+ * Cap every instance's pipe-pane log, walking the instances **directory** rather
3982
+ * than the config.
3983
+ *
3984
+ * A running instance rotates its own log on each health tick, so the ones that
3985
+ * need this are the ones nothing else looks at:
3986
+ *
3987
+ * - deleted instances, whose directory outlives the config entry. Nothing ever
3988
+ * touched these again. On the machine this was found on, one held 122 MB and
3989
+ * another 74 MB, out of 622 MB of pipe-pane logs in total.
3990
+ * - classic instances, which live in classicChannels, not fleetConfig.instances,
3991
+ * and so were never in the old config-driven loop at all.
3992
+ * - stopped instances, which have no health tick running.
3993
+ *
3994
+ * pipe-pane writes raw TUI output, so a wedged splash screen can emit ANSI frames
3995
+ * at animation rate. Unbounded growth here fills the disk, which takes the whole
3996
+ * fleet down rather than one instance.
3997
+ */
3998
+ rotateAllInstanceLogs() {
3999
+ const root = join(this.dataDir, "instances");
4000
+ let entries;
4001
+ try {
4002
+ entries = readdirSync(root, { withFileTypes: true });
4003
+ }
4004
+ catch {
4005
+ return; // no instances directory yet
4006
+ }
4007
+ for (const entry of entries) {
4008
+ if (!entry.isDirectory())
4009
+ continue;
4010
+ // rotateLogIfNeeded is already best-effort and returns early on a missing
4011
+ // file, so a directory without a pipe-pane log costs one stat.
4012
+ rotateLogIfNeeded(join(root, entry.name, "output.log"));
4013
+ }
4014
+ }
4015
+ pruneEventLog() {
4016
+ try {
4017
+ this.eventLog?.prune(FleetManager.EVENT_LOG_RETENTION_DAYS);
4018
+ }
4019
+ catch (err) {
4020
+ this.logger.warn({ err }, "Event log prune failed");
4021
+ }
4022
+ }
4023
+ openEventLog() {
4024
+ const dbPath = join(this.dataDir, "events.db");
4025
+ try {
4026
+ return new EventLog(dbPath);
4027
+ }
4028
+ catch (err) {
4029
+ this.logger.error({ err, dbPath }, "events.db unusable — moving it aside and starting a fresh one");
4030
+ const stamp = new Date().toISOString().replace(/[:.]/g, "-");
4031
+ for (const suffix of ["", "-wal", "-shm"]) {
4032
+ try {
4033
+ renameSync(`${dbPath}${suffix}`, `${dbPath}${suffix}.corrupt-${stamp}`);
4034
+ }
4035
+ catch { /* may not exist */ }
4036
+ }
4037
+ try {
4038
+ return new EventLog(dbPath);
4039
+ }
4040
+ catch (retryErr) {
4041
+ // History is worth losing; a fleet that won't start is not.
4042
+ this.logger.error({ err: retryErr, dbPath }, "Could not open a fresh events.db — continuing without event logging");
4043
+ return null;
4044
+ }
4045
+ }
4046
+ }
4047
+ /**
4048
+ * Report a fleet-level fault (not attributable to one instance) to the General
4049
+ * topic, so the operator learns about it without reading daemon.log.
4050
+ *
4051
+ * Throttled per distinct message: an unhandled rejection typically comes from a
4052
+ * loop (a poller, a repeating timer), and one channel message per occurrence
4053
+ * would bury the topic — which is worse than silence. First occurrence goes out
4054
+ * immediately, repeats are suppressed for THROTTLE_MS and then re-sent with a
4055
+ * count.
4056
+ *
4057
+ * The log line is written by the caller regardless: if every adapter is down,
4058
+ * the only notification path is the one that is broken.
4059
+ */
4060
+ notifyFleetError(text) {
4061
+ const now = Date.now();
4062
+ const key = text.slice(0, 200);
4063
+ const seen = this.fleetErrorNotices.get(key);
4064
+ if (seen && now - seen.at < FleetManager.FLEET_ERROR_THROTTLE_MS) {
4065
+ seen.suppressed++;
4066
+ return;
4067
+ }
4068
+ const suppressed = seen?.suppressed ?? 0;
4069
+ this.fleetErrorNotices.set(key, { at: now, suppressed: 0 });
4070
+ // Bound the map: it is keyed by message text, and a message with a varying
4071
+ // suffix (a path, an id) would otherwise grow it without limit.
4072
+ if (this.fleetErrorNotices.size > 100) {
4073
+ const oldest = this.fleetErrorNotices.keys().next().value;
4074
+ if (oldest !== undefined)
4075
+ this.fleetErrorNotices.delete(oldest);
4076
+ }
4077
+ const body = suppressed > 0
4078
+ ? `${text}\n(plus ${suppressed} more in the last ${Math.round(FleetManager.FLEET_ERROR_THROTTLE_MS / 60_000)}m)`
4079
+ : text;
4080
+ // Resolved from config, NOT findGeneralInstance(): that requires a live daemon,
4081
+ // and a fleet-level fault is exactly when the General may be down. The topic
4082
+ // itself still exists, and notifyInstanceTopic only needs adapter + group +
4083
+ // topic_id to post into it.
4084
+ const general = Object.entries(this.fleetConfig?.instances ?? {})
4085
+ .find(([, config]) => config.general_topic === true)?.[0];
4086
+ if (general) {
4087
+ this.notifyInstanceTopic(general, body);
4088
+ return;
4089
+ }
4090
+ // No General instance — fall back to the primary channel's group.
4091
+ const channelCfg = this.getChannelConfig();
4092
+ const groupId = channelCfg?.group_id;
4093
+ if (this.adapter && groupId) {
4094
+ this.adapter.sendText(String(groupId), body)
4095
+ .catch(err => this.logger.warn({ err }, "Failed to send fleet error notification"));
4096
+ return;
4097
+ }
4098
+ this.logger.warn({ text: body }, "Fleet error had no notification target (no General instance, no adapter)");
4099
+ }
4100
+ static FLEET_ERROR_THROTTLE_MS = 10 * 60_000;
4101
+ fleetErrorNotices = new Map();
3447
4102
  notifyInstanceTopic(instanceName, text, extraOpts) {
3448
4103
  const adapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
3449
4104
  if (!adapter)
@@ -3517,6 +4172,40 @@ export class FleetManager {
3517
4172
  await data.respond(t("save.sent", cmd, target.name));
3518
4173
  }
3519
4174
  /** Whether the instance currently has at least one live cancel button. */
4175
+ /**
4176
+ * A click on a cancel button, whether or not the fleet still tracks it.
4177
+ *
4178
+ * The old rule was "act only while an entry is live", which made a click on a
4179
+ * button the fleet had forgotten a silent no-op — no cancel, no message, not
4180
+ * even a log line. That is indistinguishable from a broken button, and it is
4181
+ * what the "按鈕點了沒反應" reports were: the entry is briefly absent while a
4182
+ * button is being replaced, and a delete that fails leaves the message on
4183
+ * screen with no entry at all.
4184
+ *
4185
+ * So: honour the click if the instance is actually running, and say so plainly
4186
+ * if it is not. The stale-click path is rate-limited because the original
4187
+ * concern was real — a second click must not fire a second interrupt key at an
4188
+ * instance that has already started a new turn.
4189
+ */
4190
+ handleCancelClick(instanceName, adapter, data) {
4191
+ if (this.hasCancelButton(instanceName)) {
4192
+ this.cancelInstance(instanceName);
4193
+ return;
4194
+ }
4195
+ const lastAt = this.staleCancelClickAt.get(instanceName) ?? 0;
4196
+ if (Date.now() - lastAt < STALE_CANCEL_CLICK_COOLDOWN_MS)
4197
+ return;
4198
+ this.staleCancelClickAt.set(instanceName, Date.now());
4199
+ // cancelInstance returns false when there is no daemon — i.e. nothing to
4200
+ // cancel, which is the one case where the button really is dead.
4201
+ if (this.cancelInstance(instanceName)) {
4202
+ this.logger.info({ instanceName }, "Cancel click honoured with no live button entry");
4203
+ return;
4204
+ }
4205
+ this.logger.info({ instanceName }, "Cancel click on an expired button — instance not running");
4206
+ adapter?.editMessage(data.chatId, data.messageId, t("cancel.button_stale", instanceName), data.threadId)
4207
+ .catch(() => { });
4208
+ }
3520
4209
  hasCancelButton(instanceName) {
3521
4210
  for (const e of this.cancelButtons.values()) {
3522
4211
  if (e.instanceName === instanceName)
@@ -3525,15 +4214,22 @@ export class FleetManager {
3525
4214
  return false;
3526
4215
  }
3527
4216
  async sendCancelButton(instanceName, correlationId) {
3528
- // At most one button shown per instance: retire any existing ones first
3529
- // (delete + bounded retry). Each is tracked separately, so a failed delete
3530
- // here doesn't strand it it keeps retrying on its own timer.
3531
- this.retireInstanceButtons(instanceName);
4217
+ // Post first, retire after (see the tail of this method). Retiring up front
4218
+ // meant that from the delete until the new message came back a chat API
4219
+ // round trip, and every reply goes through here the instance had NO live
4220
+ // entry, while the old button was still on screen. A click in that window
4221
+ // hit `hasCancelButton() === false` and was silently dropped: the reported
4222
+ // "按鈕失效". If notifyAlert then failed, the button was simply gone.
3532
4223
  const adapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
3533
4224
  if (!adapter)
3534
4225
  return;
4226
+ // Resolve the group through the world fallback (first world when unbound),
4227
+ // NOT through getChannelConfig(binding)?.group_id: on a fleet configured with
4228
+ // `channels:` worlds the primary `channel:` block is empty, so an instance
4229
+ // with no world binding yet (fresh restart, cross-instance delegation)
4230
+ // resolved group_id to undefined and the button silently never appeared.
3535
4231
  const adapterId = this.instanceWorldBinding.get(instanceName);
3536
- const groupId = this.getChannelConfig(adapterId)?.group_id;
4232
+ const groupId = this.getGroupIdForInstance(instanceName) || undefined;
3537
4233
  const topicId = this.fleetConfig?.instances[instanceName]?.topic_id;
3538
4234
  let chatId;
3539
4235
  let threadId;
@@ -3549,8 +4245,12 @@ export class FleetManager {
3549
4245
  if (!chatId && groupId)
3550
4246
  chatId = String(groupId);
3551
4247
  }
3552
- if (!chatId)
4248
+ if (!chatId) {
4249
+ // A button that cannot be addressed must say so — this exact silence is how
4250
+ // "the cancel button sometimes never appears" stayed unreported-in-logs.
4251
+ this.logger.warn({ instanceName, topicId, groupId }, "Cannot address cancel button (no chat id resolved)");
3553
4252
  return;
4253
+ }
3554
4254
  try {
3555
4255
  const sent = await adapter.notifyAlert(chatId, {
3556
4256
  type: "cancel",
@@ -3558,13 +4258,6 @@ export class FleetManager {
3558
4258
  message: "👀 處理中…",
3559
4259
  choices: [{ id: `cancel:${instanceName}`, label: t("cancel.button") }],
3560
4260
  }, threadId ? { threadId } : undefined);
3561
- // A concurrent sendCancelButton for the same instance may have posted its
3562
- // own button while we awaited notifyAlert. Retire any other buttons for
3563
- // this instance (not the one we just posted) so only the newest shows.
3564
- for (const other of this.cancelButtons.values()) {
3565
- if (other.instanceName === instanceName)
3566
- this.retireButton(other);
3567
- }
3568
4261
  const entry = {
3569
4262
  instanceName,
3570
4263
  adapterId,
@@ -3573,7 +4266,16 @@ export class FleetManager {
3573
4266
  threadId: sent.threadId ?? threadId,
3574
4267
  correlationId,
3575
4268
  retryCount: 0,
4269
+ // Elapsed time is measured from when this button was posted — i.e. from
4270
+ // when the work was handed over — not from the pane's working transition,
4271
+ // which resets if the CLI blips idle mid-turn.
4272
+ startedAt: Date.now(),
4273
+ // Matches the text notifyAlert just posted, so the first 60s tick does
4274
+ // not re-edit identical text — which put a "(edited)" mark on Discord
4275
+ // with nothing visibly changed.
4276
+ lastProgressText: "👀 處理中…",
3576
4277
  };
4278
+ this.startProgressTicker(entry);
3577
4279
  // Idle-check backstop: every 5min, if the instance is idle, retire the
3578
4280
  // button. Covers turns that end without hitting a clear trigger (reply /
3579
4281
  // cancel / correlation). Cleared in discardButton when the entry is removed.
@@ -3582,18 +4284,167 @@ export class FleetManager {
3582
4284
  clearInterval(entry.idleCheckTimer);
3583
4285
  return;
3584
4286
  }
3585
- if (this.getInstanceIdle(instanceName)) {
3586
- this.logger.info({ instanceName, messageId: entry.messageId }, "Cancel button idle backstop retiring");
4287
+ const reason = this.getInstanceIdle(instanceName) ? "idle"
4288
+ : this.stateReportDead(instanceName) ? "state reports stopped"
4289
+ : Date.now() - (entry.startedAt ?? 0) > CANCEL_BTN_MAX_LIFETIME_MS ? "24h ceiling"
4290
+ : null;
4291
+ if (reason) {
4292
+ this.logger.info({ instanceName, messageId: entry.messageId, reason }, "Cancel button backstop retiring");
3587
4293
  this.retireButton(entry);
4294
+ return;
3588
4295
  }
4296
+ // Still looks busy. The daemon only broadcasts on transitions, so ask for
4297
+ // a fresh snapshot — a live daemon's answer refreshes receivedAt and keeps
4298
+ // the staleness check honest; a dead one's silence is the evidence.
4299
+ this.instanceIpcClients.get(instanceName)?.send({
4300
+ type: "query_instance_state", requestId: `cancel-btn-${Date.now()}`,
4301
+ });
3589
4302
  }, CANCEL_BTN_IDLE_CHECK_INTERVAL_MS);
3590
4303
  this.cancelButtons.set(sent.messageId, entry);
4304
+ // Only now: at most one button per instance, but never zero. Covers both
4305
+ // the previous turn's button and any button a concurrent
4306
+ // sendCancelButton posted while we were awaiting notifyAlert.
4307
+ for (const other of [...this.cancelButtons.values()]) {
4308
+ if (other.instanceName === instanceName && other.messageId !== sent.messageId) {
4309
+ this.retireButton(other);
4310
+ }
4311
+ }
4312
+ this.persistCancelButtons();
3591
4313
  this.logger.info({ instanceName, messageId: sent.messageId }, "Cancel button sent");
3592
4314
  }
3593
4315
  catch (e) {
3594
4316
  this.logger.warn({ err: e.message, instanceName }, "Failed to send cancel button");
3595
4317
  }
3596
4318
  }
4319
+ /**
4320
+ * The cancel button's text for a given elapsed time.
4321
+ *
4322
+ * Below the threshold it keeps the original wording, so a normal quick answer
4323
+ * looks exactly as it did before. Past it, the button doubles as the live
4324
+ * progress indicator (#409) — the channel showed nothing at all during long work,
4325
+ * and once the agent had replied once there was no sign it was still going.
4326
+ */
4327
+ static progressText(elapsedMs, activity, minElapsedMs = PROGRESS_MIN_ELAPSED_MS) {
4328
+ if (elapsedMs < minElapsedMs)
4329
+ return "👀 處理中…";
4330
+ const totalSeconds = Math.floor(elapsedMs / 1000);
4331
+ const minutes = Math.floor(totalSeconds / 60);
4332
+ const seconds = totalSeconds % 60;
4333
+ const elapsed = minutes >= 60
4334
+ ? `${Math.floor(minutes / 60)}h ${minutes % 60}m`
4335
+ : `${minutes}m ${String(seconds).padStart(2, "0")}s`;
4336
+ const detail = FleetManager.sanitizeActivity(activity);
4337
+ return detail
4338
+ ? `⏳ 處理中… (已進行 ${elapsed} · ${detail})`
4339
+ : `⏳ 處理中… (已進行 ${elapsed})`;
4340
+ }
4341
+ /**
4342
+ * Make a tool summary safe to paste into a channel message.
4343
+ *
4344
+ * The text is agent-controlled (it is built from tool inputs — file paths,
4345
+ * shell commands), so it gets flattened to one line, capped, and stripped of
4346
+ * the two Discord mass-mention triggers. Neither channel renders it with a
4347
+ * parse mode, so no markup escaping is needed beyond that.
4348
+ */
4349
+ static sanitizeActivity(activity) {
4350
+ if (!activity)
4351
+ return null;
4352
+ const flat = activity.replace(/\s+/g, " ").replace(/@(everyone|here)/g, "@​$1").trim();
4353
+ if (!flat)
4354
+ return null;
4355
+ return flat.length > PROGRESS_ACTIVITY_MAX_CHARS
4356
+ ? `${flat.slice(0, PROGRESS_ACTIVITY_MAX_CHARS - 1)}…`
4357
+ : flat;
4358
+ }
4359
+ /**
4360
+ * Remember what an instance is currently doing, for the progress line.
4361
+ *
4362
+ * Best-effort by design: only backends that expose a live activity feed report
4363
+ * anything, and the progress line simply omits the detail for the rest. It is
4364
+ * never used to decide anything — purely what the user is shown.
4365
+ */
4366
+ cacheInstanceActivity(name, activity) {
4367
+ if (activity)
4368
+ this.instanceActivity.set(name, activity);
4369
+ else
4370
+ this.instanceActivity.delete(name);
4371
+ }
4372
+ /**
4373
+ * Refresh the button's text in place while the instance keeps working.
4374
+ *
4375
+ * Uses `editAlert`, NOT `editMessage`: on Telegram the latter omits reply_markup,
4376
+ * and the Bot API treats that as "clear the keyboard" — so editing with it would
4377
+ * delete the very cancel button this is trying to keep alive.
4378
+ */
4379
+ /** Configured threshold before elapsed time appears, in ms. */
4380
+ progressMinElapsedMs() {
4381
+ const seconds = this.fleetConfig?.defaults
4382
+ ?.progress_min_elapsed;
4383
+ if (typeof seconds === "number" && Number.isFinite(seconds) && seconds >= 0) {
4384
+ return seconds * 1000;
4385
+ }
4386
+ return PROGRESS_MIN_ELAPSED_MS;
4387
+ }
4388
+ startProgressTicker(entry) {
4389
+ const tick = () => {
4390
+ if (!this.cancelButtons.has(entry.messageId)) {
4391
+ clearInterval(entry.progressTimer);
4392
+ return;
4393
+ }
4394
+ const text = FleetManager.progressText(Date.now() - (entry.startedAt ?? Date.now()), this.instanceActivity.get(entry.instanceName), this.progressMinElapsedMs());
4395
+ if (text === entry.lastProgressText)
4396
+ return; // nothing changed — skip the API call
4397
+ const adapter = this.getAdapterForInstance(entry.instanceName) ?? this.adapter;
4398
+ if (!adapter?.editAlert)
4399
+ return;
4400
+ entry.lastProgressText = text;
4401
+ adapter.editAlert(entry.chatId, entry.messageId, {
4402
+ type: "cancel",
4403
+ instanceName: entry.instanceName,
4404
+ message: text,
4405
+ choices: [{ id: `cancel:${entry.instanceName}`, label: t("cancel.button") }],
4406
+ }, entry.threadId ? { threadId: entry.threadId } : undefined)
4407
+ .catch(err => {
4408
+ // A failed progress edit must never escalate: the button still works and
4409
+ // the next tick retries. Common causes are a deleted message or a
4410
+ // rate limit.
4411
+ this.logger.debug({ err, instanceName: entry.instanceName }, "Progress edit failed");
4412
+ });
4413
+ };
4414
+ entry.progressTimer = setInterval(tick, PROGRESS_UPDATE_INTERVAL_MS);
4415
+ entry.progressTimer.unref?.();
4416
+ // One extra tick right when the threshold passes, so a 30s threshold shows
4417
+ // time at ~30s instead of waiting for the first 60s interval. Costs at most
4418
+ // one additional edit per turn that lives past the threshold.
4419
+ const firstAt = this.progressMinElapsedMs() - (Date.now() - (entry.startedAt ?? Date.now()));
4420
+ if (firstAt > 0 && firstAt < PROGRESS_UPDATE_INTERVAL_MS) {
4421
+ const firstTick = setTimeout(tick, firstAt);
4422
+ firstTick.unref?.();
4423
+ }
4424
+ }
4425
+ /**
4426
+ * After a reply: give the instance REPLY_RETIRE_GRACE_MS to resume working; if
4427
+ * it has not, retire its button. Re-arming replaces the previous timer, so a
4428
+ * burst of replies ends with exactly one pending check.
4429
+ */
4430
+ armReplyGrace(instanceName) {
4431
+ for (const entry of this.cancelButtons.values()) {
4432
+ if (entry.instanceName !== instanceName)
4433
+ continue;
4434
+ if (entry.replyGraceTimer)
4435
+ clearTimeout(entry.replyGraceTimer);
4436
+ entry.replyGraceTimer = setTimeout(() => {
4437
+ entry.replyGraceTimer = undefined;
4438
+ if (!this.cancelButtons.has(entry.messageId))
4439
+ return;
4440
+ if (!this.getInstanceIdle(instanceName))
4441
+ return; // resumed — a long run keeps its button
4442
+ this.logger.info({ instanceName, messageId: entry.messageId }, "Cancel button retired — no work resumed after reply");
4443
+ this.retireButton(entry);
4444
+ }, REPLY_RETIRE_GRACE_MS);
4445
+ entry.replyGraceTimer.unref?.();
4446
+ }
4447
+ }
3597
4448
  /** Retire (delete) every cancel button belonging to an instance. */
3598
4449
  retireInstanceButtons(instanceName) {
3599
4450
  // Snapshot first — retireButton may delete entries from the map on success.
@@ -3625,7 +4476,70 @@ export class FleetManager {
3625
4476
  clearTimeout(entry.retryTimer);
3626
4477
  if (entry.idleCheckTimer)
3627
4478
  clearInterval(entry.idleCheckTimer);
4479
+ if (entry.progressTimer)
4480
+ clearInterval(entry.progressTimer);
4481
+ if (entry.replyGraceTimer)
4482
+ clearTimeout(entry.replyGraceTimer);
3628
4483
  this.cancelButtons.delete(entry.messageId);
4484
+ this.persistCancelButtons();
4485
+ }
4486
+ /**
4487
+ * Mirror the live buttons to disk. The map is memory-only, so before this a
4488
+ * fleet restart orphaned every button on screen: frozen "處理中…" text and a
4489
+ * click that did nothing, forever. The ledger is tiny (a handful of rows) and
4490
+ * written on every add/remove — no debounce needed at that rate.
4491
+ */
4492
+ persistCancelButtons() {
4493
+ try {
4494
+ const rows = [...this.cancelButtons.values()].map(e => ({
4495
+ instanceName: e.instanceName,
4496
+ adapterId: e.adapterId,
4497
+ chatId: e.chatId,
4498
+ messageId: e.messageId,
4499
+ threadId: e.threadId,
4500
+ }));
4501
+ writeFileSync(join(this.dataDir, CANCEL_BTN_LEDGER_FILE), JSON.stringify(rows));
4502
+ }
4503
+ catch (err) {
4504
+ this.logger.debug({ err }, "Cancel button ledger write failed");
4505
+ }
4506
+ }
4507
+ /**
4508
+ * Delete the previous process's buttons. Runs once adapters are up: nothing
4509
+ * from a previous fleet process can still be mid-turn from this process's
4510
+ * point of view, so every ledger row is an orphan by definition.
4511
+ */
4512
+ async sweepOrphanedCancelButtons() {
4513
+ const ledgerPath = join(this.dataDir, CANCEL_BTN_LEDGER_FILE);
4514
+ let rows;
4515
+ try {
4516
+ if (!existsSync(ledgerPath))
4517
+ return;
4518
+ rows = JSON.parse(readFileSync(ledgerPath, "utf-8"));
4519
+ }
4520
+ catch {
4521
+ try {
4522
+ unlinkSync(ledgerPath);
4523
+ }
4524
+ catch { /* corrupt ledger — drop it */ }
4525
+ return;
4526
+ }
4527
+ for (const row of rows) {
4528
+ const adapter = (row.adapterId ? this.worlds.get(row.adapterId)?.adapter : undefined)
4529
+ ?? this.getAdapterForInstance?.(row.instanceName) ?? this.adapter;
4530
+ if (!adapter?.deleteMessage)
4531
+ continue;
4532
+ try {
4533
+ await adapter.deleteMessage(row.chatId, row.messageId, row.threadId);
4534
+ this.logger.info({ instanceName: row.instanceName, messageId: row.messageId }, "Swept orphaned cancel button from previous run");
4535
+ }
4536
+ catch (err) {
4537
+ // Best effort: the message may already be gone, or too old to delete.
4538
+ this.logger.debug({ err, messageId: row.messageId }, "Orphaned cancel button sweep failed");
4539
+ }
4540
+ }
4541
+ // The current process owns the ledger from here on.
4542
+ this.persistCancelButtons();
3629
4543
  }
3630
4544
  /** Re-attempt a failed button delete up to CANCEL_BTN_MAX_RETRIES times. */
3631
4545
  scheduleButtonRetry(entry, err) {
@@ -4290,6 +5204,12 @@ When users create specialized instances, suggest these configurations:
4290
5204
  meta.image_path = saves[saves.length - 1][1].split(",")[0].trim();
4291
5205
  }
4292
5206
  }
5207
+ // Classic channels queue reactions like everyone else (#432 stored them, but
5208
+ // this path never attached them — reactions in a ClassicBot channel went into
5209
+ // the DB and were never seen again). Same contract as the topic paths:
5210
+ // consumed only after the delivery succeeded.
5211
+ const reactions = this.pendingReactionsMeta(instanceName);
5212
+ Object.assign(meta, reactions.meta);
4293
5213
  try {
4294
5214
  await this.deliverToInstance(instanceName, {
4295
5215
  type: "fleet_inbound",
@@ -4297,6 +5217,7 @@ When users create specialized instances, suggest these configurations:
4297
5217
  targetSession: instanceName,
4298
5218
  meta,
4299
5219
  });
5220
+ reactions.consume();
4300
5221
  }
4301
5222
  catch (err) {
4302
5223
  this.logger.warn({ err: err.message, instanceName }, "Classic wake/delivery failed");
@@ -4435,6 +5356,170 @@ When users create specialized instances, suggest these configurations:
4435
5356
  return env?.models ?? [];
4436
5357
  }
4437
5358
  /** `/model` slash handler (admin only). No arg → DC menu; `/model <name>` → apply directly. */
5359
+ /** Label an effort choice, marking the one currently configured. */
5360
+ effortChoiceLabel(level, current) {
5361
+ return level === current ? `✓ ${level}` : level;
5362
+ }
5363
+ effortMenuHeader(instanceName) {
5364
+ const { effort, source } = this.resolveInstanceEffort(instanceName);
5365
+ if (!effort)
5366
+ return "Current effort: (CLI default)";
5367
+ return source === "fleet-default"
5368
+ ? `Current effort: ${effort} (fleet default)`
5369
+ : `Current effort: ${effort}`;
5370
+ }
5371
+ /** `/effort` — DC Select Menu, or apply directly when a level is given. */
5372
+ async handleEffortSlash(data, adapterId) {
5373
+ if (!this.isModelAdmin(data.userId, data.channelId, adapterId)) {
5374
+ await data.respond(t("permission.denied"));
5375
+ return;
5376
+ }
5377
+ const name = this.resolveSlashTarget(data.channelId, adapterId);
5378
+ if (!name) {
5379
+ await data.respond(t("classic.no_agent"));
5380
+ return;
5381
+ }
5382
+ const requested = (typeof data.options?.level === "string" ? data.options.level.trim() : "")
5383
+ || (data.text?.trim() ?? "");
5384
+ if (requested) {
5385
+ await data.respond(await this.applyEffort(name, requested));
5386
+ return;
5387
+ }
5388
+ const levels = this.effortLevelsFor(name);
5389
+ if (levels.length === 0) {
5390
+ await data.respond(`❌ ${this.backendNameForInstance(name)} has no reasoning-effort setting.`);
5391
+ return;
5392
+ }
5393
+ if (!data.respondChoices) {
5394
+ await data.respond(`Usage: /effort <${levels.join("|")}>`);
5395
+ return;
5396
+ }
5397
+ const current = this.resolveInstanceEffort(name).effort;
5398
+ const nonce = randomBytes(6).toString("hex");
5399
+ const choices = levels.map(l => ({
5400
+ id: `${EFFORT_SELECT_CALLBACK_PREFIX}${nonce}:${l}`,
5401
+ label: this.effortChoiceLabel(l, current),
5402
+ }));
5403
+ const timer = setTimeout(() => this.pendingEffortSelects.delete(nonce), CLASSIC_BACKEND_SELECTION_TIMEOUT_MS);
5404
+ timer.unref?.();
5405
+ this.pendingEffortSelects.set(nonce, { instanceName: name, userId: data.userId, channelId: data.channelId, timer, respond: data.respond });
5406
+ try {
5407
+ await data.respondChoices(`${this.effortMenuHeader(name)}\nSelect a new effort level:`, choices);
5408
+ }
5409
+ catch (err) {
5410
+ this.pendingEffortSelects.delete(nonce);
5411
+ clearTimeout(timer);
5412
+ this.logger.warn({ err, instanceName: name }, "effort menu failed");
5413
+ await data.respond(`Usage: /effort <${levels.join("|")}>`);
5414
+ }
5415
+ }
5416
+ /** TG inline-keyboard effort menu. Returns null on success, else a fallback string. */
5417
+ async promptEffortMenu(instanceName, userId, channelId, adapter, chatId, threadId) {
5418
+ const levels = this.effortLevelsFor(instanceName);
5419
+ if (levels.length === 0) {
5420
+ return `❌ ${this.backendNameForInstance(instanceName)} has no reasoning-effort setting.`;
5421
+ }
5422
+ const current = this.resolveInstanceEffort(instanceName).effort;
5423
+ const nonce = randomBytes(6).toString("hex");
5424
+ const choices = levels.map(l => ({
5425
+ id: `${EFFORT_SELECT_CALLBACK_PREFIX}${nonce}:${l}`,
5426
+ label: this.effortChoiceLabel(l, current),
5427
+ }));
5428
+ const respond = async (text) => {
5429
+ await adapter.sendText(chatId, text, { threadId });
5430
+ return undefined;
5431
+ };
5432
+ const timer = setTimeout(() => {
5433
+ const p = this.pendingEffortSelects.get(nonce);
5434
+ if (p) {
5435
+ this.pendingEffortSelects.delete(nonce);
5436
+ p.respond("⏰ Effort selection expired.").catch(() => { });
5437
+ }
5438
+ }, CLASSIC_BACKEND_SELECTION_TIMEOUT_MS);
5439
+ timer.unref?.();
5440
+ this.pendingEffortSelects.set(nonce, { instanceName, userId, channelId, timer, respond, adapter, adapterChatId: chatId, adapterThreadId: threadId });
5441
+ try {
5442
+ const menuMessageId = await adapter.promptUser(chatId, `${this.effortMenuHeader(instanceName)}\nSelect a new effort level:`, choices, { threadId });
5443
+ const pending = this.pendingEffortSelects.get(nonce);
5444
+ if (pending)
5445
+ pending.menuMessageId = menuMessageId;
5446
+ return null;
5447
+ }
5448
+ catch (err) {
5449
+ this.pendingEffortSelects.delete(nonce);
5450
+ clearTimeout(timer);
5451
+ this.logger.warn({ err, instanceName }, "TG effort menu failed");
5452
+ return `Usage: /effort <${levels.join("|")}>`;
5453
+ }
5454
+ }
5455
+ /** Consume an `/effort` selection callback. Mirrors handleModelSelection. */
5456
+ async handleEffortSelection(data) {
5457
+ if (!data.callbackData.startsWith(EFFORT_SELECT_CALLBACK_PREFIX))
5458
+ return false;
5459
+ const match = data.callbackData.match(/^effort-select:([0-9a-f]+):(.+)$/);
5460
+ if (!match)
5461
+ return true;
5462
+ const pending = this.pendingEffortSelects.get(match[1]);
5463
+ if (!pending)
5464
+ return true;
5465
+ if (data.userId && data.userId !== pending.userId)
5466
+ return true;
5467
+ const cbChannel = data.threadId ?? data.chatId;
5468
+ if (cbChannel !== pending.channelId && data.chatId !== pending.channelId)
5469
+ return true;
5470
+ this.pendingEffortSelects.delete(match[1]);
5471
+ clearTimeout(pending.timer);
5472
+ const level = match[2];
5473
+ const progressText = `⏳ Setting ${pending.instanceName} effort to \`${level}\`…`;
5474
+ let progressMsgId;
5475
+ if (pending.adapter && pending.adapterChatId) {
5476
+ const menuMessageId = pending.menuMessageId ?? data.messageId;
5477
+ if (menuMessageId && pending.adapter.editMessageRemoveButtons) {
5478
+ try {
5479
+ await pending.adapter.editMessageRemoveButtons(pending.adapterChatId, menuMessageId, progressText, pending.adapterThreadId);
5480
+ progressMsgId = menuMessageId;
5481
+ }
5482
+ catch { /* fall back to a new message */ }
5483
+ }
5484
+ if (!progressMsgId) {
5485
+ try {
5486
+ const sent = await pending.adapter.sendText(pending.adapterChatId, progressText, { threadId: pending.adapterThreadId });
5487
+ progressMsgId = sent.messageId;
5488
+ }
5489
+ catch { /* non-fatal */ }
5490
+ }
5491
+ }
5492
+ else {
5493
+ await pending.respond(progressText).catch(() => { });
5494
+ }
5495
+ // Background-applied and guarded for the same reason as the model path: a
5496
+ // restart backend respawns the instance here, and an unguarded rejection
5497
+ // from a menu click must not take the fleet down.
5498
+ void (async () => {
5499
+ let result;
5500
+ try {
5501
+ result = await this.applyEffort(pending.instanceName, level);
5502
+ }
5503
+ catch (err) {
5504
+ this.logger.error({ err, instance: pending.instanceName, level }, "Effort switch failed");
5505
+ result = `Effort switch to \`${level}\` failed: ${err instanceof Error ? err.message : String(err)}`;
5506
+ }
5507
+ if (pending.adapter && pending.adapterChatId) {
5508
+ if (progressMsgId) {
5509
+ pending.adapter.editMessage(pending.adapterChatId, progressMsgId, result, pending.adapterThreadId).catch(() => {
5510
+ pending.adapter.sendText(pending.adapterChatId, result, { threadId: pending.adapterThreadId }).catch(() => { });
5511
+ });
5512
+ }
5513
+ else {
5514
+ pending.adapter.sendText(pending.adapterChatId, result, { threadId: pending.adapterThreadId }).catch(() => { });
5515
+ }
5516
+ }
5517
+ else {
5518
+ await pending.respond(result).catch(() => { });
5519
+ }
5520
+ })();
5521
+ return true;
5522
+ }
4438
5523
  async handleModelSlash(data, adapterId) {
4439
5524
  if (!this.isModelAdmin(data.userId, data.channelId, adapterId)) {
4440
5525
  await data.respond(t("permission.denied"));
@@ -4571,9 +5656,19 @@ When users create specialized instances, suggest these configurations:
4571
5656
  // DC path: respond immediately with progress text
4572
5657
  await pending.respond(progressText).catch(() => { });
4573
5658
  }
4574
- // Apply model in background — don't await here (keeps callback handler fast)
5659
+ // Apply model in background — don't await here (keeps callback handler fast).
5660
+ // Guarded: applyModel() restarts the instance, and an unguarded rejection here
5661
+ // meant a user picking from the /model menu could take the whole fleet down.
5662
+ // On failure the user gets told, rather than the click silently doing nothing.
4575
5663
  void (async () => {
4576
- const result = await this.applyModel(pending.instanceName, model);
5664
+ let result;
5665
+ try {
5666
+ result = await this.applyModel(pending.instanceName, model);
5667
+ }
5668
+ catch (err) {
5669
+ this.logger.error({ err, instance: pending.instanceName, model }, "Model switch failed");
5670
+ result = `Model switch to \`${model}\` failed: ${err instanceof Error ? err.message : String(err)}`;
5671
+ }
4577
5672
  if (pending.adapter && pending.adapterChatId) {
4578
5673
  if (progressMsgId) {
4579
5674
  pending.adapter.editMessage(pending.adapterChatId, progressMsgId, result, pending.adapterThreadId).catch(() => {
@@ -4591,6 +5686,110 @@ When users create specialized instances, suggest these configurations:
4591
5686
  return true;
4592
5687
  }
4593
5688
  /** Apply a model to an instance: runtime paste (claude-code) or persist + restart (others). */
5689
+ /** AgEnD's canonical effort ladder, low → max. Backends expose a subset. */
5690
+ static EFFORT_LEVELS = ["low", "medium", "high", "xhigh", "max"];
5691
+ /** How this instance's backend applies an effort change. */
5692
+ effortStrategyFor(instanceName) {
5693
+ try {
5694
+ const backend = createBackend(this.backendNameForInstance(instanceName), this.getInstanceDir(instanceName));
5695
+ const strategy = backend.getEffortStrategy?.() ?? "unsupported";
5696
+ // A backend claiming support but listing no levels is unusable either way.
5697
+ return strategy !== "unsupported" && (backend.getEffortLevels?.() ?? []).length > 0
5698
+ ? strategy
5699
+ : "unsupported";
5700
+ }
5701
+ catch {
5702
+ return "unsupported";
5703
+ }
5704
+ }
5705
+ /** Effort levels this instance's backend actually accepts (empty = unsupported). */
5706
+ effortLevelsFor(instanceName) {
5707
+ try {
5708
+ const backend = createBackend(this.backendNameForInstance(instanceName), this.getInstanceDir(instanceName));
5709
+ if ((backend.getEffortStrategy?.() ?? "unsupported") === "unsupported")
5710
+ return [];
5711
+ return backend.getEffortLevels?.() ?? [];
5712
+ }
5713
+ catch {
5714
+ return [];
5715
+ }
5716
+ }
5717
+ /** Configured effort for an instance: per-instance, else fleet default, else none. */
5718
+ resolveInstanceEffort(instanceName) {
5719
+ const own = this.fleetConfig?.instances[instanceName]?.effort;
5720
+ if (own)
5721
+ return { effort: own, source: "instance" };
5722
+ const fallback = this.fleetConfig?.defaults?.effort;
5723
+ if (fallback)
5724
+ return { effort: fallback, source: "fleet-default" };
5725
+ return { effort: null, source: "unset" };
5726
+ }
5727
+ /**
5728
+ * Clamp a canonical level to the nearest one this backend supports.
5729
+ *
5730
+ * Clamping DOWN the ladder, never up: asking for `max` on a CLI that stops at
5731
+ * `high` should get high, not silently fall to low. The caller reports the
5732
+ * clamp — a user who asks for max and quietly receives high has been told the
5733
+ * request succeeded when it did not.
5734
+ */
5735
+ static clampEffort(level, supported) {
5736
+ if (supported.includes(level))
5737
+ return level;
5738
+ const ladder = FleetManager.EFFORT_LEVELS;
5739
+ const wanted = ladder.indexOf(level);
5740
+ if (wanted < 0)
5741
+ return null;
5742
+ for (let i = wanted - 1; i >= 0; i--) {
5743
+ if (supported.includes(ladder[i]))
5744
+ return ladder[i];
5745
+ }
5746
+ return supported[0] ?? null;
5747
+ }
5748
+ /**
5749
+ * Apply a reasoning-effort level, mirroring applyModel's shape.
5750
+ *
5751
+ * runtime backends take `/effort <level>` in the pane and keep working;
5752
+ * restart backends only read it at launch, so it is persisted and the
5753
+ * instance respawns.
5754
+ */
5755
+ async applyEffort(instanceName, requested) {
5756
+ const level = requested.trim().toLowerCase();
5757
+ const backendName = this.backendNameForInstance(instanceName);
5758
+ let strategy = "unsupported";
5759
+ let supported = [];
5760
+ try {
5761
+ const backend = createBackend(backendName, this.getInstanceDir(instanceName));
5762
+ strategy = backend.getEffortStrategy?.() ?? "unsupported";
5763
+ supported = backend.getEffortLevels?.() ?? [];
5764
+ }
5765
+ catch { /* treated as unsupported below */ }
5766
+ if (strategy === "unsupported" || supported.length === 0) {
5767
+ return `❌ ${backendName} has no reasoning-effort setting.`;
5768
+ }
5769
+ if (!FleetManager.EFFORT_LEVELS.includes(level)) {
5770
+ return `❌ Unknown effort level \`${level}\`. Use: ${FleetManager.EFFORT_LEVELS.join(", ")}.`;
5771
+ }
5772
+ const applied = FleetManager.clampEffort(level, supported);
5773
+ if (!applied)
5774
+ return `❌ ${backendName} accepts none of the canonical effort levels.`;
5775
+ const warn = applied === level
5776
+ ? ""
5777
+ : `⚠️ Clamped to \`${applied}\` (\`${level}\` not supported by ${backendName}).\n`;
5778
+ // Persist either way: a runtime switch must survive the next respawn too,
5779
+ // or the instance silently reverts on restart.
5780
+ if (this.fleetConfig?.instances[instanceName]) {
5781
+ this.fleetConfig.instances[instanceName].effort = applied;
5782
+ this.saveFleetConfig();
5783
+ }
5784
+ if (strategy === "runtime") {
5785
+ if (!this.instanceIpcClients.get(instanceName))
5786
+ return `${warn}❌ ${instanceName} is not running.`;
5787
+ this.pasteRawToClassicInstance(instanceName, `/effort ${applied}`);
5788
+ return `${warn}✅ Set ${instanceName} effort to \`${applied}\` (runtime).`;
5789
+ }
5790
+ await this.restartSingleInstance(instanceName);
5791
+ return `${warn}✅ Set ${instanceName} effort to \`${applied}\` and restarted.`;
5792
+ }
4594
5793
  async applyModel(instanceName, model) {
4595
5794
  const backendName = this.backendNameForInstance(instanceName);
4596
5795
  let strategy = "restart";
@@ -4603,7 +5802,7 @@ When users create specialized instances, suggest these configurations:
4603
5802
  if (!this.instanceIpcClients.get(instanceName))
4604
5803
  return `${warn}❌ ${instanceName} is not running.`;
4605
5804
  this.pasteRawToClassicInstance(instanceName, `/model ${model}`);
4606
- return `${warn}✅ Switched ${instanceName} to \`${model}\` (runtime).`;
5805
+ return `${warn}✅ Switched ${instanceName} to \`${model}\` (runtime).${this.effortSuffix(instanceName)}`;
4607
5806
  }
4608
5807
  // restart: persist the model so the respawned CLI launches with it.
4609
5808
  let persisted = false;
@@ -4618,7 +5817,24 @@ When users create specialized instances, suggest these configurations:
4618
5817
  if (!persisted)
4619
5818
  return `${warn}❌ Could not set model for ${instanceName}.`;
4620
5819
  await this.restartSingleInstance(instanceName);
4621
- return `${warn}✅ Set ${instanceName} to \`${model}\` and restarted.`;
5820
+ return `${warn}✅ Set ${instanceName} to \`${model}\` and restarted.${this.effortSuffix(instanceName)}`;
5821
+ }
5822
+ /**
5823
+ * The trailing "Current effort: …" line for a /model reply.
5824
+ *
5825
+ * Model and effort interact (a cheaper model at max effort is a different
5826
+ * trade than a bigger one at low), so showing the effort in force right after
5827
+ * a switch saves the round trip of asking. Empty when the backend has none.
5828
+ */
5829
+ effortSuffix(instanceName) {
5830
+ if (this.effortLevelsFor(instanceName).length === 0)
5831
+ return "";
5832
+ const { effort, source } = this.resolveInstanceEffort(instanceName);
5833
+ if (!effort)
5834
+ return "\nCurrent effort: (CLI default)";
5835
+ return source === "fleet-default"
5836
+ ? `\nCurrent effort: ${effort} (fleet default)`
5837
+ : `\nCurrent effort: ${effort}`;
4622
5838
  }
4623
5839
  /** Read recent chat log for agent context */
4624
5840
  getRecentChatLog(instanceName, maxLines = 10) {
@@ -4811,7 +6027,7 @@ When users create specialized instances, suggest these configurations:
4811
6027
  ...(preTaskCommand ? { pre_task_command: preTaskCommand } : {}),
4812
6028
  };
4813
6029
  const topicMode = this.fleetConfig?.channel?.mode === "topic";
4814
- await this.startInstance(instanceName, config, topicMode);
6030
+ await this.startInstance(instanceName, config, topicMode, "classic");
4815
6031
  }
4816
6032
  /** Handle /start slash command — register classic channel */
4817
6033
  async handleClassicStart(channelId, channelName, userId, guildId, adapterId, backend) {
@@ -4853,11 +6069,28 @@ When users create specialized instances, suggest these configurations:
4853
6069
  this.logger.info({ channelId, adapterId, instanceName: ch.instanceName }, "Classic channel stopped");
4854
6070
  return t("classic.stopped");
4855
6071
  }
4856
- async stopAll() {
6072
+ /**
6073
+ * Idempotent while in flight: SIGINT and SIGTERM share one handler and the
6074
+ * uncaughtException path calls this too, so overlapping runs were possible —
6075
+ * each snapshotting the daemon map and calling stop() on the same daemons
6076
+ * concurrently. Deliberately NOT `async`, so callers receive the same promise
6077
+ * object rather than a fresh wrapper around it. The latch clears when the run
6078
+ * settles, so a later genuine stop (after a restart) still does the work.
6079
+ */
6080
+ stopAll() {
6081
+ this.stopAllInFlight ??= this.doStopAll().finally(() => { this.stopAllInFlight = null; });
6082
+ return this.stopAllInFlight;
6083
+ }
6084
+ stopAllInFlight = null;
6085
+ async doStopAll() {
4857
6086
  this.startupComplete = false;
4858
6087
  this.reloadPending = false;
6088
+ // Before anything is stopped: everything that dies from here on dies
6089
+ // because we asked it to. Set synchronously — doStopAll runs to its first
6090
+ // await in the same tick as the signal handler, so no event can slip in.
6091
+ this.shuttingDown = true;
4859
6092
  this.ipcStoppingInstances.add("__fleet_stopping__");
4860
- sdNotify("STOPPING=1");
6093
+ sdNotifyBlocking("STOPPING=1");
4861
6094
  if (this.watchdogTimer) {
4862
6095
  clearInterval(this.watchdogTimer);
4863
6096
  this.watchdogTimer = null;
@@ -4877,6 +6110,29 @@ When users create specialized instances, suggest these configurations:
4877
6110
  clearInterval(this.updateCheckTimer);
4878
6111
  this.updateCheckTimer = null;
4879
6112
  }
6113
+ if (this.eventLogPruneTimer) {
6114
+ clearInterval(this.eventLogPruneTimer);
6115
+ this.eventLogPruneTimer = null;
6116
+ }
6117
+ if (this.logRotateTimer) {
6118
+ clearInterval(this.logRotateTimer);
6119
+ this.logRotateTimer = null;
6120
+ }
6121
+ // Cancel-button timers were never cleared here. The idle-check interval is not
6122
+ // unref'd, so it held the event loop open past shutdown and kept retrying
6123
+ // deletes against an adapter that was already gone.
6124
+ for (const entry of [...this.cancelButtons.values()]) {
6125
+ if (entry.retryTimer)
6126
+ clearTimeout(entry.retryTimer);
6127
+ if (entry.idleCheckTimer)
6128
+ clearInterval(entry.idleCheckTimer);
6129
+ if (entry.progressTimer)
6130
+ clearInterval(entry.progressTimer);
6131
+ }
6132
+ this.cancelButtons.clear();
6133
+ for (const timer of this.cancelButtonIdleRetireTimers.values())
6134
+ clearTimeout(timer);
6135
+ this.cancelButtonIdleRetireTimers.clear();
4880
6136
  if (this.topicCleanupTimer) {
4881
6137
  clearInterval(this.topicCleanupTimer);
4882
6138
  this.topicCleanupTimer = null;
@@ -4944,6 +6200,9 @@ When users create specialized instances, suggest these configurations:
4944
6200
  catch (e) {
4945
6201
  this.logger.debug({ err: e }, "Failed to remove fleet PID file");
4946
6202
  }
6203
+ // The lock contains a nonce, so an older/shutting-down process can never
6204
+ // remove a lock acquired by a newer fleet owner.
6205
+ releaseProcessFleetLock();
4947
6206
  }
4948
6207
  /**
4949
6208
  * Prune stale external sessions by re-querying each daemon for live sessions.
@@ -5095,6 +6354,15 @@ When users create specialized instances, suggest these configurations:
5095
6354
  removedRatio,
5096
6355
  validationErrors: validation.errors,
5097
6356
  }, "Refusing unsafe fleet config reload; running configuration was kept");
6357
+ // Tell the operator. A silently ignored config edit is the most confusing
6358
+ // possible outcome: they change fleet.yaml, send SIGHUP, and nothing happens
6359
+ // with no explanation anywhere they are looking.
6360
+ const why = !validation.valid
6361
+ ? `validation failed:\n${validation.errors.map(e => `• ${e.path}: ${e.message}`).join("\n")}`
6362
+ : unsafeEmpty
6363
+ ? `it removed every instance (${oldCount} → 0)`
6364
+ : `it removed more than half the instances (${oldCount} → ${newCount})`;
6365
+ this.notifyFleetError(`⚠️ fleet.yaml reload REJECTED — ${why}\nThe previous configuration is still running.`);
5098
6366
  return;
5099
6367
  }
5100
6368
  this.routing.rebuild(this.fleetConfig);
@@ -5294,10 +6562,20 @@ When users create specialized instances, suggest these configurations:
5294
6562
  // ── Update check ────────────────────────────────────────────────────
5295
6563
  async checkForUpdates() {
5296
6564
  try {
5297
- const { execSync } = await import("node:child_process");
6565
+ // Both npm lookups are async: as execSync they froze the fleet event loop for
6566
+ // up to 15s each, and on a beta build BOTH ran — 30s with no WATCHDOG ping,
6567
+ // past WatchdogSec's half-interval and enough for systemd to SIGABRT the fleet
6568
+ // for a background version check.
6569
+ const { execFile } = await import("node:child_process");
6570
+ const { promisify } = await import("node:util");
6571
+ const execFileP = promisify(execFile);
6572
+ const npmVersion = async (spec) => {
6573
+ const { stdout } = await execFileP("npm", ["view", spec, "version"], { timeout: 15_000 });
6574
+ return stdout.toString().trim();
6575
+ };
5298
6576
  const pkgPath = join(dirname(fileURLToPath(import.meta.url)), "..", "package.json");
5299
6577
  const currentVersion = JSON.parse(readFileSync(pkgPath, "utf-8")).version ?? "0.0.0";
5300
- const latest = execSync("npm view @songsid/agend version", { stdio: "pipe", timeout: 15_000 }).toString().trim();
6578
+ const latest = await npmVersion("@songsid/agend");
5301
6579
  let target = latest;
5302
6580
  if (currentVersion.includes("-beta")) {
5303
6581
  // Beta users track the @beta channel (never fall back to @latest, which is
@@ -5305,7 +6583,7 @@ When users create specialized instances, suggest these configurations:
5305
6583
  // of beta/latest is the newest.
5306
6584
  let beta = "";
5307
6585
  try {
5308
- beta = execSync("npm view @songsid/agend@beta version", { stdio: "pipe", timeout: 15_000 }).toString().trim();
6586
+ beta = await npmVersion("@songsid/agend@beta");
5309
6587
  }
5310
6588
  catch { /* no beta tag */ }
5311
6589
  target = beta || latest;
@@ -5412,6 +6690,10 @@ When users create specialized instances, suggest these configurations:
5412
6690
  // /view routes accept the read-only view.token (or web.token) and do
5413
6691
  // their own per-method auth in view-api.ts — skip the web-token gate.
5414
6692
  }
6693
+ else if (isUsagePath(new URL(req.url ?? "/", `http://localhost:${port}`).pathname)) {
6694
+ // /api/ai-usage is read-only GET data for the /view Usage panel — open
6695
+ // like the other /view data routes (usage-api.ts rejects non-GET).
6696
+ }
5415
6697
  else {
5416
6698
  // All other endpoints require a valid token (query ?token= or X-Agend-Token header).
5417
6699
  // /ui/* will also re-check in web-api.ts, which is harmless.
@@ -5426,34 +6708,34 @@ When users create specialized instances, suggest these configurations:
5426
6708
  }
5427
6709
  }
5428
6710
  if (req.method === "GET" && req.url === "/health") {
5429
- const instanceCount = this.fleetConfig?.instances
5430
- ? Object.keys(this.fleetConfig.instances).length
5431
- : 0;
5432
- res.writeHead(200);
5433
- res.end(JSON.stringify({
5434
- status: "ok",
5435
- instances: instanceCount,
5436
- uptime: Math.floor((Date.now() - this.startedAt) / 1000),
5437
- }));
6711
+ const health = this.getFleetHealth();
6712
+ // 503 when the fleet cannot do its job, so an external monitor sees it.
6713
+ // This used to always answer 200 "ok" with a count of CONFIGURED instances,
6714
+ // so every agent could be dead and every adapter down and it still looked
6715
+ // green.
6716
+ res.writeHead(health.status === "ok" ? 200 : 503);
6717
+ res.end(JSON.stringify(health));
5438
6718
  return;
5439
6719
  }
5440
6720
  if (req.method === "GET" && req.url === "/status") {
5441
6721
  const instances = Object.keys(this.fleetConfig?.instances ?? {}).map(name => {
5442
6722
  const statusFile = join(this.getInstanceDir(name), "statusline.json");
5443
- let context_pct = 0;
5444
6723
  let cost = 0;
5445
6724
  try {
5446
6725
  const data = JSON.parse(readFileSync(statusFile, "utf-8"));
5447
- context_pct = data.context_window?.used_percentage ?? 0;
5448
6726
  cost = data.cost?.total_cost_usd ?? 0;
5449
6727
  }
5450
6728
  catch (err) {
5451
6729
  this.logger.debug({ err, name }, "statusline.json read failed (/status)");
5452
6730
  }
6731
+ const backend = this.fleetConfig?.instances[name]?.backend
6732
+ ?? this.fleetConfig?.defaults?.backend
6733
+ ?? "claude-code";
6734
+ const { context } = resolveInstanceContext(this.dataDir, name, backend);
5453
6735
  return {
5454
6736
  name,
5455
6737
  status: this.getInstanceStatus(name),
5456
- context_pct,
6738
+ context_pct: context ?? 0,
5457
6739
  cost,
5458
6740
  };
5459
6741
  });
@@ -5568,7 +6850,10 @@ When users create specialized instances, suggest these configurations:
5568
6850
  res.writeHead(500);
5569
6851
  res.end(JSON.stringify({ error: `Start failed: ${err.message}` }));
5570
6852
  }
5571
- })();
6853
+ // The inner catch can itself throw (writeHead after a successful
6854
+ // writeHead is ERR_HTTP_HEADERS_SENT), and that rejection escapes the
6855
+ // IIFE. Same for the two handlers below.
6856
+ })().catch(err => this.logger.error({ err, name }, "HTTP start handler failed"));
5572
6857
  return;
5573
6858
  }
5574
6859
  // Instance restart (immediate, no idle wait)
@@ -5589,7 +6874,7 @@ When users create specialized instances, suggest these configurations:
5589
6874
  res.writeHead(status);
5590
6875
  res.end(JSON.stringify({ error: `Restart failed: ${err.message}` }));
5591
6876
  }
5592
- })();
6877
+ })().catch(err => this.logger.error({ err, name }, "HTTP restart handler failed"));
5593
6878
  return;
5594
6879
  }
5595
6880
  if (req.method === "POST" && req.url?.startsWith("/stop/")) {
@@ -5612,7 +6897,7 @@ When users create specialized instances, suggest these configurations:
5612
6897
  res.writeHead(500);
5613
6898
  res.end(JSON.stringify({ error: `Stop failed: ${err.message}` }));
5614
6899
  }
5615
- })();
6900
+ })().catch(err => this.logger.error({ err, name }, "HTTP stop handler failed"));
5616
6901
  return;
5617
6902
  }
5618
6903
  // ── Agent CLI endpoint ─────
@@ -5624,6 +6909,8 @@ When users create specialized instances, suggest these configurations:
5624
6909
  const url = new URL(req.url ?? "/", `http://localhost:${port}`);
5625
6910
  if (handleViewRequest(req, res, url, this))
5626
6911
  return;
6912
+ if (handleUsageRequest(req, res, url, this))
6913
+ return;
5627
6914
  if (handleSettingsRequest(req, res, url, this))
5628
6915
  return;
5629
6916
  if (handleWebRequest(req, res, url, this))
@@ -5670,20 +6957,35 @@ When users create specialized instances, suggest these configurations:
5670
6957
  this.logger.info({ url: `http://localhost:${port}/view?token=${this.viewToken}` }, "Web View available");
5671
6958
  }
5672
6959
  getUiStatus() {
5673
- const instances = Object.keys(this.fleetConfig?.instances ?? {}).map(name => {
6960
+ const fleetNames = Object.keys(this.fleetConfig?.instances ?? {});
6961
+ // Classic rooms live only in classicBot.yaml — /api/profiles merges them into
6962
+ // the View roster, but previously getUiStatus skipped them so context_pct was
6963
+ // always 0 (live map miss → l?.context_pct ?? 0).
6964
+ const classicOnly = (this.classicChannels?.getAll() ?? [])
6965
+ .map(ch => ch.instanceName)
6966
+ .filter(name => !fleetNames.includes(name));
6967
+ const names = [...fleetNames, ...classicOnly];
6968
+ const instances = names.map(name => {
5674
6969
  const statusFile = join(this.getInstanceDir(name), "statusline.json");
5675
- let context_pct = 0;
5676
6970
  let cost = 0;
5677
6971
  let model = "";
5678
6972
  try {
5679
6973
  const data = JSON.parse(readFileSync(statusFile, "utf-8"));
5680
- context_pct = data.context_window?.used_percentage ?? 0;
5681
6974
  cost = data.cost?.total_cost_usd ?? 0;
5682
6975
  model = data.model?.display_name ?? "";
5683
6976
  }
5684
6977
  catch (err) {
5685
6978
  this.logger.debug({ err, name }, "statusline.json read failed (getUiStatus)");
5686
6979
  }
6980
+ // Align with /ctx: statusline for claude-code, pane scrape for kiro/grok/codex.
6981
+ const classic = classicOnly.includes(name);
6982
+ const backend = classic
6983
+ ? this.classicChannels.getBackendByInstance(name, this.fleetConfig?.defaults?.backend)
6984
+ : (this.fleetConfig?.instances[name]?.backend
6985
+ ?? this.fleetConfig?.defaults?.backend
6986
+ ?? "claude-code");
6987
+ const { context } = resolveInstanceContext(this.dataDir, name, backend);
6988
+ const context_pct = context ?? 0;
5687
6989
  return { name, status: this.getInstanceStatus(name), context_pct, cost, model };
5688
6990
  });
5689
6991
  return {