@songsid/agend 2.1.2-beta.9 → 2.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (172) hide show
  1. package/dist/access-path.js +15 -6
  2. package/dist/access-path.js.map +1 -1
  3. package/dist/agent-cli-instructions.md +1 -0
  4. package/dist/agent-cli.js +14 -0
  5. package/dist/agent-cli.js.map +1 -1
  6. package/dist/agent-endpoint.d.ts +2 -0
  7. package/dist/agent-endpoint.js +25 -1
  8. package/dist/agent-endpoint.js.map +1 -1
  9. package/dist/backend/antigravity.d.ts +20 -0
  10. package/dist/backend/antigravity.js +51 -4
  11. package/dist/backend/antigravity.js.map +1 -1
  12. package/dist/backend/claude-code.d.ts +59 -0
  13. package/dist/backend/claude-code.js +67 -1
  14. package/dist/backend/claude-code.js.map +1 -1
  15. package/dist/backend/codex.d.ts +26 -0
  16. package/dist/backend/codex.js +100 -0
  17. package/dist/backend/codex.js.map +1 -1
  18. package/dist/backend/grok.d.ts +25 -0
  19. package/dist/backend/grok.js +31 -0
  20. package/dist/backend/grok.js.map +1 -1
  21. package/dist/backend/kiro.d.ts +23 -0
  22. package/dist/backend/kiro.js +120 -2
  23. package/dist/backend/kiro.js.map +1 -1
  24. package/dist/backend/types.d.ts +58 -0
  25. package/dist/backend/types.js +13 -1
  26. package/dist/backend/types.js.map +1 -1
  27. package/dist/channel/adapters/discord.d.ts +15 -0
  28. package/dist/channel/adapters/discord.js +120 -8
  29. package/dist/channel/adapters/discord.js.map +1 -1
  30. package/dist/channel/adapters/telegram.d.ts +21 -0
  31. package/dist/channel/adapters/telegram.js +86 -0
  32. package/dist/channel/adapters/telegram.js.map +1 -1
  33. package/dist/channel/ipc-bridge.d.ts +9 -1
  34. package/dist/channel/ipc-bridge.js +12 -3
  35. package/dist/channel/ipc-bridge.js.map +1 -1
  36. package/dist/channel/ipc-timeouts.d.ts +46 -0
  37. package/dist/channel/ipc-timeouts.js +65 -0
  38. package/dist/channel/ipc-timeouts.js.map +1 -0
  39. package/dist/channel/mcp-server.js +25 -14
  40. package/dist/channel/mcp-server.js.map +1 -1
  41. package/dist/channel/mcp-tools.js +30 -1
  42. package/dist/channel/mcp-tools.js.map +1 -1
  43. package/dist/channel/reconnect-backoff.d.ts +17 -0
  44. package/dist/channel/reconnect-backoff.js +21 -0
  45. package/dist/channel/reconnect-backoff.js.map +1 -0
  46. package/dist/channel/types.d.ts +29 -0
  47. package/dist/classic-channel-manager.js +1 -5
  48. package/dist/classic-channel-manager.js.map +1 -1
  49. package/dist/cli.js +242 -91
  50. package/dist/cli.js.map +1 -1
  51. package/dist/completion.d.ts +27 -0
  52. package/dist/completion.js +121 -0
  53. package/dist/completion.js.map +1 -0
  54. package/dist/config-validator.js +21 -0
  55. package/dist/config-validator.js.map +1 -1
  56. package/dist/config.js +2 -0
  57. package/dist/config.js.map +1 -1
  58. package/dist/cost-guard.d.ts +3 -1
  59. package/dist/cost-guard.js +3 -1
  60. package/dist/cost-guard.js.map +1 -1
  61. package/dist/daemon.d.ts +298 -6
  62. package/dist/daemon.js +1142 -313
  63. package/dist/daemon.js.map +1 -1
  64. package/dist/event-log.d.ts +31 -0
  65. package/dist/event-log.js +96 -0
  66. package/dist/event-log.js.map +1 -1
  67. package/dist/fleet-context.d.ts +14 -0
  68. package/dist/fleet-lock.d.ts +28 -0
  69. package/dist/fleet-lock.js +130 -0
  70. package/dist/fleet-lock.js.map +1 -0
  71. package/dist/fleet-manager.d.ts +348 -6
  72. package/dist/fleet-manager.js +1859 -266
  73. package/dist/fleet-manager.js.map +1 -1
  74. package/dist/general-instance.d.ts +9 -0
  75. package/dist/general-instance.js +11 -0
  76. package/dist/general-instance.js.map +1 -0
  77. package/dist/general-knowledge/skills/cross-instance-messaging/SKILL.md +22 -0
  78. package/dist/general-knowledge/skills/model-discovery/SKILL.md +14 -23
  79. package/dist/general-knowledge/skills/session-management/SKILL.md +15 -34
  80. package/dist/hang-detector.d.ts +19 -13
  81. package/dist/hang-detector.js +19 -49
  82. package/dist/hang-detector.js.map +1 -1
  83. package/dist/instance-lifecycle.d.ts +58 -2
  84. package/dist/instance-lifecycle.js +231 -46
  85. package/dist/instance-lifecycle.js.map +1 -1
  86. package/dist/locale.js +7 -1
  87. package/dist/locale.js.map +1 -1
  88. package/dist/mcp-liveness.d.ts +21 -0
  89. package/dist/mcp-liveness.js +27 -0
  90. package/dist/mcp-liveness.js.map +1 -0
  91. package/dist/outbound-handlers.d.ts +9 -0
  92. package/dist/outbound-handlers.js +194 -31
  93. package/dist/outbound-handlers.js.map +1 -1
  94. package/dist/outbound-schemas.d.ts +11 -3
  95. package/dist/outbound-schemas.js +16 -2
  96. package/dist/outbound-schemas.js.map +1 -1
  97. package/dist/pane-write-lock.d.ts +48 -0
  98. package/dist/pane-write-lock.js +73 -0
  99. package/dist/pane-write-lock.js.map +1 -0
  100. package/dist/process-memory.d.ts +31 -0
  101. package/dist/process-memory.js +79 -0
  102. package/dist/process-memory.js.map +1 -0
  103. package/dist/quickstart.js +17 -16
  104. package/dist/quickstart.js.map +1 -1
  105. package/dist/reply-dedup.d.ts +41 -0
  106. package/dist/reply-dedup.js +0 -0
  107. package/dist/reply-dedup.js.map +1 -0
  108. package/dist/restart-progress.d.ts +41 -0
  109. package/dist/restart-progress.js +97 -0
  110. package/dist/restart-progress.js.map +1 -0
  111. package/dist/scheduler/db.js +3 -0
  112. package/dist/scheduler/db.js.map +1 -1
  113. package/dist/sd-notify.d.ts +27 -0
  114. package/dist/sd-notify.js +33 -1
  115. package/dist/sd-notify.js.map +1 -1
  116. package/dist/secret-file.d.ts +33 -0
  117. package/dist/secret-file.js +36 -0
  118. package/dist/secret-file.js.map +1 -0
  119. package/dist/service-installer.d.ts +28 -0
  120. package/dist/service-installer.js +72 -1
  121. package/dist/service-installer.js.map +1 -1
  122. package/dist/settings-api.js +5 -3
  123. package/dist/settings-api.js.map +1 -1
  124. package/dist/setup-wizard.js +9 -7
  125. package/dist/setup-wizard.js.map +1 -1
  126. package/dist/tmux-control.d.ts +58 -5
  127. package/dist/tmux-control.js +102 -14
  128. package/dist/tmux-control.js.map +1 -1
  129. package/dist/tmux-manager.d.ts +31 -1
  130. package/dist/tmux-manager.js +40 -11
  131. package/dist/tmux-manager.js.map +1 -1
  132. package/dist/topic-commands.d.ts +49 -9
  133. package/dist/topic-commands.js +270 -109
  134. package/dist/topic-commands.js.map +1 -1
  135. package/dist/tui-glyphs.d.ts +5 -1
  136. package/dist/tui-glyphs.js +6 -2
  137. package/dist/tui-glyphs.js.map +1 -1
  138. package/dist/types.d.ts +19 -2
  139. package/dist/ui/settings.html +1 -1
  140. package/dist/ui/view.html +123 -2
  141. package/dist/update-marker.d.ts +15 -0
  142. package/dist/update-marker.js +69 -0
  143. package/dist/update-marker.js.map +1 -0
  144. package/dist/usage/format-rich.d.ts +11 -0
  145. package/dist/usage/format-rich.js +121 -0
  146. package/dist/usage/format-rich.js.map +1 -0
  147. package/dist/usage/provider-alerts.d.ts +33 -0
  148. package/dist/usage/provider-alerts.js +57 -0
  149. package/dist/usage/provider-alerts.js.map +1 -0
  150. package/dist/usage/providers.d.ts +130 -0
  151. package/dist/usage/providers.js +1063 -0
  152. package/dist/usage/providers.js.map +1 -0
  153. package/dist/usage/statusline-usage.d.ts +20 -0
  154. package/dist/usage/statusline-usage.js +120 -0
  155. package/dist/usage/statusline-usage.js.map +1 -0
  156. package/dist/usage/usage-api.d.ts +50 -0
  157. package/dist/usage/usage-api.js +197 -0
  158. package/dist/usage/usage-api.js.map +1 -0
  159. package/dist/view-api.js +8 -3
  160. package/dist/view-api.js.map +1 -1
  161. package/dist/web-api.js +7 -3
  162. package/dist/web-api.js.map +1 -1
  163. package/dist/web-auth.d.ts +6 -0
  164. package/dist/web-auth.js +54 -0
  165. package/dist/web-auth.js.map +1 -0
  166. package/package.json +4 -1
  167. package/dist/channel/tool-tracker.d.ts +0 -13
  168. package/dist/channel/tool-tracker.js +0 -58
  169. package/dist/channel/tool-tracker.js.map +0 -1
  170. package/dist/daemon-entry.d.ts +0 -1
  171. package/dist/daemon-entry.js +0 -30
  172. package/dist/daemon-entry.js.map +0 -1
@@ -4,7 +4,10 @@ import { createServer } from "node:http";
4
4
  import { join, dirname, basename } from "node:path";
5
5
  import { fileURLToPath } from "node:url";
6
6
  import { getAgendHome, ensureWorkspaceGit } from "./paths.js";
7
- import { sdNotify } from "./sd-notify.js";
7
+ import { clearUpdateMarker, isUpdateInProgress } from "./update-marker.js";
8
+ import { sdNotify, sdNotifyBlocking } from "./sd-notify.js";
9
+ import { readFleetMemory } from "./process-memory.js";
10
+ import { ReplyDeduper } from "./reply-dedup.js";
8
11
  import { isScalar, parseDocument } from "yaml";
9
12
  const __filename = fileURLToPath(import.meta.url);
10
13
  const __dirname = dirname(__filename);
@@ -21,12 +24,12 @@ import { IpcClient } from "./channel/ipc-bridge.js";
21
24
  import { createAdapter } from "./channel/factory.js";
22
25
  import { createBackend } from "./backend/factory.js";
23
26
  import { isModelCompatible } from "./backend/types.js";
24
- import { createLogger } from "./logger.js";
27
+ import { createLogger, rotateLogIfNeeded } from "./logger.js";
25
28
  import { processAttachments } from "./channel/attachment-handler.js";
26
29
  import { routeToolCall } from "./channel/tool-router.js";
27
30
  import { Scheduler } from "./scheduler/index.js";
28
31
  import { DEFAULT_SCHEDULER_CONFIG } from "./scheduler/index.js";
29
- import { TopicCommands, saveCommandForBackend, parseSaveFilename, parsePauseWakeCommand, SAVE_FILENAME_RE, SAVE_UNSUPPORTED_MSG } from "./topic-commands.js";
32
+ import { TopicCommands, saveCommandForBackend, parseSaveFilename, parsePauseWakeCommand, SAVE_FILENAME_RE, SAVE_UNSUPPORTED_MSG, resolveInstanceContext, forgetInstanceContext } from "./topic-commands.js";
30
33
  import { DailySummary } from "./daily-summary.js";
31
34
  import { WebhookEmitter } from "./webhook-emitter.js";
32
35
  import { TmuxControlClient } from "./tmux-control.js";
@@ -38,6 +41,7 @@ import { StatuslineWatcher } from "./statusline-watcher.js";
38
41
  import { outboundHandlers } from "./outbound-handlers.js";
39
42
  import { handleWebRequest, broadcastSseEvent } from "./web-api.js";
40
43
  import { handleViewRequest, isViewPath } from "./view-api.js";
44
+ import { handleUsageRequest, isUsagePath } from "./usage/usage-api.js";
41
45
  import { handleSettingsRequest } from "./settings-api.js";
42
46
  import { setLocale, detectLocale, t } from "./locale.js";
43
47
  import { handleAgentRequest } from "./agent-endpoint.js";
@@ -45,6 +49,10 @@ import { ClassicChannelManager, getClassicBackendChoices, isSelectableClassicBac
45
49
  import { validateFleetConfig } from "./config-validator.js";
46
50
  import { readLastInboundAt } from "./daemon.js";
47
51
  import { clearPausedMarker } from "./pause-marker.js";
52
+ import { releaseProcessFleetLock } from "./fleet-lock.js";
53
+ import { GENERAL_PAUSE_ERROR, isGeneralInstance } from "./general-instance.js";
54
+ import { loadOrCreateWebToken, WEB_TOKEN_INVALID_MESSAGE } from "./web-auth.js";
55
+ import { RestartProgress } from "./restart-progress.js";
48
56
  import { getTmuxSession } from "./config.js";
49
57
  export function resolveReplyThreadId(argsThreadId, instanceConfig) {
50
58
  if (typeof argsThreadId === "string" && argsThreadId.length > 0) {
@@ -83,9 +91,80 @@ const CANCEL_BTN_MAX_RETRIES = 3;
83
91
  * buttons no clear trigger reached (e.g. a scheduled/HTTP turn that never called
84
92
  * reply). 5min (not the old 2s idle-watch) so Thinking isn't misread as idle. */
85
93
  const CANCEL_BTN_IDLE_CHECK_INTERVAL_MS = 5 * 60_000;
94
+ /**
95
+ * A queued turn can produce a very short idle edge while the CLI hands off to
96
+ * the next message. Do not retire the cancel button until that edge remains
97
+ * idle for this long; a working/stuck report during the grace cancels it.
98
+ */
99
+ const CANCEL_BTN_IDLE_RETIRE_GRACE_MS = 2_000;
100
+ /**
101
+ * How long after a reply an instance gets to resume working before its cancel
102
+ * button is retired. A short turn ends with a reply and never works again → the
103
+ * button disappears ~2 minutes after the answer. A multi-step run replies
104
+ * mid-flight and keeps going → the grace check sees "working" and leaves the
105
+ * button alone (the idle edge retires it when the run really ends).
106
+ */
107
+ const REPLY_RETIRE_GRACE_MS = 2 * 60_000;
108
+ /**
109
+ * The daemon only broadcasts execution state on TRANSITIONS, so a long
110
+ * single-state run sends nothing for hours. The idle backstop therefore pokes a
111
+ * query each tick; a live daemon answers within milliseconds and refreshes the
112
+ * cache. When nothing has refreshed it for this long despite those pokes, the
113
+ * reporting chain (daemon, IPC, or state monitor) is dead and a "working" state
114
+ * from 30 minutes ago proves nothing — the button may be retired.
115
+ */
116
+ const STATE_REPORT_STALE_MS = 30 * 60_000;
117
+ /**
118
+ * Unconditional ceiling on a cancel button's life. Deliberately far beyond any
119
+ * legitimate run (multi-hour tasks are normal on this fleet): everything below
120
+ * this is decided by real state; a button that somehow survives a full day is
121
+ * wreckage, stuck or not.
122
+ */
123
+ const CANCEL_BTN_MAX_LIFETIME_MS = 24 * 60 * 60_000;
124
+ /** A click on a button the fleet no longer tracks may fire at most this often. */
125
+ const STALE_CANCEL_CLICK_COOLDOWN_MS = 10_000;
126
+ /** Orphaned-button ledger, swept at startup. Lives in the fleet data dir. */
127
+ const CANCEL_BTN_LEDGER_FILE = "cancel-buttons.json";
128
+ /**
129
+ * How often the cancel button's text is refreshed with elapsed working time.
130
+ *
131
+ * One edit per working instance per interval — at 60s that is trivial for both
132
+ * platforms' rate limits, and it reads as a live counter rather than a stale
133
+ * snapshot. Nothing new is posted, so the channel is never spammed: there is
134
+ * exactly one progress message per turn, and it is the cancel button itself.
135
+ */
136
+ const PROGRESS_UPDATE_INTERVAL_MS = 60_000;
137
+ /** Elapsed time is only shown once work has clearly outlasted a quick answer. */
138
+ /**
139
+ * Default delay before the button starts showing elapsed time. Configurable via
140
+ * `defaults.progress_min_elapsed` (seconds) in fleet.yaml. 30s is the balance
141
+ * point: most quick answers finish inside it (no churn for ordinary turns),
142
+ * while anything real shows signs of life well before the old two minutes.
143
+ */
144
+ const PROGRESS_MIN_ELAPSED_MS = 30_000;
145
+ /** How much of a tool summary the progress line will show before eliding. */
146
+ const PROGRESS_ACTIVITY_MAX_CHARS = 48;
147
+ /**
148
+ * Emoji AgEnD itself stamps on messages as the delivery-status ladder
149
+ * (⏳ queued, 👀 delivered, ✅ confirmed, ❌ failed). These are machine
150
+ * indicators, not opinions, so they never enter the reactions queue — from
151
+ * anyone. This exact-emoji filter is the ONLY bot filtering left: bot-to-bot
152
+ * reactions are otherwise delivered on purpose (agents signal each other), and
153
+ * 🫡 passes too — it reads as a deliberate acknowledgement, not plumbing.
154
+ */
155
+ const DELIVERY_STATUS_EMOJIS = new Set(["👀", "⏳", "✅", "❌"]);
156
+ /**
157
+ * How long a delivery waits out a disconnected instance IPC before giving up.
158
+ *
159
+ * Sized for a daemon restart (socket close → respawn → CLI ready), which is the
160
+ * event this exists for. Past it the delivery fails loudly as it always did.
161
+ */
162
+ const IPC_RECONNECT_GRACE_MS = 30_000;
163
+ const IPC_RECONNECT_POLL_MS = 250;
86
164
  const CLASSIC_BACKEND_SELECTION_TIMEOUT_MS = 60_000;
87
165
  const CLASSIC_BACKEND_CALLBACK_PREFIX = "classic-backend:";
88
166
  const MODEL_SELECT_CALLBACK_PREFIX = "model-select:";
167
+ const EFFORT_SELECT_CALLBACK_PREFIX = "effort-select:";
89
168
  const CLI_ENV_TTL_MS = 24 * 60 * 60 * 1000; // /model reads cached CLI env within 24h
90
169
  export class FleetManager {
91
170
  dataDir;
@@ -93,7 +172,7 @@ export class FleetManager {
93
172
  static sighupHandlerInstalled = false;
94
173
  children = new Map();
95
174
  lifecycle;
96
- /** @deprecated Use lifecycle.daemons — kept for backward compat */
175
+ /** Live view of lifecycle.daemons — used throughout; not deprecated. */
97
176
  get daemons() { return this.lifecycle.daemons; }
98
177
  fleetConfig = null;
99
178
  rawFleetConfig = {};
@@ -108,8 +187,11 @@ export class FleetManager {
108
187
  // same guild both receive every message). Bounded FIFO of recent message keys.
109
188
  recentMessageIds = new Set();
110
189
  accessManager = null;
111
- /** Primary world (first adapter) used for fleet-level notifications */
112
- get primaryWorld() { return this.worlds.values().next().value; }
190
+ /** Primary world (channels[0]), independent of concurrent adapter startup order. */
191
+ get primaryWorld() {
192
+ const adapterId = this.getPrimaryAdapterId();
193
+ return adapterId ? this.worlds.get(adapterId) : undefined;
194
+ }
113
195
  routing = new RoutingEngine();
114
196
  get routingTable() { return this.routing.map; }
115
197
  instanceIpcClients = new Map();
@@ -151,6 +233,16 @@ export class FleetManager {
151
233
  // reply, on cancel, or when a newer button supersedes it for the same
152
234
  // instance. Per-button tracking means a failed delete never strands a button.
153
235
  cancelButtons = new Map();
236
+ /** Pending idle-edge retirement, one timer per instance. */
237
+ cancelButtonIdleRetireTimers = new Map();
238
+ /** Duplicate-reply suppression across both the MCP and HTTP reply paths. */
239
+ replyDeduper = new ReplyDeduper();
240
+ /** instanceName → what it is doing right now, when the backend can tell us. */
241
+ instanceActivity = new Map();
242
+ /** instanceName → tail of deliveries waiting for its IPC to come back. */
243
+ ipcWaitTails = new Map();
244
+ /** instanceName → restart currently executing; concurrent callers join it. */
245
+ restartsInFlight = new Map();
154
246
  // Last user message delivered to each instance — used to react ✅ on completion.
155
247
  lastInboundMsg = new Map();
156
248
  topicArchiver;
@@ -158,11 +250,17 @@ export class FleetManager {
158
250
  classicChannels = null;
159
251
  pendingClassicStarts = new Map();
160
252
  /** In-flight /model selections, keyed by nonce (see handleModelSelection). */
253
+ /** In-flight /effort selections, same coordinator shape as pendingModelSelects. */
254
+ pendingEffortSelects = new Map();
161
255
  pendingModelSelects = new Map();
162
256
  // Model failover state
163
257
  failoverActive = new Map(); // instance → current failover model
164
258
  // IPC reconnect: tracks instances being intentionally stopped (skip reconnect)
259
+ /** instance → when a click with no live button entry last fired a cancel. */
260
+ staleCancelClickAt = new Map();
165
261
  ipcStoppingInstances = new Set();
262
+ /** Set the moment a graceful stop begins — see isPlannedRestart(). */
263
+ shuttingDown = false;
166
264
  /** Coalesce concurrent connection attempts for the same daemon socket. */
167
265
  ipcConnectInFlight = new Map();
168
266
  /** At most one reconnect/backoff loop may exist per instance. */
@@ -176,6 +274,10 @@ export class FleetManager {
176
274
  healthServer = null;
177
275
  healthPortRetried = false;
178
276
  updateCheckTimer = null;
277
+ eventLogPruneTimer = null;
278
+ logRotateTimer = null;
279
+ /** Days of event/activity history to keep. */
280
+ static EVENT_LOG_RETENTION_DAYS = 30;
179
281
  watchdogTimer = null;
180
282
  startedAt = 0;
181
283
  // Mirror topic: buffer cross-instance messages, flush every 3s
@@ -185,6 +287,7 @@ export class FleetManager {
185
287
  sseClients = new Set();
186
288
  webToken = null;
187
289
  viewToken = null;
290
+ healthServerListening = false;
188
291
  constructor(dataDir) {
189
292
  this.dataDir = dataDir;
190
293
  FleetManager.signalTarget = this;
@@ -214,7 +317,13 @@ export class FleetManager {
214
317
  }
215
318
  this.reloadPending = false;
216
319
  this.reconcileInFlight = this.reconcileInstances()
217
- .catch(err => this.logger.error({ err }, "SIGHUP config reload failed"))
320
+ .catch(err => {
321
+ // Almost always a YAML parse error. Log-only meant the user edited
322
+ // fleet.yaml, sent SIGHUP, and got no reaction and no explanation.
323
+ this.logger.error({ err }, "SIGHUP config reload failed");
324
+ const message = err instanceof Error ? err.message : String(err);
325
+ this.notifyFleetError(`⚠️ fleet.yaml reload FAILED — ${message}\nThe previous configuration is still running.`);
326
+ })
218
327
  .finally(() => {
219
328
  this.reconcileInFlight = null;
220
329
  if (this.reloadPending && this.startupComplete) {
@@ -222,16 +331,55 @@ export class FleetManager {
222
331
  }
223
332
  });
224
333
  }
334
+ /**
335
+ * Is the fleet going down (or coming back up) on purpose?
336
+ *
337
+ * Instances dying during a planned restart is the restart working, not an
338
+ * incident — but the code that notices a dead pane or a dead MCP server
339
+ * cannot tell the difference on its own. Two sources, because the noise
340
+ * starts before this process is even told to stop: `agend update` replaces
341
+ * the package on disk while this daemon is still running and still watching.
342
+ */
343
+ isPlannedRestart() {
344
+ return this.shuttingDown || isUpdateInProgress(this.dataDir);
345
+ }
225
346
  finishStartup() {
226
347
  this.startupComplete = true;
348
+ // We are the post-update fleet: the update is over by definition. Clearing
349
+ // it here (rather than in the update command, which exits before the new
350
+ // fleet is up) is what keeps the quiet window from outliving the restart.
351
+ clearUpdateMarker(this.dataDir);
227
352
  if (this.reloadPending)
228
353
  this.scheduleReconcile();
354
+ void this.sweepOrphanedCancelButtons();
229
355
  }
230
356
  // ── ArchiverContext bridge ────────────────────────────────────────────
231
357
  lastActivityMs(name) {
232
358
  return this.lastActivity.get(name) ?? 0;
233
359
  }
360
+ /**
361
+ * Is the instance between turns?
362
+ *
363
+ * Prefers the daemon's pane state machine (debounced, busy-pattern aware) over
364
+ * the control client's raw 2-second output-silence heuristic. The raw heuristic
365
+ * reads every >2s output lull as idle — and long silent tools (a build, a test
366
+ * run) or an LLM pause produce those constantly mid-turn. That misreading is
367
+ * what retired cancel buttons in the middle of long work (the 5-minute backstop
368
+ * fired during a lull) and froze their progress text (ticker skipped "idle"
369
+ * ticks). The silence heuristic remains only as the fallback for instances
370
+ * whose daemon has not reported a state yet.
371
+ */
234
372
  getInstanceIdle(name) {
373
+ // A daemon that is not running cannot be mid-turn. This is what a stale
374
+ // "working" cache after a hard daemon kill (SIGKILL/OOM — no IPC crash
375
+ // report ever arrives) must not override.
376
+ if (this.getInstanceStatus(name) !== "running")
377
+ return true;
378
+ const state = this.getInstanceExecutionState(name);
379
+ if (state === "working" || state === "stuck")
380
+ return false;
381
+ if (state === "idle")
382
+ return true;
235
383
  try {
236
384
  const widFile = join(this.getInstanceDir(name), "window-id");
237
385
  if (!existsSync(widFile))
@@ -243,6 +391,18 @@ export class FleetManager {
243
391
  return true;
244
392
  }
245
393
  }
394
+ /**
395
+ * True when the instance claims working/stuck but nothing has refreshed that
396
+ * claim for STATE_REPORT_STALE_MS despite the backstop's per-tick queries.
397
+ * Measures the CACHE's age, not the button's — a healthy multi-hour run
398
+ * answers every query and never trips this.
399
+ */
400
+ stateReportDead(name) {
401
+ const cached = this.instanceStateCache.get(name);
402
+ if (!cached)
403
+ return false; // no claim to distrust — getInstanceIdle owns this case
404
+ return Date.now() - cached.receivedAt > STATE_REPORT_STALE_MS;
405
+ }
246
406
  // ── LifecycleContext bridge methods ──────────────────────────────────────
247
407
  webhookEmit(event, name, data) {
248
408
  this.webhookEmitter?.emit(event, name, data);
@@ -304,10 +464,20 @@ export class FleetManager {
304
464
  if (!this.classicChannels)
305
465
  return;
306
466
  const channels = this.classicChannels.getAll();
467
+ // Classic's persisted adapter is authoritative. Legacy adapter-less rows
468
+ // deterministically belong to channels[0], never to whichever bot happens
469
+ // to deliver the first message after startup/reconnect.
470
+ for (const ch of channels) {
471
+ const adapterId = ch.adapterId ?? this.getPrimaryAdapterId();
472
+ if (adapterId)
473
+ this.instanceWorldBinding.set(ch.instanceName, adapterId);
474
+ }
307
475
  // Always update adapter openChannels (including empty — clears stale entries on /stop)
308
476
  for (const [adapterId, w] of this.worlds) {
309
477
  if (typeof w.adapter?.setOpenChannels === "function") {
310
- const owned = channels.filter(ch => ch.adapterId === adapterId).map(ch => ch.channelId);
478
+ const owned = channels
479
+ .filter(ch => (ch.adapterId ?? this.getPrimaryAdapterId()) === adapterId)
480
+ .map(ch => ch.channelId);
311
481
  w.adapter.setOpenChannels(owned);
312
482
  }
313
483
  }
@@ -385,58 +555,113 @@ export class FleetManager {
385
555
  }
386
556
  /** Get the adapter bound to an instance, falling back to primary adapter */
387
557
  getAdapterForInstance(name) {
388
- const worldId = this.instanceWorldBinding.get(name);
558
+ const worldId = this.getInstanceAdapterId(name);
389
559
  if (worldId)
390
560
  return this.worlds.get(worldId)?.adapter ?? this.adapter;
391
561
  return this.adapter;
392
562
  }
393
563
  /** Get the world for an instance */
394
564
  getWorldForInstance(name) {
395
- const worldId = this.instanceWorldBinding.get(name);
396
- return worldId ? this.worlds.get(worldId) : this.worlds.values().next().value;
565
+ const worldId = this.getInstanceAdapterId(name);
566
+ return worldId ? this.worlds.get(worldId) : undefined;
397
567
  }
398
568
  /** Get channel config for a specific adapter (by id), falling back to primary */
399
569
  getChannelConfig(adapterId) {
570
+ const channels = this.fleetConfig?.channels
571
+ ?? (this.fleetConfig?.channel ? [this.fleetConfig.channel] : []);
400
572
  if (adapterId) {
401
- const world = this.worlds.get(adapterId);
402
- if (world)
403
- return world.channelConfig;
573
+ return channels.find(ch => (ch.id ?? ch.type) === adapterId)
574
+ ?? this.worlds.get(adapterId)?.channelConfig
575
+ ?? channels[0];
404
576
  }
405
- return this.fleetConfig?.channel;
577
+ return channels[0];
406
578
  }
407
579
  /** Get the group_id for an instance's bound adapter */
408
580
  getGroupIdForInstance(name) {
409
- const world = this.getWorldForInstance(name);
410
- return world?.groupId ?? String(this.fleetConfig?.channel?.group_id ?? "");
581
+ const adapterId = this.getInstanceAdapterId(name);
582
+ const world = adapterId ? this.worlds.get(adapterId) : undefined;
583
+ return world?.groupId ?? String(this.getChannelConfig(adapterId)?.group_id ?? "");
584
+ }
585
+ /** Configured primary adapter id. Never infer this from Map insertion order. */
586
+ getPrimaryAdapterId() {
587
+ const primary = this.fleetConfig?.channels?.[0] ?? this.fleetConfig?.channel;
588
+ if (primary)
589
+ return primary.id ?? primary.type;
590
+ // Defensive compatibility for callers/tests that provide a live world but
591
+ // no channel config. Real multi-adapter fleets always have channels[].
592
+ return this.worlds.keys().next().value;
593
+ }
594
+ /** Warn when a coordinator's adapter identity is ambiguous to the operator. */
595
+ warnUnboundGeneralChannelIds(fleet) {
596
+ const channels = fleet.channels ?? (fleet.channel ? [fleet.channel] : []);
597
+ if (channels.length <= 1)
598
+ return;
599
+ const adapterIds = channels.map(ch => ch.id ?? ch.type);
600
+ for (const [name, config] of Object.entries(fleet.instances)) {
601
+ if (!config.general_topic || config.channel_id)
602
+ continue;
603
+ this.logger.warn({
604
+ instance: name,
605
+ defaultAdapter: adapterIds[0],
606
+ availableAdapters: adapterIds,
607
+ }, "General instance has no channel_id in a multi-channel fleet; defaulting to the first adapter. Set channel_id explicitly.");
608
+ }
609
+ }
610
+ /**
611
+ * Resolve the authoritative adapter identity for an instance.
612
+ * Fleet instances without channel_id and legacy Classic entries both belong
613
+ * to channels[0]. Runtime bindings remain available for external sessions.
614
+ */
615
+ getInstanceAdapterId(name) {
616
+ const cfg = this.fleetConfig?.instances[name];
617
+ if (cfg)
618
+ return cfg.channel_id ?? this.getPrimaryAdapterId();
619
+ if (this.classicChannels?.getChannelIdByInstance(name) !== undefined) {
620
+ return this.classicChannels.getAdapterIdByInstance(name) ?? this.getPrimaryAdapterId();
621
+ }
622
+ return this.instanceWorldBinding.get(name) ?? this.getPrimaryAdapterId();
411
623
  }
412
624
  /**
413
625
  * Bind an instance to a specific world (the bot that answers for it).
414
626
  * fromInbound=true (binding inferred from which adapter received a message)
415
- * must not override a configured identity: skip when the instance is a general
416
- * or has an explicit channel_id otherwise a persona instance whose message
417
- * was also seen by the main bot would get rebound to the wrong bot.
627
+ * must not override a configured identity. Fleet instances use channel_id or
628
+ * channels[0]; Classic instances use their persisted adapter (or channels[0]
629
+ * for a legacy entry). Only external sessions may bind from inbound traffic.
418
630
  */
419
631
  bindInstanceAdapter(name, adapterId, fromInbound = false) {
420
- const cfg = this.fleetConfig?.instances[name];
421
632
  if (fromInbound) {
422
- // Skip inbound-derived binding for any instance that doesn't have an
423
- // explicit channel_id — those default to primary adapter deterministically.
424
- // This prevents a non-deterministic race where whichever adapter delivers
425
- // first after restart wins the binding.
426
- if (cfg?.general_topic || cfg?.channel_id)
427
- return;
428
- if (cfg && !cfg.channel_id)
429
- return; // fleet instance without explicit binding → use primary
430
- // Classic instance: don't override an existing binding (authoritative from /start)
431
- if (this.classicChannels?.getChannelIdByInstance(name) !== undefined && this.instanceWorldBinding.has(name))
633
+ const configuredId = this.getInstanceAdapterId(name);
634
+ if (this.fleetConfig?.instances[name]
635
+ || this.classicChannels?.getChannelIdByInstance(name) !== undefined) {
636
+ if (configuredId)
637
+ this.instanceWorldBinding.set(name, configuredId);
432
638
  return;
639
+ }
433
640
  }
434
641
  this.instanceWorldBinding.set(name, adapterId);
435
642
  }
436
643
  getInstanceStatus(name) {
437
644
  if (this.lifecycle.isPaused(name))
438
645
  return "paused";
646
+ const daemon = this.lifecycle.daemons.get(name);
439
647
  const processStatus = this.instanceProcessStatus.get(name);
648
+ // IPC can be disconnected during a respawn, so the event which announces
649
+ // the new live pane may be missed. The in-process daemon is authoritative
650
+ // in that case; do not leave a stale `crashed` cache masking an instance
651
+ // that has already recovered and can answer messages.
652
+ const daemonStatus = daemon?.getProcessStatus?.();
653
+ if (daemonStatus === "running") {
654
+ if (processStatus)
655
+ this.instanceProcessStatus.delete(name);
656
+ // A recovered in-process daemon is also the authority for clearing a
657
+ // marker left by a crash-loop/reconnect race. This keeps standalone
658
+ // `agend ls` from seeing the old marker after the next API outage.
659
+ try {
660
+ unlinkSync(join(this.getInstanceDir(name), "crash-state.json"));
661
+ }
662
+ catch { /* absent */ }
663
+ return "running";
664
+ }
440
665
  if (processStatus)
441
666
  return processStatus;
442
667
  const pidPath = join(this.getInstanceDir(name), "daemon.pid");
@@ -454,6 +679,10 @@ export class FleetManager {
454
679
  getInstanceExecutionState(name) {
455
680
  if (this.lifecycle.isPaused(name))
456
681
  return null;
682
+ // Process status wins over a stale pane snapshot. A dead remain-on-exit pane
683
+ // can still contain the old ready marker and must never surface as Idle.
684
+ if (this.instanceProcessStatus.has(name))
685
+ return null;
457
686
  return this.instanceStateCache.get(name)?.state ?? null;
458
687
  }
459
688
  isClassicInstance(name) {
@@ -471,22 +700,61 @@ export class FleetManager {
471
700
  unchangedForMs: numberOr(msg.unchangedForMs, previous?.unchangedForMs ?? 0),
472
701
  observedAt: numberOr(msg.observedAt, now),
473
702
  stateChangedAt: numberOr(msg.stateChangedAt, previous?.state === state ? previous.stateChangedAt : now),
703
+ // Fleet-manager receipt time, NOT the daemon's observation time: staleness
704
+ // asks "is anyone still reporting", which only the receiver can date.
705
+ receivedAt: now,
474
706
  });
475
707
  for (const check of this.instanceIdleWaiters.get(name) ?? [])
476
708
  check();
477
709
  // warm_cap: a fresh transition into idle may free this instance for eviction,
478
710
  // or (more usefully) reveal that the fleet is now over cap. Only fire on the
479
711
  // edge into idle, not on every idle heartbeat.
480
- if (state === "idle" && previous?.state !== "idle")
712
+ if (state === "idle" && previous?.state !== "idle") {
481
713
  this.enforceWarmCap();
714
+ // A queued message may turn this edge back into working almost
715
+ // immediately. Give that handoff a short grace before retiring the button.
716
+ this.scheduleIdleButtonRetirement(name);
717
+ }
718
+ else if (state !== "idle") {
719
+ this.cancelIdleButtonRetirement(name);
720
+ }
721
+ }
722
+ cancelIdleButtonRetirement(name) {
723
+ const timer = this.cancelButtonIdleRetireTimers.get(name);
724
+ if (!timer)
725
+ return;
726
+ clearTimeout(timer);
727
+ this.cancelButtonIdleRetireTimers.delete(name);
728
+ }
729
+ scheduleIdleButtonRetirement(name) {
730
+ this.cancelIdleButtonRetirement(name);
731
+ const timer = setTimeout(() => {
732
+ // Ignore a superseded timer even if it was already queued to run.
733
+ if (this.cancelButtonIdleRetireTimers.get(name) !== timer)
734
+ return;
735
+ this.cancelButtonIdleRetireTimers.delete(name);
736
+ if (this.getInstanceExecutionState(name) === "idle") {
737
+ this.retireInstanceButtons(name);
738
+ }
739
+ }, CANCEL_BTN_IDLE_RETIRE_GRACE_MS);
740
+ timer.unref?.();
741
+ this.cancelButtonIdleRetireTimers.set(name, timer);
482
742
  }
483
743
  cacheInstanceProcessStatus(name, status) {
484
744
  if (status === "running") {
485
745
  this.instanceProcessStatus.delete(name);
746
+ // A prior crash-loop marker is one-shot. Successful respawn is the
747
+ // authoritative recovery signal even when the marker outlived an IPC
748
+ // disconnect and was not consumed by a new Daemon constructor.
749
+ try {
750
+ unlinkSync(join(this.getInstanceDir(name), "crash-state.json"));
751
+ }
752
+ catch { /* absent */ }
486
753
  return;
487
754
  }
488
755
  if (status !== "crashed" && status !== "stopped")
489
756
  return;
757
+ this.cancelIdleButtonRetirement(name);
490
758
  this.instanceProcessStatus.set(name, status);
491
759
  // Never display the last ready prompt as current execution state after its
492
760
  // owning CLI process has exited.
@@ -517,7 +785,7 @@ export class FleetManager {
517
785
  const victims = selectLruEvictions(warm, cap, {
518
786
  exclude,
519
787
  isEvicting: name => this.warmCapEvicting.has(name),
520
- isGeneral: name => this.fleetConfig?.instances[name]?.general_topic === true,
788
+ isGeneral: name => isGeneralInstance(this.fleetConfig, name),
521
789
  isIdle: name => this.getInstanceExecutionState(name) === "idle",
522
790
  lastInboundAt: name => readLastInboundAt(this.getInstanceDir(name)) ?? 0,
523
791
  });
@@ -583,12 +851,70 @@ export class FleetManager {
583
851
  if (!idle) {
584
852
  this.logger.warn({ instanceName, timeoutMs }, "Idle gate timed out; forcing delivery");
585
853
  }
586
- const ipc = this.instanceIpcClients.get(instanceName);
587
- if (!ipc?.connected)
588
- throw new Error(`Instance '${instanceName}' IPC is unavailable`);
589
- ipc.send(payload);
854
+ await this.sendWhenConnected(instanceName, payload);
590
855
  this.lastDeliveryAt.set(instanceName, Date.now());
591
856
  }
857
+ /**
858
+ * Hand a payload to an instance's IPC, waiting out a *transient* disconnect.
859
+ *
860
+ * A daemon that is restarting — `/restart`, crash recovery, a model switch —
861
+ * drops its socket for a few seconds. Any message arriving in that window used
862
+ * to fail instantly: the caller logged a warning, put ❌ on the user's message,
863
+ * and the message was gone. The user had to notice the ❌ and retype it. That is
864
+ * the "instance 訊息不容易掉" goal failing on the most predictable event there is.
865
+ *
866
+ * The wait is bounded. If the instance is genuinely down, this still throws and
867
+ * the ❌ still appears — just for a real failure rather than a restart.
868
+ *
869
+ * Ordering is preserved by serialising behind any waiter already queued for this
870
+ * instance, *including* when the socket happens to be up: otherwise a message
871
+ * arriving after the reconnect could overtake one that has been waiting for it.
872
+ */
873
+ async sendWhenConnected(instanceName, payload) {
874
+ const queued = this.ipcWaitTails.get(instanceName);
875
+ if (!queued) {
876
+ const ipc = this.instanceIpcClients.get(instanceName);
877
+ if (ipc?.connected && ipc.send(payload))
878
+ return;
879
+ }
880
+ const attempt = (queued ?? Promise.resolve())
881
+ .catch(() => { })
882
+ .then(() => this.sendAfterIpcReturns(instanceName, payload));
883
+ // The chain stores a settled-either-way promise so one failed delivery cannot
884
+ // wedge every later one, and so `queued` above is safe to await unguarded.
885
+ const tail = attempt.catch(() => { });
886
+ this.ipcWaitTails.set(instanceName, tail);
887
+ try {
888
+ await attempt;
889
+ }
890
+ finally {
891
+ // Only the last waiter clears the chain; while a queue is still draining the
892
+ // map must keep pointing at it or ordering is lost.
893
+ if (this.ipcWaitTails.get(instanceName) === tail) {
894
+ this.ipcWaitTails.delete(instanceName);
895
+ }
896
+ }
897
+ }
898
+ /** Poll for the instance's IPC to come back, then send. Throws if it does not. */
899
+ async sendAfterIpcReturns(instanceName, payload) {
900
+ const deadline = Date.now() + IPC_RECONNECT_GRACE_MS;
901
+ let warned = false;
902
+ for (;;) {
903
+ // Re-read every round: a reconnect replaces the IpcClient object entirely,
904
+ // so a cached reference would stay dead forever.
905
+ const ipc = this.instanceIpcClients.get(instanceName);
906
+ if (ipc?.connected && ipc.send(payload))
907
+ return;
908
+ if (Date.now() >= deadline) {
909
+ throw new Error(`Instance '${instanceName}' IPC is unavailable`);
910
+ }
911
+ if (!warned) {
912
+ warned = true;
913
+ this.logger.info({ instanceName }, "Instance IPC is down — holding delivery until it reconnects");
914
+ }
915
+ await new Promise(resolve => setTimeout(resolve, IPC_RECONNECT_POLL_MS));
916
+ }
917
+ }
592
918
  /** Single delivery facade: wake paused CLIs and serialize non-user work behind idle. */
593
919
  async deliverToInstance(instanceName, payload, options = {}) {
594
920
  const meta = payload.meta && typeof payload.meta === "object"
@@ -604,10 +930,7 @@ export class FleetManager {
604
930
  await this.lifecycle.wake(instanceName, 30_000);
605
931
  this.enforceWarmCap(instanceName); // woke one → evict a different LRU idle if over cap
606
932
  }
607
- const ipc = this.instanceIpcClients.get(instanceName);
608
- if (!ipc?.connected)
609
- throw new Error(`Instance '${instanceName}' IPC is unavailable`);
610
- ipc.send(payload);
933
+ await this.sendWhenConnected(instanceName, payload);
611
934
  // A cross-instance item arriving before the daemon observes this turn as
612
935
  // working must not trust the stale idle snapshot from before the send.
613
936
  this.lastDeliveryAt.set(instanceName, Date.now());
@@ -636,6 +959,9 @@ export class FleetManager {
636
959
  this.enforceWarmCap(name); // manual wake still respects the fleet warm cap
637
960
  return "awake";
638
961
  }
962
+ if (isGeneralInstance(this.fleetConfig, name)) {
963
+ throw new Error(GENERAL_PAUSE_ERROR);
964
+ }
639
965
  await this.lifecycle.pause(name);
640
966
  return this.lifecycle.isPaused(name) ? "paused" : "not_idle";
641
967
  }
@@ -655,11 +981,41 @@ export class FleetManager {
655
981
  await new Promise(resolve => setTimeout(resolve, 250));
656
982
  await this.startClassicInstance(instanceName, this.classicChannels.getBackendByInstance(instanceName, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(channel.channelId, channel.adapterId), this.classicChannels.getModel(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
657
983
  }
658
- async startInstance(name, config, topicMode, kind = "fleet-topic") {
984
+ async startInstance(name, config, topicMode, kind = "fleet-topic",
985
+ /**
986
+ * Explicit starts (CLI/API) may resume a paused or failed daemon. Startup
987
+ * and reconcile calls leave this false so a persisted pause remains paused
988
+ * across a fleet restart.
989
+ */
990
+ resumePaused = false) {
991
+ if (resumePaused && this.lifecycle.isPaused(name)) {
992
+ await this.lifecycle.wake(name, 30_000);
993
+ // A successful wake clears the persisted pause marker and produces a
994
+ // fresh instance_state snapshot. Drop any stale process error left by a
995
+ // pre-pause crash so /api/fleet and `agend ls` converge on running.
996
+ this.instanceProcessStatus.delete(name);
997
+ return;
998
+ }
659
999
  if (this.lifecycle.isPaused(name)) {
660
1000
  this.logger.info({ name }, "Persisted paused instance — skipping startup");
661
1001
  return;
662
1002
  }
1003
+ if (this.lifecycle.daemons.has(name)) {
1004
+ // A crash-loop daemon remains in the lifecycle map so its health monitor
1005
+ // can expose the failure. The old start path treated that object as
1006
+ // already running and merely deleted the process-status cache, leaving a
1007
+ // dead pane (and crash marker) behind. An explicit start is a recovery
1008
+ // request: tear down the failed daemon and build a fresh one.
1009
+ if (resumePaused) {
1010
+ const status = this.getInstanceStatus(name);
1011
+ if (status === "crashed" || status === "stopped") {
1012
+ await this.restartSingleInstance(name);
1013
+ return;
1014
+ }
1015
+ }
1016
+ this.logger.info({ name }, "Instance already running, skipping");
1017
+ return;
1018
+ }
663
1019
  if (config.general_topic) {
664
1020
  // antigravity (agy) does not read MCP instructions — fleet context and
665
1021
  // routing instructions are not injected, so it cannot act as a dispatcher.
@@ -670,12 +1026,19 @@ export class FleetManager {
670
1026
  }
671
1027
  this.ensureGeneralInstructions(config.working_directory, config.backend);
672
1028
  }
673
- this.instanceProcessStatus.delete(name);
674
1029
  await this.lifecycle.start(name, config, topicMode, {
675
1030
  kind,
676
1031
  backend: config.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code",
677
1032
  model: this.resolveInstanceModel(name).display,
678
1033
  });
1034
+ // Only clear a stale process status after a real start succeeded. Clearing
1035
+ // it before lifecycle.start() can turn a crash-loop daemon's dead pane into
1036
+ // a falsely running instance when lifecycle.start() returns early.
1037
+ this.instanceProcessStatus.delete(name);
1038
+ try {
1039
+ unlinkSync(join(this.getInstanceDir(name), "crash-state.json"));
1040
+ }
1041
+ catch { /* consumed or absent */ }
679
1042
  // Auto-connect IPC — daemon.start() ensures socket is ready before resolving
680
1043
  await this.connectIpcToInstance(name);
681
1044
  }
@@ -699,7 +1062,18 @@ export class FleetManager {
699
1062
  * to avoid config file races. Stagger delay is group-to-group, not instance-to-instance.
700
1063
  * TODO: per-instance startup timeout (existing issue, not introduced here)
701
1064
  */
702
- async startInstancesWithConcurrency(entries, topicMode) {
1065
+ async startInstancesWithConcurrency(entries, topicMode, onReady) {
1066
+ // Persisted pauses are intentionally preserved across fleet restarts. Filter
1067
+ // them before grouping/staggering: startInstance() retains its own guard as
1068
+ // a final backstop, but putting a no-op entry in this queue still consumes a
1069
+ // full stagger slot for every distinct working directory.
1070
+ const runnableEntries = entries.filter(([name]) => !this.lifecycle.isPaused(name));
1071
+ const pausedCount = entries.length - runnableEntries.length;
1072
+ if (pausedCount > 0) {
1073
+ this.logger.info({ pausedCount }, "Paused instances excluded from startup queue");
1074
+ }
1075
+ if (runnableEntries.length === 0)
1076
+ return;
703
1077
  const raw = this.fleetConfig?.defaults?.startup;
704
1078
  const explicitConcurrency = raw?.concurrency;
705
1079
  const staggerMs = Math.max(0, Math.min(30_000, raw?.stagger_delay_ms ?? 500));
@@ -714,10 +1088,10 @@ export class FleetManager {
714
1088
  else {
715
1089
  const freeMemMB = Math.round(freemem() / (1024 * 1024));
716
1090
  concurrency = Math.max(2, Math.min(10, Math.floor(freeMemMB / ESTIMATED_MB_PER_INSTANCE)));
717
- this.logger.info({ concurrency, freeMemMB: freeMemMB, totalInstances: entries.length }, "Adaptive startup concurrency");
1091
+ this.logger.info({ concurrency, freeMemMB: freeMemMB, totalInstances: runnableEntries.length }, "Adaptive startup concurrency");
718
1092
  }
719
1093
  const byWorkDir = new Map();
720
- for (const [name, config] of entries) {
1094
+ for (const [name, config] of runnableEntries) {
721
1095
  const dir = config.working_directory;
722
1096
  if (!byWorkDir.has(dir))
723
1097
  byWorkDir.set(dir, []);
@@ -760,7 +1134,14 @@ export class FleetManager {
760
1134
  lastStartAt = Date.now();
761
1135
  (async () => {
762
1136
  for (const [name, config] of group) {
763
- await this.startInstance(name, config, topicMode).catch((err) => this.logger.error({ err, name }, "Failed to start instance"));
1137
+ try {
1138
+ await this.startInstance(name, config, topicMode);
1139
+ if (this.daemons.has(name))
1140
+ onReady?.(name);
1141
+ }
1142
+ catch (err) {
1143
+ this.logger.error({ err, name }, "Failed to start instance");
1144
+ }
764
1145
  }
765
1146
  })().finally(() => {
766
1147
  running--;
@@ -774,8 +1155,41 @@ export class FleetManager {
774
1155
  startNext();
775
1156
  });
776
1157
  }
1158
+ runnableStartupCount(fleet, includeClassic) {
1159
+ const names = this.configuredStartupInstanceNames(fleet, includeClassic);
1160
+ let count = 0;
1161
+ for (const name of names) {
1162
+ if (!this.lifecycle.isPaused(name))
1163
+ count++;
1164
+ }
1165
+ return count;
1166
+ }
1167
+ configuredStartupInstanceNames(fleet, includeClassic) {
1168
+ const names = new Set(Object.keys(fleet.instances));
1169
+ if (includeClassic) {
1170
+ for (const channel of this.classicChannels?.getAll() ?? [])
1171
+ names.add(channel.instanceName);
1172
+ }
1173
+ return [...names];
1174
+ }
1175
+ restartProgressTarget() {
1176
+ const generalName = this.findGeneralInstance();
1177
+ if (!generalName)
1178
+ return null;
1179
+ const adapter = this.getAdapterForInstance(generalName);
1180
+ const chatId = this.getGroupIdForInstance(generalName);
1181
+ if (!adapter || !chatId)
1182
+ return null;
1183
+ const topicId = this.fleetConfig?.instances[generalName]?.topic_id;
1184
+ return {
1185
+ adapter,
1186
+ chatId,
1187
+ threadId: topicId != null ? String(topicId) : undefined,
1188
+ };
1189
+ }
777
1190
  async stopInstance(name) {
778
1191
  this.failoverActive.delete(name);
1192
+ this.cancelIdleButtonRetirement(name);
779
1193
  this.instanceStateCache.delete(name);
780
1194
  this.instanceProcessStatus.delete(name);
781
1195
  this.lastDeliveryAt.delete(name);
@@ -783,6 +1197,21 @@ export class FleetManager {
783
1197
  }
784
1198
  /** Restart a single instance, reloading fleet.yaml first to pick up config changes. */
785
1199
  async restartSingleInstance(name, opts) {
1200
+ // One restart at a time per instance. Multiple sources can ask concurrently
1201
+ // (MCP revival, /restart, pty_error, model failover); a second stop/start
1202
+ // interleaved with the first tears down the window the first just created.
1203
+ // Later callers join the in-flight restart instead — its opts win.
1204
+ const inFlight = this.restartsInFlight.get(name);
1205
+ if (inFlight) {
1206
+ this.logger.info({ name }, "restartSingleInstance: joining the restart already in flight");
1207
+ return inFlight;
1208
+ }
1209
+ const run = this.doRestartSingleInstance(name, opts)
1210
+ .finally(() => this.restartsInFlight.delete(name));
1211
+ this.restartsInFlight.set(name, run);
1212
+ return run;
1213
+ }
1214
+ async doRestartSingleInstance(name, opts) {
786
1215
  if (this.configPath) {
787
1216
  this.loadConfig(this.configPath);
788
1217
  this.routing.rebuild(this.fleetConfig);
@@ -838,7 +1267,10 @@ export class FleetManager {
838
1267
  const eqIdx = trimmed.indexOf("=");
839
1268
  if (eqIdx < 0)
840
1269
  continue;
841
- const key = trimmed.slice(0, eqIdx);
1270
+ // Accept `export KEY=value` the shell-style form people paste from their
1271
+ // .bashrc. Without this the variable landed in process.env under the key
1272
+ // "export KEY" and silently did nothing.
1273
+ const key = trimmed.slice(0, eqIdx).replace(/^export\s+/, "").trim();
842
1274
  const raw = trimmed.slice(eqIdx + 1);
843
1275
  const value = raw.replace(/^["'](.*)["']$/, '$1');
844
1276
  // .env file always wins over inherited shell env vars, so that
@@ -846,17 +1278,36 @@ export class FleetManager {
846
1278
  process.env[key] = value;
847
1279
  }
848
1280
  }
1281
+ /** Initialize auth before any adapter can answer /dashboard. */
1282
+ initializeWebAuthTokens() {
1283
+ this.webToken = loadOrCreateWebToken(this.dataDir);
1284
+ this.viewToken = randomBytes(24).toString("hex");
1285
+ const viewTokenPath = join(this.dataDir, "view.token");
1286
+ writeFileSync(viewTokenPath, this.viewToken, { encoding: "utf8", mode: 0o600 });
1287
+ try {
1288
+ chmodSync(viewTokenPath, 0o600);
1289
+ }
1290
+ catch { /* best effort */ }
1291
+ this.healthServerListening = false;
1292
+ }
1293
+ getDashboardAccess() {
1294
+ return { ready: this.healthServerListening, token: this.webToken };
1295
+ }
849
1296
  /** Start all instances from fleet config */
850
1297
  async startAll(configPath) {
1298
+ const startupStartedAt = Date.now();
851
1299
  FleetManager.signalTarget = this;
852
1300
  this.startupComplete = false;
1301
+ // Cleared here, not at the end of doStopAll: a stop has an async tail, and
1302
+ // anything arriving during it is still part of the stop.
1303
+ this.shuttingDown = false;
853
1304
  this.configPath = configPath;
854
1305
  this.loadEnvFile();
855
1306
  // Rotate fleet.log if oversized (before any logging)
856
- const { rotateLogIfNeeded } = await import("./logger.js");
857
1307
  rotateLogIfNeeded(join(this.dataDir, "fleet.log"));
858
1308
  const fleet = this.loadConfig(configPath);
859
1309
  setLocale(detectLocale(fleet)); // user-facing text language (fleet.yaml defaults.locale / timezone)
1310
+ this.initializeWebAuthTokens();
860
1311
  const topicMode = fleet.channel?.mode === "topic" || !!fleet.channels?.some(ch => ch.mode === "topic");
861
1312
  // Set tmux socket isolation for custom AGEND_HOME
862
1313
  const { getTmuxSocketName: getSocket } = await import("./paths.js");
@@ -897,7 +1348,7 @@ export class FleetManager {
897
1348
  }
898
1349
  const pidPath = join(this.dataDir, "fleet.pid");
899
1350
  writeFileSync(pidPath, String(process.pid), "utf-8");
900
- this.eventLog = new EventLog(join(this.dataDir, "events.db"));
1351
+ this.eventLog = this.openEventLog();
901
1352
  // Initialize classic channel manager. The primary adapter (channels[0])
902
1353
  // migrates legacy single-bot entries and names without a suffix. Classic
903
1354
  // routing does NOT go through the routing engine (single-key, can't hold two
@@ -986,11 +1437,10 @@ export class FleetManager {
986
1437
  // Rotate fleet.log daily too (besides the startup size check above), so a
987
1438
  // long-running fleet doesn't accumulate an unbounded log.
988
1439
  rotateLogIfNeeded(join(this.dataDir, "fleet.log"));
989
- // Instance output.log is pipe-pane (TUI ANSI). Daemon health ticks also
990
- // rotate it; this is the daily safety net for idle/stopped instances.
991
- for (const name of Object.keys(this.fleetConfig?.instances ?? {})) {
992
- rotateLogIfNeeded(join(this.dataDir, "instances", name, "output.log"));
993
- }
1440
+ // Instance output.log is pipe-pane (TUI ANSI). Daemon health ticks rotate a
1441
+ // running instance's own log; this sweep is the safety net for every other
1442
+ // kind. One implementation, so the two cannot cover different sets.
1443
+ this.rotateAllInstanceLogs();
994
1444
  }, () => {
995
1445
  const instances = Object.keys(this.fleetConfig?.instances ?? {});
996
1446
  const costMap = new Map();
@@ -1005,6 +1455,7 @@ export class FleetManager {
1005
1455
  this.rotateInboxes();
1006
1456
  // Auto-create/adopt a general dispatcher — ONLY for the primary adapter.
1007
1457
  const channelConfigs = fleet.channels ?? (fleet.channel ? [fleet.channel] : []);
1458
+ this.warnUnboundGeneralChannelIds(fleet);
1008
1459
  const primaryAdapterId = channelConfigs[0] ? (channelConfigs[0].id ?? channelConfigs[0].type) : undefined;
1009
1460
  const generalInstances = Object.entries(fleet.instances).filter(([, inst]) => inst.general_topic === true);
1010
1461
  let generalsCreated = false;
@@ -1112,10 +1563,13 @@ export class FleetManager {
1112
1563
  const allEntries = Object.entries(fleet.instances);
1113
1564
  const generals = allEntries.filter(([_, cfg]) => cfg.general_topic);
1114
1565
  const others = allEntries.filter(([_, cfg]) => !cfg.general_topic);
1566
+ const startupProgress = new RestartProgress(this.runnableStartupCount(fleet, topicMode), startupStartedAt, this.logger);
1115
1567
  if (generals.length > 0) {
1116
1568
  for (const [name, cfg] of generals) {
1117
1569
  try {
1118
1570
  await this.startInstance(name, cfg, topicMode);
1571
+ if (this.daemons.has(name))
1572
+ startupProgress.markReady();
1119
1573
  }
1120
1574
  catch (err) {
1121
1575
  this.logger.error({ err, name }, "Failed to start general instance");
@@ -1130,39 +1584,63 @@ export class FleetManager {
1130
1584
  }
1131
1585
  }
1132
1586
  }
1133
- // Signal systemd: generals ready
1134
- sdNotify("READY=1");
1587
+ // The adapter must exist before General can receive the progress message.
1588
+ // Start it after General is ready, in parallel with the remaining CLIs, so
1589
+ // progress is visible without adding adapter startup time to the critical path.
1590
+ let adapterStartup = null;
1591
+ let progressStart = Promise.resolve(false);
1592
+ if (topicMode && (fleet.channel || fleet.channels?.length)) {
1593
+ // An adapter becoming reachable during startup can receive messages; make
1594
+ // all existing topic ids routable before opening that inbound path.
1595
+ this.routing.rebuild(fleet);
1596
+ this.reregisterClassicChannels();
1597
+ adapterStartup = (async () => {
1598
+ try {
1599
+ await this.startSharedAdapter(fleet);
1600
+ }
1601
+ catch (err) {
1602
+ this.logger.error({ err }, "startSharedAdapter failed — fleet continues without some adapters");
1603
+ }
1604
+ })();
1605
+ progressStart = adapterStartup.then(() => startupProgress.start(this.restartProgressTarget()));
1606
+ }
1607
+ // The systemd watchdog answers exactly one question: is this process still
1608
+ // turning its event loop? Pinging from a timer proves that, and after the
1609
+ // blocking child-process calls were made async it is a meaningful signal —
1610
+ // a deadlocked or frozen fleet stops pinging and systemd restarts it.
1611
+ //
1612
+ // It deliberately does NOT gate on fleet health. "No adapter connected" or
1613
+ // "an instance crashed" must not kill the process: the fleet would be restarted
1614
+ // into the same broken state, and a user who has legitimately stopped every
1615
+ // instance would get a restart loop. Those conditions surface through /health
1616
+ // (which now returns 503) and through the General-topic notifications instead.
1135
1617
  this.watchdogTimer = setInterval(() => sdNotify("WATCHDOG=1"), 30_000);
1618
+ // EventLog.prune() existed but was never called, so `events` and `activity`
1619
+ // grew without bound for the life of the install. Prune once at startup and
1620
+ // daily after that; the timer is unref'd so it never holds the loop open.
1621
+ this.pruneEventLog();
1622
+ this.eventLogPruneTimer = setInterval(() => this.pruneEventLog(), 24 * 60 * 60_000);
1623
+ this.eventLogPruneTimer.unref?.();
1624
+ // Same shape for pipe-pane logs, and for the same reason: the only sweep that
1625
+ // covered them lived inside the daily-summary callback, so it did not run at
1626
+ // all when summaries were off.
1627
+ this.rotateAllInstanceLogs();
1628
+ this.logRotateTimer = setInterval(() => this.rotateAllInstanceLogs(), 24 * 60 * 60_000);
1629
+ this.logRotateTimer.unref?.();
1136
1630
  // Phase 2: Start remaining instances with staggered concurrency
1137
1631
  if (others.length > 0) {
1138
- await this.startInstancesWithConcurrency(others, topicMode);
1632
+ await this.startInstancesWithConcurrency(others, topicMode, () => startupProgress.markReady());
1139
1633
  }
1140
1634
  if (topicMode && (fleet.channel || fleet.channels?.length)) {
1141
- try {
1142
- await this.startSharedAdapter(fleet);
1143
- }
1144
- catch (err) {
1145
- this.logger.error({ err }, "startSharedAdapter failed — fleet continues without some adapters");
1146
- }
1147
- // Bind instances to their adapter (which bot answers on their behalf).
1148
- // An explicit channel_id is authoritative — this is how a persona instance
1149
- // picks its bot when several share one guild. Generals without a channel_id
1150
- // fall back to a name-contains-adapterId heuristic.
1151
- const channelConfigsForBind = fleet.channels ?? (fleet.channel ? [fleet.channel] : []);
1635
+ await adapterStartup;
1636
+ // Bind every fleet instance deterministically. Explicit channel_id wins;
1637
+ // otherwise channels[0] is authoritative. Do not infer identity from
1638
+ // concurrent adapter startup or whichever bot receives a message first.
1639
+ const primaryAdapterId = this.getPrimaryAdapterId();
1152
1640
  for (const [name, config] of Object.entries(fleet.instances)) {
1153
- if (config.channel_id) {
1154
- this.bindInstanceAdapter(name, config.channel_id);
1155
- continue;
1156
- }
1157
- if (!config.general_topic)
1158
- continue;
1159
- for (const ch of channelConfigsForBind) {
1160
- const id = ch.id ?? ch.type;
1161
- if (name.includes(id)) {
1162
- this.bindInstanceAdapter(name, id);
1163
- break;
1164
- }
1165
- }
1641
+ const adapterId = config.channel_id ?? primaryAdapterId;
1642
+ if (adapterId)
1643
+ this.bindInstanceAdapter(name, adapterId);
1166
1644
  }
1167
1645
  // Guard against a stale/invalid general topic_id. An old auto-general
1168
1646
  // could have written the TG-convention "1" for a Discord general; the DC
@@ -1172,7 +1650,7 @@ export class FleetManager {
1172
1650
  for (const [name, cfg] of Object.entries(this.fleetConfig.instances)) {
1173
1651
  if (!cfg.general_topic || cfg.topic_id == null)
1174
1652
  continue;
1175
- const adapterId = this.instanceWorldBinding.get(name) ?? cfg.channel_id;
1653
+ const adapterId = this.getInstanceAdapterId(name);
1176
1654
  if (this.getChannelConfig(adapterId)?.type === "discord" && !/^\d{17,}$/.test(String(cfg.topic_id))) {
1177
1655
  this.logger.warn({ name, topic_id: cfg.topic_id }, "Discord general topic_id is not a valid channel — unbinding to avoid a crash loop");
1178
1656
  delete cfg.topic_id;
@@ -1195,12 +1673,22 @@ export class FleetManager {
1195
1673
  // Start classic channel instances (parallel, concurrency 3)
1196
1674
  if (this.classicChannels) {
1197
1675
  const fleetBackend = this.fleetConfig?.defaults?.backend;
1198
- const channels = this.classicChannels.getAll();
1676
+ const channels = this.classicChannels.getAll()
1677
+ .filter(ch => !this.lifecycle.isPaused(ch.instanceName));
1199
1678
  const concurrency = 3;
1200
1679
  let idx = 0;
1201
1680
  while (idx < channels.length) {
1202
1681
  const batch = channels.slice(idx, idx + concurrency);
1203
- await Promise.allSettled(batch.map(ch => this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after)).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance"))));
1682
+ await Promise.allSettled(batch.map(async (ch) => {
1683
+ try {
1684
+ await this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
1685
+ if (this.daemons.has(ch.instanceName))
1686
+ startupProgress.markReady();
1687
+ }
1688
+ catch (err) {
1689
+ this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance");
1690
+ }
1691
+ }));
1204
1692
  idx += concurrency;
1205
1693
  }
1206
1694
  }
@@ -1208,10 +1696,10 @@ export class FleetManager {
1208
1696
  this.startStatuslineWatcher(name);
1209
1697
  }
1210
1698
  // Notify General topic that fleet is up
1211
- const classicCount = this.classicChannels?.getAll().length ?? 0;
1212
- const total = Object.keys(fleet.instances).length + classicCount;
1213
- const started = this.daemons.size;
1214
- const allNotRunning = Object.keys(fleet.instances).filter(n => !this.daemons.has(n));
1699
+ const configuredNames = this.configuredStartupInstanceNames(fleet, topicMode);
1700
+ const total = configuredNames.length;
1701
+ const started = configuredNames.filter(name => this.daemons.has(name)).length;
1702
+ const allNotRunning = configuredNames.filter(name => !this.daemons.has(name));
1215
1703
  const pausedNames = allNotRunning.filter(n => this.lifecycle.isPaused(n));
1216
1704
  const failedNames = allNotRunning.filter(n => !this.lifecycle.isPaused(n));
1217
1705
  const generalName = this.findGeneralInstance();
@@ -1219,7 +1707,15 @@ export class FleetManager {
1219
1707
  const { createRequire } = await import("node:module");
1220
1708
  const _require = createRequire(import.meta.url);
1221
1709
  const agendVersion = _require("../package.json").version ?? "unknown";
1222
- if (this.adapter && fleet.channel?.group_id) {
1710
+ await progressStart;
1711
+ const progressCompleted = await startupProgress.finish({
1712
+ running: started,
1713
+ total,
1714
+ version: agendVersion,
1715
+ pausedNames,
1716
+ failedNames,
1717
+ });
1718
+ if (!progressCompleted && this.adapter && fleet.channel?.group_id) {
1223
1719
  let text;
1224
1720
  if (failedNames.length === 0 && pausedNames.length === 0) {
1225
1721
  text = t("fleet.ready", started, total, agendVersion);
@@ -1268,6 +1764,15 @@ export class FleetManager {
1268
1764
  // rest of startup finishes. Replay one coalesced reload only after all
1269
1765
  // startup-owned lifecycle work and signal handlers are in place.
1270
1766
  this.finishStartup();
1767
+ // Tell systemd we are ready only now. This used to fire right after the
1768
+ // generals started — before adapters, classic instances, topic creation and the
1769
+ // health server — so `systemctl start` returned success while the fleet was
1770
+ // still deaf: no path existed for a user message to arrive.
1771
+ sdNotify("READY=1");
1772
+ const health = this.getFleetHealth();
1773
+ if (health.status !== "ok") {
1774
+ this.logger.warn({ health }, "Fleet started with problems — see /health");
1775
+ }
1271
1776
  }
1272
1777
  /**
1273
1778
  * Delete inbox files older than retentionDays (by mtime). Cleans the shared
@@ -1384,6 +1889,67 @@ export class FleetManager {
1384
1889
  getAdapterStates() {
1385
1890
  return this.adapterState;
1386
1891
  }
1892
+ /**
1893
+ * Real, checkable fleet health for `/health` and the operator.
1894
+ *
1895
+ * `status` is:
1896
+ * - `ok` — at least one adapter connected and every configured instance
1897
+ * that should be running is running
1898
+ * - `degraded` — reachable, but something the operator should look at (an
1899
+ * adapter retrying, an instance crashed or stopped)
1900
+ * - `down` — the fleet cannot do its job: no adapter is connected, so no
1901
+ * message can arrive or be answered
1902
+ *
1903
+ * Deliberately does NOT gate the systemd watchdog — see the comment at the
1904
+ * WATCHDOG timer for why.
1905
+ */
1906
+ getFleetHealth() {
1907
+ const names = Object.keys(this.fleetConfig?.instances ?? {});
1908
+ const counts = { configured: names.length, running: 0, crashed: 0, paused: 0, stopped: 0 };
1909
+ for (const name of names) {
1910
+ const state = this.getInstanceStatus(name);
1911
+ if (state === "running")
1912
+ counts.running++;
1913
+ else if (state === "crashed")
1914
+ counts.crashed++;
1915
+ else if (state === "paused")
1916
+ counts.paused++;
1917
+ else
1918
+ counts.stopped++;
1919
+ }
1920
+ const states = {};
1921
+ let connected = 0;
1922
+ for (const [id, state] of this.adapterState) {
1923
+ states[id] = state.status;
1924
+ if (state.status === "connected")
1925
+ connected++;
1926
+ }
1927
+ const problems = [];
1928
+ if (this.adapterState.size > 0 && connected === 0)
1929
+ problems.push("no channel adapter is connected");
1930
+ if (counts.crashed > 0)
1931
+ problems.push(`${counts.crashed} instance(s) crashed`);
1932
+ for (const [id, state] of this.adapterState) {
1933
+ if (state.status !== "connected")
1934
+ problems.push(`adapter ${id} is ${state.status}`);
1935
+ }
1936
+ if (!this.startupComplete)
1937
+ problems.push("startup has not completed");
1938
+ // "down" is reserved for "cannot receive or answer a message at all". A fleet
1939
+ // with adapters configured but none connected is exactly that.
1940
+ const status = this.adapterState.size > 0 && connected === 0
1941
+ ? "down"
1942
+ : problems.length > 0 ? "degraded" : "ok";
1943
+ return {
1944
+ status,
1945
+ uptime: Math.floor((Date.now() - this.startedAt) / 1000),
1946
+ instances: counts,
1947
+ adapters: { total: this.adapterState.size, connected, states },
1948
+ startupComplete: this.startupComplete,
1949
+ memory: readFleetMemory(),
1950
+ problems,
1951
+ };
1952
+ }
1387
1953
  /** Start the primary adapter (backward-compatible, sets this.adapter) */
1388
1954
  async startSingleAdapter(fleet, channelConfig) {
1389
1955
  const botToken = process.env[channelConfig.bot_token_env];
@@ -1410,11 +1976,16 @@ export class FleetManager {
1410
1976
  this.adapter.on("message", safeHandler(async (msg) => {
1411
1977
  await this.handleInboundMessage(msg);
1412
1978
  }, this.logger, "adapter.message"));
1979
+ this.adapter.on("reaction", safeHandler(async (r) => {
1980
+ await this.handleInboundReaction(r);
1981
+ }, this.logger, "adapter.reaction"));
1413
1982
  this.adapter.on("callback_query", safeHandler(async (data) => {
1414
1983
  if (await this.handleClassicBackendSelection(data))
1415
1984
  return;
1416
1985
  if (await this.handleModelSelection(data))
1417
1986
  return;
1987
+ if (await this.handleEffortSelection(data))
1988
+ return;
1418
1989
  if (data.callbackData.startsWith("hang:")) {
1419
1990
  const parts = data.callbackData.split(":");
1420
1991
  const action = parts[1];
@@ -1435,12 +2006,7 @@ export class FleetManager {
1435
2006
  return;
1436
2007
  }
1437
2008
  if (data.callbackData.startsWith("cancel:")) {
1438
- const instanceName = data.callbackData.slice("cancel:".length);
1439
- // Idempotent: a button click only acts while the button is live. A
1440
- // second click (entry already cleared) is a no-op — don't re-send the
1441
- // interrupt key. (The /cancel command path calls cancelInstance directly.)
1442
- if (this.hasCancelButton(instanceName))
1443
- this.cancelInstance(instanceName);
2009
+ this.handleCancelClick(data.callbackData.slice("cancel:".length), this.adapter, data);
1444
2010
  return;
1445
2011
  }
1446
2012
  }, this.logger, "adapter.callback_query"));
@@ -1520,6 +2086,9 @@ export class FleetManager {
1520
2086
  else if (data.command === "model") {
1521
2087
  await this.handleModelSlash(data, adapterId);
1522
2088
  }
2089
+ else if (data.command === "effort") {
2090
+ await this.handleEffortSlash(data, adapterId);
2091
+ }
1523
2092
  else if (data.command === "cancel") {
1524
2093
  const name = this.resolveSlashTarget(data.channelId, adapterId);
1525
2094
  if (!name) {
@@ -1584,19 +2153,29 @@ export class FleetManager {
1584
2153
  await data.respond(t("not_authorized"));
1585
2154
  return;
1586
2155
  }
2156
+ await data.respond(await this.runBackendDoctor());
2157
+ }
2158
+ else if (data.command === "usage") {
2159
+ // Same permission level as /ctx (none). The reply is still ephemeral —
2160
+ // the adapter defers non-chat commands that way — so it never spams the
2161
+ // channel either way.
1587
2162
  try {
1588
- const { execSync } = await import("node:child_process");
1589
- const backend = this.fleetConfig?.defaults?.backend || "claude-code";
1590
- const result = execSync(`agend backend doctor ${backend}`, { timeout: 30_000, encoding: "utf-8" });
1591
- const clean = result.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
1592
- await data.respond(clean || "No output");
2163
+ const { getUsageSnapshot } = await import("./usage/usage-api.js");
2164
+ const { renderUsageMarkdown } = await import("./usage/format-rich.js");
2165
+ // slash_command is Discord-only; editReply renders Markdown natively.
2166
+ await data.respond(renderUsageMarkdown(await getUsageSnapshot()));
1593
2167
  }
1594
2168
  catch (err) {
1595
- const output = (err.stdout ?? err.message ?? "Doctor failed").replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
1596
- await data.respond(output);
2169
+ await data.respond(`⚠️ Usage fetch failed: ${err.message}`);
1597
2170
  }
1598
2171
  }
1599
2172
  else if (data.command === "status") {
2173
+ // Admin-gated (like the topic path): the merged table shows every
2174
+ // instance's cost and IPC health.
2175
+ if (!this.isFleetAdmin(data.userId, adapterId)) {
2176
+ await data.respond(t("cmd.admin_required", "/status"));
2177
+ return;
2178
+ }
1600
2179
  const text = await this.topicCommands.getStatusText();
1601
2180
  await data.respond(text);
1602
2181
  }
@@ -1642,7 +2221,9 @@ export class FleetManager {
1642
2221
  this.probeCliEnvs();
1643
2222
  this.adapter.on("started", safeHandler((username, userId) => {
1644
2223
  this.logger.info(`Bot @${username} polling started. Ensure no other service is polling this bot token.`);
1645
- const w = this.worlds.values().next().value;
2224
+ // Concurrent startup can insert a secondary world first. Update the
2225
+ // configured primary world, not Map insertion order.
2226
+ const w = this.worlds.get(adapterId);
1646
2227
  if (w) {
1647
2228
  w.botUsername = username;
1648
2229
  if (userId)
@@ -1705,11 +2286,16 @@ export class FleetManager {
1705
2286
  adapter.on("message", safeHandler(async (msg) => {
1706
2287
  await this.handleInboundMessage(msg);
1707
2288
  }, this.logger, `adapter[${adapterId}].message`));
2289
+ adapter.on("reaction", safeHandler(async (r) => {
2290
+ await this.handleInboundReaction(r);
2291
+ }, this.logger, `adapter[${adapterId}].reaction`));
1708
2292
  adapter.on("callback_query", safeHandler(async (data) => {
1709
2293
  if (await this.handleClassicBackendSelection(data))
1710
2294
  return;
1711
2295
  if (await this.handleModelSelection(data))
1712
2296
  return;
2297
+ if (await this.handleEffortSelection(data))
2298
+ return;
1713
2299
  if (data.callbackData.startsWith("hang:")) {
1714
2300
  const parts = data.callbackData.split(":");
1715
2301
  const action = parts[1];
@@ -1729,10 +2315,7 @@ export class FleetManager {
1729
2315
  return;
1730
2316
  }
1731
2317
  if (data.callbackData.startsWith("cancel:")) {
1732
- const instanceName = data.callbackData.slice("cancel:".length);
1733
- // Idempotent: only the first click (while the button is live) acts.
1734
- if (this.hasCancelButton(instanceName))
1735
- this.cancelInstance(instanceName);
2318
+ this.handleCancelClick(data.callbackData.slice("cancel:".length), adapter, data);
1736
2319
  return;
1737
2320
  }
1738
2321
  }, this.logger, `adapter[${adapterId}].callback_query`));
@@ -1802,6 +2385,9 @@ export class FleetManager {
1802
2385
  else if (data.command === "model") {
1803
2386
  await this.handleModelSlash(data, adapterId);
1804
2387
  }
2388
+ else if (data.command === "effort") {
2389
+ await this.handleEffortSlash(data, adapterId);
2390
+ }
1805
2391
  else if (data.command === "cancel") {
1806
2392
  const name = this.resolveSlashTarget(data.channelId, adapterId);
1807
2393
  if (!name) {
@@ -1866,19 +2452,29 @@ export class FleetManager {
1866
2452
  await data.respond(t("not_authorized"));
1867
2453
  return;
1868
2454
  }
2455
+ await data.respond(await this.runBackendDoctor());
2456
+ }
2457
+ else if (data.command === "usage") {
2458
+ // Same permission level as /ctx (none). The reply is still ephemeral —
2459
+ // the adapter defers non-chat commands that way — so it never spams the
2460
+ // channel either way.
1869
2461
  try {
1870
- const { execSync } = await import("node:child_process");
1871
- const backend = this.fleetConfig?.defaults?.backend || "claude-code";
1872
- const result = execSync(`agend backend doctor ${backend}`, { timeout: 30_000, encoding: "utf-8" });
1873
- const clean = result.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
1874
- await data.respond(clean || "No output");
2462
+ const { getUsageSnapshot } = await import("./usage/usage-api.js");
2463
+ const { renderUsageMarkdown } = await import("./usage/format-rich.js");
2464
+ // slash_command is Discord-only; editReply renders Markdown natively.
2465
+ await data.respond(renderUsageMarkdown(await getUsageSnapshot()));
1875
2466
  }
1876
2467
  catch (err) {
1877
- const output = (err.stdout ?? err.message ?? "Doctor failed").replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
1878
- await data.respond(output);
2468
+ await data.respond(`⚠️ Usage fetch failed: ${err.message}`);
1879
2469
  }
1880
2470
  }
1881
2471
  else if (data.command === "status") {
2472
+ // Admin-gated (like the topic path): the merged table shows every
2473
+ // instance's cost and IPC health.
2474
+ if (!this.isFleetAdmin(data.userId, adapterId)) {
2475
+ await data.respond(t("cmd.admin_required", "/status"));
2476
+ return;
2477
+ }
1882
2478
  const text = await this.topicCommands.getStatusText();
1883
2479
  await data.respond(text);
1884
2480
  }
@@ -2034,6 +2630,9 @@ export class FleetManager {
2034
2630
  else if (msg.type === "instance_process_state") {
2035
2631
  this.cacheInstanceProcessStatus(name, msg.status);
2036
2632
  }
2633
+ else if (msg.type === "instance_activity") {
2634
+ this.cacheInstanceActivity(name, msg.activity);
2635
+ }
2037
2636
  else if (msg.type === "instance_state" || msg.type === "instance_state_response") {
2038
2637
  this.cacheInstanceExecutionState(name, msg);
2039
2638
  if (msg.type === "instance_state_response") {
@@ -2104,9 +2703,22 @@ export class FleetManager {
2104
2703
  if (existsSync(windowIdPath)) {
2105
2704
  const windowId = readFileSync(windowIdPath, "utf-8").trim();
2106
2705
  if (windowId) {
2706
+ // Async with an explicit timeout: this was execSync with NO timeout at
2707
+ // all, so a wedged tmux server blocked the whole fleet event loop
2708
+ // indefinitely — while we were here to diagnose a lost connection.
2709
+ // A timeout is also the correct signal: an unresponsive tmux server
2710
+ // means we cannot verify the pane, which is treated as dead (the same
2711
+ // conclusion the old code reached only by throwing).
2107
2712
  try {
2108
- const { execSync } = await import("node:child_process");
2109
- execSync(`tmux list-panes -t "${windowId}"`, { stdio: "ignore" });
2713
+ const { execFile } = await import("node:child_process");
2714
+ const { promisify } = await import("node:util");
2715
+ const { getTmuxSocketName } = await import("./paths.js");
2716
+ // Honour socket isolation: without -L this queried the user's default
2717
+ // tmux server instead of the fleet's, so under a custom AGEND_HOME the
2718
+ // check was meaningless (it reported every pane dead).
2719
+ const socket = getTmuxSocketName();
2720
+ const args = socket ? ["-L", socket, "list-panes", "-t", windowId] : ["list-panes", "-t", windowId];
2721
+ await promisify(execFile)("tmux", args, { timeout: 5_000 });
2110
2722
  }
2111
2723
  catch {
2112
2724
  // Pane dead — respawn
@@ -2127,6 +2739,12 @@ export class FleetManager {
2127
2739
  if (this.adapterRestarting.has(id))
2128
2740
  return;
2129
2741
  this.adapterRestarting.add(id);
2742
+ // Reflect reality in adapterState throughout. This loop used to leave the state
2743
+ // untouched, so getAdapterStates() — and therefore /health and the dashboard —
2744
+ // kept reporting "connected" for an adapter that had been down for hours. An
2745
+ // adapter's true status was simply not knowable from inside the process.
2746
+ const previous = this.adapterState.get(id);
2747
+ this.adapterState.set(id, { status: "retrying", retryCount: 0, lastError: previous?.lastError });
2130
2748
  try {
2131
2749
  for (let attempt = 1;; attempt++) {
2132
2750
  if (this.ipcStoppingInstances.has("__fleet_stopping__"))
@@ -2139,9 +2757,16 @@ export class FleetManager {
2139
2757
  await adapter.stop().catch(() => { });
2140
2758
  await adapter.start();
2141
2759
  this.logger.info({ id, attempt }, "Adapter restarted successfully");
2760
+ this.adapterState.set(id, { status: "connected", retryCount: 0 });
2142
2761
  return;
2143
2762
  }
2144
- catch { /* retry */ }
2763
+ catch (err) {
2764
+ this.adapterState.set(id, {
2765
+ status: "retrying",
2766
+ retryCount: attempt,
2767
+ lastError: err?.message ?? String(err),
2768
+ });
2769
+ }
2145
2770
  if (attempt % 10 === 0) {
2146
2771
  this.logger.warn({ id, attempt }, "Adapter restart still failing");
2147
2772
  }
@@ -2177,6 +2802,49 @@ export class FleetManager {
2177
2802
  }
2178
2803
  return generals[0];
2179
2804
  }
2805
+ /**
2806
+ * A user reacted to one of the bot's messages (#408).
2807
+ *
2808
+ * A reaction is context, not a message (#432, reworking #413): it never triggers
2809
+ * an agent turn and never wakes anything. It is queued in the event log and rides
2810
+ * into the instance's NEXT real message as one compact leading line —
2811
+ * `[Recent reactions: 👍×2 from hanhanv]` — after which it is marked consumed.
2812
+ * No pending reactions → no line → zero context spent, which is the common case.
2813
+ */
2814
+ async handleInboundReaction(r) {
2815
+ const instanceName = this.resolveSlashTarget(r.threadId ?? r.chatId, r.adapterId);
2816
+ if (!instanceName) {
2817
+ this.logger.debug({ emoji: r.emoji, chatId: r.chatId }, "Reaction in an unrouted channel — ignoring");
2818
+ return;
2819
+ }
2820
+ if (DELIVERY_STATUS_EMOJIS.has(r.emoji)) {
2821
+ this.logger.debug({ emoji: r.emoji, user: r.username }, "Ignoring delivery-status emoji as a reaction");
2822
+ return;
2823
+ }
2824
+ this.eventLog?.logActivity("reaction", r.username, `${r.emoji} ${r.action}`, instanceName);
2825
+ if (r.action === "add") {
2826
+ this.eventLog?.addReaction(instanceName, r.messageId, r.username, r.emoji);
2827
+ }
2828
+ else {
2829
+ // Withdrawn before anyone saw it → it never happened. See removeReaction.
2830
+ this.eventLog?.removeReaction(instanceName, r.messageId, r.username, r.emoji);
2831
+ }
2832
+ }
2833
+ /**
2834
+ * The queued-reaction summary for an instance's next real message, or {} when
2835
+ * nothing is pending (the common case must add zero context). The consume
2836
+ * callback is separate from the fetch so reactions are only marked once the
2837
+ * message actually went out — a failed delivery keeps them queued.
2838
+ */
2839
+ pendingReactionsMeta(instanceName) {
2840
+ const pending = this.eventLog?.pendingReactions(instanceName);
2841
+ if (!pending)
2842
+ return { meta: {}, consume: () => { } };
2843
+ return {
2844
+ meta: { pending_reactions: pending.summary },
2845
+ consume: () => this.eventLog?.markReactionsConsumed(instanceName, pending.maxId),
2846
+ };
2847
+ }
2180
2848
  async handleInboundMessage(msg) {
2181
2849
  const threadId = msg.threadId || undefined;
2182
2850
  this.logger.debug({ source: msg.source, chatId: msg.chatId, threadId, userId: msg.userId, isBotMessage: msg.isBotMessage, textLen: (msg.text ?? "").length, text: (msg.text ?? "").slice(0, 80) }, "handleInboundMessage entry");
@@ -2514,6 +3182,7 @@ export class FleetManager {
2514
3182
  }
2515
3183
  this.warnIfRateLimited(generalInstance, msg);
2516
3184
  const { text, extraMeta } = await processAttachments(msg, inboundAdapter, this.logger, generalInstance);
3185
+ const generalReactions = this.pendingReactionsMeta(generalInstance);
2517
3186
  try {
2518
3187
  await this.deliverToInstance(generalInstance, {
2519
3188
  type: "fleet_inbound",
@@ -2526,12 +3195,18 @@ export class FleetManager {
2526
3195
  user_id: msg.userId,
2527
3196
  ts: msg.timestamp.toISOString(),
2528
3197
  thread_id: "",
2529
- adapter_id: msg.adapterId,
3198
+ // Fleet instances have an authoritative adapter binding. Multiple
3199
+ // bots in one guild can observe the same inbound message, so the
3200
+ // adapter whose event wins dedup is not necessarily the bot that
3201
+ // owns this instance.
3202
+ adapter_id: this.getInstanceAdapterId(generalInstance) ?? msg.adapterId,
2530
3203
  source: msg.source,
2531
3204
  ...(msg.replyToText ? { reply_to_text: msg.replyToText } : {}),
3205
+ ...generalReactions.meta,
2532
3206
  ...extraMeta,
2533
3207
  },
2534
3208
  });
3209
+ generalReactions.consume();
2535
3210
  this.lastInboundUser.set(generalInstance, msg.username);
2536
3211
  this.logger.info(`${msg.username} → ${generalInstance}: ${(text ?? "").slice(0, 100)}`);
2537
3212
  this.eventLog?.logActivity("message", msg.username, (text ?? "").slice(0, 200), generalInstance);
@@ -2607,6 +3282,7 @@ export class FleetManager {
2607
3282
  this.setTopicIcon(instanceName, "blue");
2608
3283
  this.warnIfRateLimited(instanceName, msg);
2609
3284
  const { text, extraMeta } = await processAttachments(msg, inboundAdapter, this.logger, instanceName);
3285
+ const reactions = this.pendingReactionsMeta(instanceName);
2610
3286
  try {
2611
3287
  await this.deliverToInstance(instanceName, {
2612
3288
  type: "fleet_inbound",
@@ -2619,12 +3295,18 @@ export class FleetManager {
2619
3295
  user_id: msg.userId,
2620
3296
  ts: msg.timestamp.toISOString(),
2621
3297
  thread_id: msg.threadId ?? "",
2622
- adapter_id: msg.adapterId,
3298
+ // Canonicalize the reply context to the configured world. Whichever
3299
+ // sibling bot wins inbound dedup must not decide which bot replies.
3300
+ adapter_id: this.getInstanceAdapterId(instanceName) ?? msg.adapterId,
2623
3301
  source: msg.source,
2624
3302
  ...(msg.replyToText ? { reply_to_text: msg.replyToText } : {}),
3303
+ ...reactions.meta,
2625
3304
  ...extraMeta,
2626
3305
  },
2627
3306
  });
3307
+ // Only after the message actually went out. A failed delivery keeps the
3308
+ // reactions queued for the retry / the next message.
3309
+ reactions.consume();
2628
3310
  }
2629
3311
  catch (err) {
2630
3312
  this.logger.warn({ err: err.message, instanceName }, "Wake/delivery failed");
@@ -2699,8 +3381,21 @@ export class FleetManager {
2699
3381
  : (senderSessionName ? undefined : this.fleetConfig?.instances[instanceName]);
2700
3382
  let threadId = resolveReplyThreadId(args.thread_id, routingConfig)
2701
3383
  ?? this.classicChannels?.getChannelIdByInstance(senderInstanceName ?? instanceName);
2702
- // Select adapter: use instance binding, or resolve from chatId in args
2703
- const outAdapter = this.getAdapterForInstance(senderInstanceName ?? instanceName) ?? this.adapter;
3384
+ // Select the adapter from the daemon's exact last-inbound context. Message
3385
+ // ids are scoped to that bot/world; routing a secondary-world id through the
3386
+ // primary adapter produces a 404. Instance binding remains the compatibility
3387
+ // fallback for older daemons and calls without a live/persisted context.
3388
+ const contextAdapterId = typeof msg.adapterId === "string" && msg.adapterId
3389
+ ? msg.adapterId
3390
+ : undefined;
3391
+ const contextWorld = contextAdapterId ? this.worlds.get(contextAdapterId) : undefined;
3392
+ if (contextAdapterId && !contextWorld) {
3393
+ respond(null, `Adapter world unavailable: ${contextAdapterId}`);
3394
+ return;
3395
+ }
3396
+ const outAdapter = contextWorld?.adapter
3397
+ ?? this.getAdapterForInstance(senderInstanceName ?? instanceName)
3398
+ ?? this.adapter;
2704
3399
  if (!outAdapter) {
2705
3400
  respond(null, "No adapter available");
2706
3401
  return;
@@ -2713,25 +3408,33 @@ export class FleetManager {
2713
3408
  delete args.thread_id;
2714
3409
  threadId = undefined;
2715
3410
  }
3411
+ // Reply dedup: retries land here when the agent was told a send failed
3412
+ // (daemon budget elapsed, shell tool killed) while the adapter send was
3413
+ // still in flight and about to succeed. One real send, everyone gets its
3414
+ // outcome; a genuinely failed send clears the entry so a retry passes.
3415
+ if (tool === "reply") {
3416
+ const ticket = this.replyDeduper.begin(instanceName, String(args.text ?? ""), Array.isArray(args.files) ? args.files : []);
3417
+ if (ticket.duplicate) {
3418
+ this.logger.info({ instanceName }, "Duplicate reply suppressed — replaying the original send's outcome");
3419
+ ticket.subscribe(respond);
3420
+ return;
3421
+ }
3422
+ const original = respond;
3423
+ const respondAndRecord = (result, error) => {
3424
+ ticket.complete(result, error);
3425
+ original(result, error);
3426
+ };
3427
+ if (routeToolCall(outAdapter, tool, args, threadId, respondAndRecord)) {
3428
+ this.afterReplyRouted(instanceName, args, senderSessionName);
3429
+ return;
3430
+ }
3431
+ // routeToolCall knows "reply"; not handling it means the world changed.
3432
+ ticket.complete(null, "reply not handled");
3433
+ original(null, "reply not handled");
3434
+ return;
3435
+ }
2716
3436
  // Route standard channel tools (reply, react, edit_message, download_attachment)
2717
3437
  if (routeToolCall(outAdapter, tool, args, threadId, respond)) {
2718
- if (tool === "reply") {
2719
- // Agent answered — retire its pending cancel button and mark ✅ done.
2720
- this.clearCancelButton(instanceName);
2721
- this.reactDone(instanceName);
2722
- const replyTo = this.lastInboundUser.get(instanceName) ?? "user";
2723
- this.logger.info(`${instanceName} → ${replyTo}: ${(args.text ?? "").slice(0, 100)}`);
2724
- this.emitSseEvent("message", {
2725
- instance: instanceName, sender: senderSessionName ?? instanceName,
2726
- text: (args.text ?? "").slice(0, 2000),
2727
- ts: new Date().toISOString(),
2728
- });
2729
- // Log bot reply to classic instance chat-log
2730
- const isClassic = this.classicChannels?.getChannelIdByInstance(instanceName) !== undefined;
2731
- if (isClassic) {
2732
- ClassicChannelManager.logMessage(instanceName, "bot", args.text ?? "", new Date());
2733
- }
2734
- }
2735
3438
  return;
2736
3439
  }
2737
3440
  // Log tool calls for activity visualization
@@ -2746,6 +3449,35 @@ export class FleetManager {
2746
3449
  respond(null, `Unknown tool: ${tool}`);
2747
3450
  }
2748
3451
  }
3452
+ /** Side effects of a routed reply: cancel-button lifecycle, logs, SSE, chat log. */
3453
+ afterReplyRouted(instanceName, args, senderSessionName) {
3454
+ // A reply is NOT proof the turn is over (#410) — but it is not proof of
3455
+ // more work either. Split the difference: an instance that is clearly
3456
+ // idle loses the button now; one that looks busy keeps it (re-posted
3457
+ // below the reply so it stays last in the channel), with a 2-minute
3458
+ // grace check — if it has NOT resumed working by then, the reply was the
3459
+ // end of the turn and the button goes. A multi-step run that keeps
3460
+ // working sails through the check and keeps its button.
3461
+ if (this.getInstanceIdle(instanceName)) {
3462
+ this.clearCancelButton(instanceName);
3463
+ }
3464
+ else {
3465
+ void this.sendCancelButton(instanceName).then(() => this.armReplyGrace(instanceName));
3466
+ }
3467
+ this.reactDone(instanceName);
3468
+ const replyTo = this.lastInboundUser.get(instanceName) ?? "user";
3469
+ this.logger.info(`${instanceName} → ${replyTo}: ${(args.text ?? "").slice(0, 100)}`);
3470
+ this.emitSseEvent("message", {
3471
+ instance: instanceName, sender: senderSessionName ?? instanceName,
3472
+ text: (args.text ?? "").slice(0, 2000),
3473
+ ts: new Date().toISOString(),
3474
+ });
3475
+ // Log bot reply to classic instance chat-log
3476
+ const isClassic = this.classicChannels?.getChannelIdByInstance(instanceName) !== undefined;
3477
+ if (isClassic) {
3478
+ ClassicChannelManager.logMessage(instanceName, "bot", args.text ?? "", new Date());
3479
+ }
3480
+ }
2749
3481
  /** Handle tool status update from a daemon instance */
2750
3482
  handleToolStatusFromInstance(instanceName, msg) {
2751
3483
  const statusAdapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
@@ -3283,7 +4015,12 @@ export class FleetManager {
3283
4015
  // the before/after runtime diff cannot observe.
3284
4016
  for (const patch of explicitPatches) {
3285
4017
  if (patch.remove) {
3286
- this.rawFleetDocument.deleteIn(patch.path);
4018
+ // YAML's deleteIn throws when an inherited nested key has no raw parent
4019
+ // (or a legacy scalar occupies that parent). Removing an override which
4020
+ // is already absent is an idempotent no-op, not a failed Settings save.
4021
+ if (this.rawFleetDocument.hasIn(patch.path)) {
4022
+ this.rawFleetDocument.deleteIn(patch.path);
4023
+ }
3287
4024
  }
3288
4025
  else {
3289
4026
  const before = this.rawFleetDocument.getIn(patch.path);
@@ -3309,8 +4046,11 @@ export class FleetManager {
3309
4046
  this.patchFleetDocument(document, [...path, i], before[i], after[i]);
3310
4047
  }
3311
4048
  // Remove from the end so YAML sequence indexes do not shift underneath us.
3312
- for (let i = before.length - 1; i >= after.length; i--)
3313
- document.deleteIn([...path, i]);
4049
+ for (let i = before.length - 1; i >= after.length; i--) {
4050
+ const itemPath = [...path, i];
4051
+ if (document.hasIn(itemPath))
4052
+ document.deleteIn(itemPath);
4053
+ }
3314
4054
  for (let i = shared; i < after.length; i++)
3315
4055
  document.setIn([...path, i], after[i]);
3316
4056
  return;
@@ -3332,7 +4072,11 @@ export class FleetManager {
3332
4072
  return;
3333
4073
  }
3334
4074
  if (after === undefined) {
3335
- document.deleteIn(path);
4075
+ // Effective config contains inherited objects that may not exist in the
4076
+ // raw YAML at all. yaml.deleteIn() is not idempotent for a missing nested
4077
+ // parent, so guard it explicitly.
4078
+ if (document.hasIn(path))
4079
+ document.deleteIn(path);
3336
4080
  }
3337
4081
  else if (path.length === 0) {
3338
4082
  document.contents = document.createNode(after);
@@ -3348,6 +4092,9 @@ export class FleetManager {
3348
4092
  }
3349
4093
  }
3350
4094
  async removeInstance(name) {
4095
+ // Drop cached pane context — the map is keyed by instance name and nothing
4096
+ // else evicted deleted entries, so it grew for the life of the process.
4097
+ forgetInstanceContext(name);
3351
4098
  // Clean up schedules (scheduler is fleet-level, not lifecycle-level)
3352
4099
  const config = this.fleetConfig?.instances[name];
3353
4100
  if (this.scheduler && config?.topic_id) {
@@ -3453,45 +4200,210 @@ export class FleetManager {
3453
4200
  this.collabInstances.add(instanceName);
3454
4201
  return true;
3455
4202
  }
3456
- notifyInstanceTopic(instanceName, text, extraOpts) {
3457
- const adapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
3458
- if (!adapter)
3459
- return;
3460
- const channelCfg = this.getChannelConfig(this.instanceWorldBinding.get(instanceName));
3461
- const groupId = channelCfg?.group_id;
3462
- // Fleet topic instance
3463
- const threadId = this.fleetConfig?.instances[instanceName]?.topic_id;
3464
- if (threadId != null && groupId) {
3465
- adapter.sendText(String(groupId), text, { threadId: String(threadId), ...extraOpts })
3466
- .catch(e => this.logger.warn({ err: e, instanceName }, "Failed to send instance topic notification"));
3467
- return;
3468
- }
3469
- // Classic instance: find its channelId from the classic manager
3470
- const classicChatId = this.classicChannels?.getChannelIdByInstance(instanceName);
3471
- if (classicChatId) {
3472
- adapter.sendText(classicChatId, text, extraOpts)
3473
- .catch(e => this.logger.warn({ err: e, instanceName }, "Failed to send classic notification"));
3474
- return;
4203
+ /**
4204
+ * Open the event log, tolerating a corrupt file.
4205
+ *
4206
+ * `events.db` holds history only — event rows and the activity feed. Nothing the
4207
+ * fleet needs to run depends on it, and every consumer already uses
4208
+ * `this.eventLog?.`. An unguarded `new EventLog(...)` here meant a corrupt or
4209
+ * unreadable history file (a truncated WAL after a hard kill, a full disk)
4210
+ * threw during startAll and the WHOLE FLEET FAILED TO BOOT — trading every
4211
+ * running agent for a file whose only job is reporting.
4212
+ *
4213
+ * So: try, move a bad file aside and retry once with a fresh one, and if even
4214
+ * that fails carry on without an event log.
4215
+ */
4216
+ /**
4217
+ * Run `agend backend doctor` for the fleet's default backend and return its
4218
+ * cleaned output.
4219
+ *
4220
+ * Async on purpose: this was `execSync` with a 30s timeout, reachable by any
4221
+ * allowlisted user through `/doctor`. While it ran, the entire fleet event loop
4222
+ * was frozen — no IPC, no adapter, no message delivery, no health responses,
4223
+ * and critically no WATCHDOG ping, so a slow doctor could push past
4224
+ * WatchdogSec and have systemd SIGABRT the fleet.
4225
+ */
4226
+ async runBackendDoctor() {
4227
+ const stripAnsi = (s) => s.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
4228
+ const backend = this.fleetConfig?.defaults?.backend || "claude-code";
4229
+ try {
4230
+ const { execFile } = await import("node:child_process");
4231
+ const { promisify } = await import("node:util");
4232
+ // execFile with an argv array — no shell, so the backend name cannot be
4233
+ // interpreted as a command even if config is malformed.
4234
+ const { stdout } = await promisify(execFile)("agend", ["backend", "doctor", backend], {
4235
+ timeout: 30_000,
4236
+ encoding: "utf-8",
4237
+ });
4238
+ return stripAnsi(stdout) || "No output";
3475
4239
  }
3476
- // Fallback: send to group without threadId
3477
- if (groupId) {
3478
- adapter.sendText(String(groupId), text, extraOpts)
3479
- .catch(e => this.logger.warn({ err: e, instanceName }, "Failed to send notification (no topic)"));
4240
+ catch (err) {
4241
+ const e = err;
4242
+ return stripAnsi(e.stdout ?? e.message ?? "Doctor failed");
3480
4243
  }
3481
4244
  }
3482
- // ── Cancel button ────────────────────────────────────────────────────
3483
- // Sent after delivering a user message to an instance; clicking it (or
3484
- // /cancel) sends Escape to the instance's pane to interrupt generation.
3485
- /** Send a "🛑 Cancel" button to the instance's topic/channel after delivery. */
4245
+ /** Drop event/activity rows older than the retention window. Best-effort. */
3486
4246
  /**
3487
- * Handle the DC `/save` slash command for both classic AND fleet-topic targets.
3488
- * Picks the backend-appropriate command (kiro → /chat save, claude → /export);
3489
- * unsupported backends get a clear error. Routes via classic paste or fleet IPC.
4247
+ * Cap every instance's pipe-pane log, walking the instances **directory** rather
4248
+ * than the config.
4249
+ *
4250
+ * A running instance rotates its own log on each health tick, so the ones that
4251
+ * need this are the ones nothing else looks at:
4252
+ *
4253
+ * - deleted instances, whose directory outlives the config entry. Nothing ever
4254
+ * touched these again. On the machine this was found on, one held 122 MB and
4255
+ * another 74 MB, out of 622 MB of pipe-pane logs in total.
4256
+ * - classic instances, which live in classicChannels, not fleetConfig.instances,
4257
+ * and so were never in the old config-driven loop at all.
4258
+ * - stopped instances, which have no health tick running.
4259
+ *
4260
+ * pipe-pane writes raw TUI output, so a wedged splash screen can emit ANSI frames
4261
+ * at animation rate. Unbounded growth here fills the disk, which takes the whole
4262
+ * fleet down rather than one instance.
3490
4263
  */
3491
- async handleSlashSave(data, adapterId) {
3492
- if (!this.classicChannels?.isAdmin(data.userId)) {
3493
- await data.respond(t("admin.required"));
3494
- return;
4264
+ rotateAllInstanceLogs() {
4265
+ const root = join(this.dataDir, "instances");
4266
+ let entries;
4267
+ try {
4268
+ entries = readdirSync(root, { withFileTypes: true });
4269
+ }
4270
+ catch {
4271
+ return; // no instances directory yet
4272
+ }
4273
+ for (const entry of entries) {
4274
+ if (!entry.isDirectory())
4275
+ continue;
4276
+ // rotateLogIfNeeded is already best-effort and returns early on a missing
4277
+ // file, so a directory without a pipe-pane log costs one stat.
4278
+ rotateLogIfNeeded(join(root, entry.name, "output.log"));
4279
+ }
4280
+ }
4281
+ pruneEventLog() {
4282
+ try {
4283
+ this.eventLog?.prune(FleetManager.EVENT_LOG_RETENTION_DAYS);
4284
+ }
4285
+ catch (err) {
4286
+ this.logger.warn({ err }, "Event log prune failed");
4287
+ }
4288
+ }
4289
+ openEventLog() {
4290
+ const dbPath = join(this.dataDir, "events.db");
4291
+ try {
4292
+ return new EventLog(dbPath);
4293
+ }
4294
+ catch (err) {
4295
+ this.logger.error({ err, dbPath }, "events.db unusable — moving it aside and starting a fresh one");
4296
+ const stamp = new Date().toISOString().replace(/[:.]/g, "-");
4297
+ for (const suffix of ["", "-wal", "-shm"]) {
4298
+ try {
4299
+ renameSync(`${dbPath}${suffix}`, `${dbPath}${suffix}.corrupt-${stamp}`);
4300
+ }
4301
+ catch { /* may not exist */ }
4302
+ }
4303
+ try {
4304
+ return new EventLog(dbPath);
4305
+ }
4306
+ catch (retryErr) {
4307
+ // History is worth losing; a fleet that won't start is not.
4308
+ this.logger.error({ err: retryErr, dbPath }, "Could not open a fresh events.db — continuing without event logging");
4309
+ return null;
4310
+ }
4311
+ }
4312
+ }
4313
+ /**
4314
+ * Report a fleet-level fault (not attributable to one instance) to the General
4315
+ * topic, so the operator learns about it without reading daemon.log.
4316
+ *
4317
+ * Throttled per distinct message: an unhandled rejection typically comes from a
4318
+ * loop (a poller, a repeating timer), and one channel message per occurrence
4319
+ * would bury the topic — which is worse than silence. First occurrence goes out
4320
+ * immediately, repeats are suppressed for THROTTLE_MS and then re-sent with a
4321
+ * count.
4322
+ *
4323
+ * The log line is written by the caller regardless: if every adapter is down,
4324
+ * the only notification path is the one that is broken.
4325
+ */
4326
+ notifyFleetError(text) {
4327
+ const now = Date.now();
4328
+ const key = text.slice(0, 200);
4329
+ const seen = this.fleetErrorNotices.get(key);
4330
+ if (seen && now - seen.at < FleetManager.FLEET_ERROR_THROTTLE_MS) {
4331
+ seen.suppressed++;
4332
+ return;
4333
+ }
4334
+ const suppressed = seen?.suppressed ?? 0;
4335
+ this.fleetErrorNotices.set(key, { at: now, suppressed: 0 });
4336
+ // Bound the map: it is keyed by message text, and a message with a varying
4337
+ // suffix (a path, an id) would otherwise grow it without limit.
4338
+ if (this.fleetErrorNotices.size > 100) {
4339
+ const oldest = this.fleetErrorNotices.keys().next().value;
4340
+ if (oldest !== undefined)
4341
+ this.fleetErrorNotices.delete(oldest);
4342
+ }
4343
+ const body = suppressed > 0
4344
+ ? `${text}\n(plus ${suppressed} more in the last ${Math.round(FleetManager.FLEET_ERROR_THROTTLE_MS / 60_000)}m)`
4345
+ : text;
4346
+ // Resolved from config, NOT findGeneralInstance(): that requires a live daemon,
4347
+ // and a fleet-level fault is exactly when the General may be down. The topic
4348
+ // itself still exists, and notifyInstanceTopic only needs adapter + group +
4349
+ // topic_id to post into it.
4350
+ const general = Object.entries(this.fleetConfig?.instances ?? {})
4351
+ .find(([, config]) => config.general_topic === true)?.[0];
4352
+ if (general) {
4353
+ this.notifyInstanceTopic(general, body);
4354
+ return;
4355
+ }
4356
+ // No General instance — fall back to the primary channel's group.
4357
+ const channelCfg = this.getChannelConfig();
4358
+ const groupId = channelCfg?.group_id;
4359
+ if (this.adapter && groupId) {
4360
+ this.adapter.sendText(String(groupId), body)
4361
+ .catch(err => this.logger.warn({ err }, "Failed to send fleet error notification"));
4362
+ return;
4363
+ }
4364
+ this.logger.warn({ text: body }, "Fleet error had no notification target (no General instance, no adapter)");
4365
+ }
4366
+ static FLEET_ERROR_THROTTLE_MS = 10 * 60_000;
4367
+ fleetErrorNotices = new Map();
4368
+ notifyInstanceTopic(instanceName, text, extraOpts) {
4369
+ const adapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
4370
+ if (!adapter)
4371
+ return;
4372
+ const channelCfg = this.getChannelConfig(this.getInstanceAdapterId(instanceName));
4373
+ const groupId = channelCfg?.group_id;
4374
+ // Fleet topic instance
4375
+ const threadId = this.fleetConfig?.instances[instanceName]?.topic_id;
4376
+ if (threadId != null && groupId) {
4377
+ adapter.sendText(String(groupId), text, { threadId: String(threadId), ...extraOpts })
4378
+ .catch(e => this.logger.warn({ err: e, instanceName }, "Failed to send instance topic notification"));
4379
+ return;
4380
+ }
4381
+ // Classic instance: find its channelId from the classic manager
4382
+ const classicChatId = this.classicChannels?.getChannelIdByInstance(instanceName);
4383
+ if (classicChatId) {
4384
+ adapter.sendText(classicChatId, text, extraOpts)
4385
+ .catch(e => this.logger.warn({ err: e, instanceName }, "Failed to send classic notification"));
4386
+ return;
4387
+ }
4388
+ // Fallback: send to group without threadId
4389
+ if (groupId) {
4390
+ adapter.sendText(String(groupId), text, extraOpts)
4391
+ .catch(e => this.logger.warn({ err: e, instanceName }, "Failed to send notification (no topic)"));
4392
+ }
4393
+ }
4394
+ // ── Cancel button ────────────────────────────────────────────────────
4395
+ // Sent after delivering a user message to an instance; clicking it (or
4396
+ // /cancel) sends Escape to the instance's pane to interrupt generation.
4397
+ /** Send a "🛑 Cancel" button to the instance's topic/channel after delivery. */
4398
+ /**
4399
+ * Handle the DC `/save` slash command for both classic AND fleet-topic targets.
4400
+ * Picks the backend-appropriate command (kiro → /chat save, claude → /export);
4401
+ * unsupported backends get a clear error. Routes via classic paste or fleet IPC.
4402
+ */
4403
+ async handleSlashSave(data, adapterId) {
4404
+ if (!this.classicChannels?.isAdmin(data.userId)) {
4405
+ await data.respond(t("admin.required"));
4406
+ return;
3495
4407
  }
3496
4408
  // Classic resolves per-bot (same-channel multi-bot); otherwise a fleet topic.
3497
4409
  const classicName = this.classicChannels.getInstanceByChannel(data.channelId, adapterId);
@@ -3526,6 +4438,40 @@ export class FleetManager {
3526
4438
  await data.respond(t("save.sent", cmd, target.name));
3527
4439
  }
3528
4440
  /** Whether the instance currently has at least one live cancel button. */
4441
+ /**
4442
+ * A click on a cancel button, whether or not the fleet still tracks it.
4443
+ *
4444
+ * The old rule was "act only while an entry is live", which made a click on a
4445
+ * button the fleet had forgotten a silent no-op — no cancel, no message, not
4446
+ * even a log line. That is indistinguishable from a broken button, and it is
4447
+ * what the "按鈕點了沒反應" reports were: the entry is briefly absent while a
4448
+ * button is being replaced, and a delete that fails leaves the message on
4449
+ * screen with no entry at all.
4450
+ *
4451
+ * So: honour the click if the instance is actually running, and say so plainly
4452
+ * if it is not. The stale-click path is rate-limited because the original
4453
+ * concern was real — a second click must not fire a second interrupt key at an
4454
+ * instance that has already started a new turn.
4455
+ */
4456
+ handleCancelClick(instanceName, adapter, data) {
4457
+ if (this.hasCancelButton(instanceName)) {
4458
+ this.cancelInstance(instanceName);
4459
+ return;
4460
+ }
4461
+ const lastAt = this.staleCancelClickAt.get(instanceName) ?? 0;
4462
+ if (Date.now() - lastAt < STALE_CANCEL_CLICK_COOLDOWN_MS)
4463
+ return;
4464
+ this.staleCancelClickAt.set(instanceName, Date.now());
4465
+ // cancelInstance returns false when there is no daemon — i.e. nothing to
4466
+ // cancel, which is the one case where the button really is dead.
4467
+ if (this.cancelInstance(instanceName)) {
4468
+ this.logger.info({ instanceName }, "Cancel click honoured with no live button entry");
4469
+ return;
4470
+ }
4471
+ this.logger.info({ instanceName }, "Cancel click on an expired button — instance not running");
4472
+ adapter?.editMessage(data.chatId, data.messageId, t("cancel.button_stale", instanceName), data.threadId)
4473
+ .catch(() => { });
4474
+ }
3529
4475
  hasCancelButton(instanceName) {
3530
4476
  for (const e of this.cancelButtons.values()) {
3531
4477
  if (e.instanceName === instanceName)
@@ -3534,15 +4480,22 @@ export class FleetManager {
3534
4480
  return false;
3535
4481
  }
3536
4482
  async sendCancelButton(instanceName, correlationId) {
3537
- // At most one button shown per instance: retire any existing ones first
3538
- // (delete + bounded retry). Each is tracked separately, so a failed delete
3539
- // here doesn't strand it it keeps retrying on its own timer.
3540
- this.retireInstanceButtons(instanceName);
4483
+ // Post first, retire after (see the tail of this method). Retiring up front
4484
+ // meant that from the delete until the new message came back a chat API
4485
+ // round trip, and every reply goes through here the instance had NO live
4486
+ // entry, while the old button was still on screen. A click in that window
4487
+ // hit `hasCancelButton() === false` and was silently dropped: the reported
4488
+ // "按鈕失效". If notifyAlert then failed, the button was simply gone.
3541
4489
  const adapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
3542
4490
  if (!adapter)
3543
4491
  return;
3544
- const adapterId = this.instanceWorldBinding.get(instanceName);
3545
- const groupId = this.getChannelConfig(adapterId)?.group_id;
4492
+ // Resolve the group through the world fallback (first world when unbound),
4493
+ // NOT through getChannelConfig(binding)?.group_id: on a fleet configured with
4494
+ // `channels:` worlds the primary `channel:` block is empty, so an instance
4495
+ // with no world binding yet (fresh restart, cross-instance delegation)
4496
+ // resolved group_id to undefined and the button silently never appeared.
4497
+ const adapterId = this.getInstanceAdapterId(instanceName);
4498
+ const groupId = this.getGroupIdForInstance(instanceName) || undefined;
3546
4499
  const topicId = this.fleetConfig?.instances[instanceName]?.topic_id;
3547
4500
  let chatId;
3548
4501
  let threadId;
@@ -3558,8 +4511,12 @@ export class FleetManager {
3558
4511
  if (!chatId && groupId)
3559
4512
  chatId = String(groupId);
3560
4513
  }
3561
- if (!chatId)
4514
+ if (!chatId) {
4515
+ // A button that cannot be addressed must say so — this exact silence is how
4516
+ // "the cancel button sometimes never appears" stayed unreported-in-logs.
4517
+ this.logger.warn({ instanceName, topicId, groupId }, "Cannot address cancel button (no chat id resolved)");
3562
4518
  return;
4519
+ }
3563
4520
  try {
3564
4521
  const sent = await adapter.notifyAlert(chatId, {
3565
4522
  type: "cancel",
@@ -3567,13 +4524,6 @@ export class FleetManager {
3567
4524
  message: "👀 處理中…",
3568
4525
  choices: [{ id: `cancel:${instanceName}`, label: t("cancel.button") }],
3569
4526
  }, threadId ? { threadId } : undefined);
3570
- // A concurrent sendCancelButton for the same instance may have posted its
3571
- // own button while we awaited notifyAlert. Retire any other buttons for
3572
- // this instance (not the one we just posted) so only the newest shows.
3573
- for (const other of this.cancelButtons.values()) {
3574
- if (other.instanceName === instanceName)
3575
- this.retireButton(other);
3576
- }
3577
4527
  const entry = {
3578
4528
  instanceName,
3579
4529
  adapterId,
@@ -3582,7 +4532,16 @@ export class FleetManager {
3582
4532
  threadId: sent.threadId ?? threadId,
3583
4533
  correlationId,
3584
4534
  retryCount: 0,
4535
+ // Elapsed time is measured from when this button was posted — i.e. from
4536
+ // when the work was handed over — not from the pane's working transition,
4537
+ // which resets if the CLI blips idle mid-turn.
4538
+ startedAt: Date.now(),
4539
+ // Matches the text notifyAlert just posted, so the first 60s tick does
4540
+ // not re-edit identical text — which put a "(edited)" mark on Discord
4541
+ // with nothing visibly changed.
4542
+ lastProgressText: "👀 處理中…",
3585
4543
  };
4544
+ this.startProgressTicker(entry);
3586
4545
  // Idle-check backstop: every 5min, if the instance is idle, retire the
3587
4546
  // button. Covers turns that end without hitting a clear trigger (reply /
3588
4547
  // cancel / correlation). Cleared in discardButton when the entry is removed.
@@ -3591,18 +4550,167 @@ export class FleetManager {
3591
4550
  clearInterval(entry.idleCheckTimer);
3592
4551
  return;
3593
4552
  }
3594
- if (this.getInstanceIdle(instanceName)) {
3595
- this.logger.info({ instanceName, messageId: entry.messageId }, "Cancel button idle backstop retiring");
4553
+ const reason = this.getInstanceIdle(instanceName) ? "idle"
4554
+ : this.stateReportDead(instanceName) ? "state reports stopped"
4555
+ : Date.now() - (entry.startedAt ?? 0) > CANCEL_BTN_MAX_LIFETIME_MS ? "24h ceiling"
4556
+ : null;
4557
+ if (reason) {
4558
+ this.logger.info({ instanceName, messageId: entry.messageId, reason }, "Cancel button backstop retiring");
3596
4559
  this.retireButton(entry);
4560
+ return;
3597
4561
  }
4562
+ // Still looks busy. The daemon only broadcasts on transitions, so ask for
4563
+ // a fresh snapshot — a live daemon's answer refreshes receivedAt and keeps
4564
+ // the staleness check honest; a dead one's silence is the evidence.
4565
+ this.instanceIpcClients.get(instanceName)?.send({
4566
+ type: "query_instance_state", requestId: `cancel-btn-${Date.now()}`,
4567
+ });
3598
4568
  }, CANCEL_BTN_IDLE_CHECK_INTERVAL_MS);
3599
4569
  this.cancelButtons.set(sent.messageId, entry);
4570
+ // Only now: at most one button per instance, but never zero. Covers both
4571
+ // the previous turn's button and any button a concurrent
4572
+ // sendCancelButton posted while we were awaiting notifyAlert.
4573
+ for (const other of [...this.cancelButtons.values()]) {
4574
+ if (other.instanceName === instanceName && other.messageId !== sent.messageId) {
4575
+ this.retireButton(other);
4576
+ }
4577
+ }
4578
+ this.persistCancelButtons();
3600
4579
  this.logger.info({ instanceName, messageId: sent.messageId }, "Cancel button sent");
3601
4580
  }
3602
4581
  catch (e) {
3603
4582
  this.logger.warn({ err: e.message, instanceName }, "Failed to send cancel button");
3604
4583
  }
3605
4584
  }
4585
+ /**
4586
+ * The cancel button's text for a given elapsed time.
4587
+ *
4588
+ * Below the threshold it keeps the original wording, so a normal quick answer
4589
+ * looks exactly as it did before. Past it, the button doubles as the live
4590
+ * progress indicator (#409) — the channel showed nothing at all during long work,
4591
+ * and once the agent had replied once there was no sign it was still going.
4592
+ */
4593
+ static progressText(elapsedMs, activity, minElapsedMs = PROGRESS_MIN_ELAPSED_MS) {
4594
+ if (elapsedMs < minElapsedMs)
4595
+ return "👀 處理中…";
4596
+ const totalSeconds = Math.floor(elapsedMs / 1000);
4597
+ const minutes = Math.floor(totalSeconds / 60);
4598
+ const seconds = totalSeconds % 60;
4599
+ const elapsed = minutes >= 60
4600
+ ? `${Math.floor(minutes / 60)}h ${minutes % 60}m`
4601
+ : `${minutes}m ${String(seconds).padStart(2, "0")}s`;
4602
+ const detail = FleetManager.sanitizeActivity(activity);
4603
+ return detail
4604
+ ? `⏳ 處理中… (已進行 ${elapsed} · ${detail})`
4605
+ : `⏳ 處理中… (已進行 ${elapsed})`;
4606
+ }
4607
+ /**
4608
+ * Make a tool summary safe to paste into a channel message.
4609
+ *
4610
+ * The text is agent-controlled (it is built from tool inputs — file paths,
4611
+ * shell commands), so it gets flattened to one line, capped, and stripped of
4612
+ * the two Discord mass-mention triggers. Neither channel renders it with a
4613
+ * parse mode, so no markup escaping is needed beyond that.
4614
+ */
4615
+ static sanitizeActivity(activity) {
4616
+ if (!activity)
4617
+ return null;
4618
+ const flat = activity.replace(/\s+/g, " ").replace(/@(everyone|here)/g, "@​$1").trim();
4619
+ if (!flat)
4620
+ return null;
4621
+ return flat.length > PROGRESS_ACTIVITY_MAX_CHARS
4622
+ ? `${flat.slice(0, PROGRESS_ACTIVITY_MAX_CHARS - 1)}…`
4623
+ : flat;
4624
+ }
4625
+ /**
4626
+ * Remember what an instance is currently doing, for the progress line.
4627
+ *
4628
+ * Best-effort by design: only backends that expose a live activity feed report
4629
+ * anything, and the progress line simply omits the detail for the rest. It is
4630
+ * never used to decide anything — purely what the user is shown.
4631
+ */
4632
+ cacheInstanceActivity(name, activity) {
4633
+ if (activity)
4634
+ this.instanceActivity.set(name, activity);
4635
+ else
4636
+ this.instanceActivity.delete(name);
4637
+ }
4638
+ /**
4639
+ * Refresh the button's text in place while the instance keeps working.
4640
+ *
4641
+ * Uses `editAlert`, NOT `editMessage`: on Telegram the latter omits reply_markup,
4642
+ * and the Bot API treats that as "clear the keyboard" — so editing with it would
4643
+ * delete the very cancel button this is trying to keep alive.
4644
+ */
4645
+ /** Configured threshold before elapsed time appears, in ms. */
4646
+ progressMinElapsedMs() {
4647
+ const seconds = this.fleetConfig?.defaults
4648
+ ?.progress_min_elapsed;
4649
+ if (typeof seconds === "number" && Number.isFinite(seconds) && seconds >= 0) {
4650
+ return seconds * 1000;
4651
+ }
4652
+ return PROGRESS_MIN_ELAPSED_MS;
4653
+ }
4654
+ startProgressTicker(entry) {
4655
+ const tick = () => {
4656
+ if (!this.cancelButtons.has(entry.messageId)) {
4657
+ clearInterval(entry.progressTimer);
4658
+ return;
4659
+ }
4660
+ const text = FleetManager.progressText(Date.now() - (entry.startedAt ?? Date.now()), this.instanceActivity.get(entry.instanceName), this.progressMinElapsedMs());
4661
+ if (text === entry.lastProgressText)
4662
+ return; // nothing changed — skip the API call
4663
+ const adapter = this.getAdapterForInstance(entry.instanceName) ?? this.adapter;
4664
+ if (!adapter?.editAlert)
4665
+ return;
4666
+ entry.lastProgressText = text;
4667
+ adapter.editAlert(entry.chatId, entry.messageId, {
4668
+ type: "cancel",
4669
+ instanceName: entry.instanceName,
4670
+ message: text,
4671
+ choices: [{ id: `cancel:${entry.instanceName}`, label: t("cancel.button") }],
4672
+ }, entry.threadId ? { threadId: entry.threadId } : undefined)
4673
+ .catch(err => {
4674
+ // A failed progress edit must never escalate: the button still works and
4675
+ // the next tick retries. Common causes are a deleted message or a
4676
+ // rate limit.
4677
+ this.logger.debug({ err, instanceName: entry.instanceName }, "Progress edit failed");
4678
+ });
4679
+ };
4680
+ entry.progressTimer = setInterval(tick, PROGRESS_UPDATE_INTERVAL_MS);
4681
+ entry.progressTimer.unref?.();
4682
+ // One extra tick right when the threshold passes, so a 30s threshold shows
4683
+ // time at ~30s instead of waiting for the first 60s interval. Costs at most
4684
+ // one additional edit per turn that lives past the threshold.
4685
+ const firstAt = this.progressMinElapsedMs() - (Date.now() - (entry.startedAt ?? Date.now()));
4686
+ if (firstAt > 0 && firstAt < PROGRESS_UPDATE_INTERVAL_MS) {
4687
+ const firstTick = setTimeout(tick, firstAt);
4688
+ firstTick.unref?.();
4689
+ }
4690
+ }
4691
+ /**
4692
+ * After a reply: give the instance REPLY_RETIRE_GRACE_MS to resume working; if
4693
+ * it has not, retire its button. Re-arming replaces the previous timer, so a
4694
+ * burst of replies ends with exactly one pending check.
4695
+ */
4696
+ armReplyGrace(instanceName) {
4697
+ for (const entry of this.cancelButtons.values()) {
4698
+ if (entry.instanceName !== instanceName)
4699
+ continue;
4700
+ if (entry.replyGraceTimer)
4701
+ clearTimeout(entry.replyGraceTimer);
4702
+ entry.replyGraceTimer = setTimeout(() => {
4703
+ entry.replyGraceTimer = undefined;
4704
+ if (!this.cancelButtons.has(entry.messageId))
4705
+ return;
4706
+ if (!this.getInstanceIdle(instanceName))
4707
+ return; // resumed — a long run keeps its button
4708
+ this.logger.info({ instanceName, messageId: entry.messageId }, "Cancel button retired — no work resumed after reply");
4709
+ this.retireButton(entry);
4710
+ }, REPLY_RETIRE_GRACE_MS);
4711
+ entry.replyGraceTimer.unref?.();
4712
+ }
4713
+ }
3606
4714
  /** Retire (delete) every cancel button belonging to an instance. */
3607
4715
  retireInstanceButtons(instanceName) {
3608
4716
  // Snapshot first — retireButton may delete entries from the map on success.
@@ -3634,7 +4742,70 @@ export class FleetManager {
3634
4742
  clearTimeout(entry.retryTimer);
3635
4743
  if (entry.idleCheckTimer)
3636
4744
  clearInterval(entry.idleCheckTimer);
4745
+ if (entry.progressTimer)
4746
+ clearInterval(entry.progressTimer);
4747
+ if (entry.replyGraceTimer)
4748
+ clearTimeout(entry.replyGraceTimer);
3637
4749
  this.cancelButtons.delete(entry.messageId);
4750
+ this.persistCancelButtons();
4751
+ }
4752
+ /**
4753
+ * Mirror the live buttons to disk. The map is memory-only, so before this a
4754
+ * fleet restart orphaned every button on screen: frozen "處理中…" text and a
4755
+ * click that did nothing, forever. The ledger is tiny (a handful of rows) and
4756
+ * written on every add/remove — no debounce needed at that rate.
4757
+ */
4758
+ persistCancelButtons() {
4759
+ try {
4760
+ const rows = [...this.cancelButtons.values()].map(e => ({
4761
+ instanceName: e.instanceName,
4762
+ adapterId: e.adapterId,
4763
+ chatId: e.chatId,
4764
+ messageId: e.messageId,
4765
+ threadId: e.threadId,
4766
+ }));
4767
+ writeFileSync(join(this.dataDir, CANCEL_BTN_LEDGER_FILE), JSON.stringify(rows));
4768
+ }
4769
+ catch (err) {
4770
+ this.logger.debug({ err }, "Cancel button ledger write failed");
4771
+ }
4772
+ }
4773
+ /**
4774
+ * Delete the previous process's buttons. Runs once adapters are up: nothing
4775
+ * from a previous fleet process can still be mid-turn from this process's
4776
+ * point of view, so every ledger row is an orphan by definition.
4777
+ */
4778
+ async sweepOrphanedCancelButtons() {
4779
+ const ledgerPath = join(this.dataDir, CANCEL_BTN_LEDGER_FILE);
4780
+ let rows;
4781
+ try {
4782
+ if (!existsSync(ledgerPath))
4783
+ return;
4784
+ rows = JSON.parse(readFileSync(ledgerPath, "utf-8"));
4785
+ }
4786
+ catch {
4787
+ try {
4788
+ unlinkSync(ledgerPath);
4789
+ }
4790
+ catch { /* corrupt ledger — drop it */ }
4791
+ return;
4792
+ }
4793
+ for (const row of rows) {
4794
+ const adapter = (row.adapterId ? this.worlds.get(row.adapterId)?.adapter : undefined)
4795
+ ?? this.getAdapterForInstance?.(row.instanceName) ?? this.adapter;
4796
+ if (!adapter?.deleteMessage)
4797
+ continue;
4798
+ try {
4799
+ await adapter.deleteMessage(row.chatId, row.messageId, row.threadId);
4800
+ this.logger.info({ instanceName: row.instanceName, messageId: row.messageId }, "Swept orphaned cancel button from previous run");
4801
+ }
4802
+ catch (err) {
4803
+ // Best effort: the message may already be gone, or too old to delete.
4804
+ this.logger.debug({ err, messageId: row.messageId }, "Orphaned cancel button sweep failed");
4805
+ }
4806
+ }
4807
+ // The current process owns the ledger from here on.
4808
+ this.persistCancelButtons();
3638
4809
  }
3639
4810
  /** Re-attempt a failed button delete up to CANCEL_BTN_MAX_RETRIES times. */
3640
4811
  scheduleButtonRetry(entry, err) {
@@ -3748,7 +4919,7 @@ export class FleetManager {
3748
4919
  const adapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
3749
4920
  if (!adapter)
3750
4921
  return;
3751
- const channelCfg = this.getChannelConfig(this.instanceWorldBinding.get(instanceName));
4922
+ const channelCfg = this.getChannelConfig(this.getInstanceAdapterId(instanceName));
3752
4923
  const groupId = channelCfg?.group_id;
3753
4924
  if (!groupId)
3754
4925
  return;
@@ -4299,6 +5470,12 @@ When users create specialized instances, suggest these configurations:
4299
5470
  meta.image_path = saves[saves.length - 1][1].split(",")[0].trim();
4300
5471
  }
4301
5472
  }
5473
+ // Classic channels queue reactions like everyone else (#432 stored them, but
5474
+ // this path never attached them — reactions in a ClassicBot channel went into
5475
+ // the DB and were never seen again). Same contract as the topic paths:
5476
+ // consumed only after the delivery succeeded.
5477
+ const reactions = this.pendingReactionsMeta(instanceName);
5478
+ Object.assign(meta, reactions.meta);
4302
5479
  try {
4303
5480
  await this.deliverToInstance(instanceName, {
4304
5481
  type: "fleet_inbound",
@@ -4306,6 +5483,7 @@ When users create specialized instances, suggest these configurations:
4306
5483
  targetSession: instanceName,
4307
5484
  meta,
4308
5485
  });
5486
+ reactions.consume();
4309
5487
  }
4310
5488
  catch (err) {
4311
5489
  this.logger.warn({ err: err.message, instanceName }, "Classic wake/delivery failed");
@@ -4444,6 +5622,170 @@ When users create specialized instances, suggest these configurations:
4444
5622
  return env?.models ?? [];
4445
5623
  }
4446
5624
  /** `/model` slash handler (admin only). No arg → DC menu; `/model <name>` → apply directly. */
5625
+ /** Label an effort choice, marking the one currently configured. */
5626
+ effortChoiceLabel(level, current) {
5627
+ return level === current ? `✓ ${level}` : level;
5628
+ }
5629
+ effortMenuHeader(instanceName) {
5630
+ const { effort, source } = this.resolveInstanceEffort(instanceName);
5631
+ if (!effort)
5632
+ return "Current effort: (CLI default)";
5633
+ return source === "fleet-default"
5634
+ ? `Current effort: ${effort} (fleet default)`
5635
+ : `Current effort: ${effort}`;
5636
+ }
5637
+ /** `/effort` — DC Select Menu, or apply directly when a level is given. */
5638
+ async handleEffortSlash(data, adapterId) {
5639
+ if (!this.isModelAdmin(data.userId, data.channelId, adapterId)) {
5640
+ await data.respond(t("permission.denied"));
5641
+ return;
5642
+ }
5643
+ const name = this.resolveSlashTarget(data.channelId, adapterId);
5644
+ if (!name) {
5645
+ await data.respond(t("classic.no_agent"));
5646
+ return;
5647
+ }
5648
+ const requested = (typeof data.options?.level === "string" ? data.options.level.trim() : "")
5649
+ || (data.text?.trim() ?? "");
5650
+ if (requested) {
5651
+ await data.respond(await this.applyEffort(name, requested));
5652
+ return;
5653
+ }
5654
+ const levels = this.effortLevelsFor(name);
5655
+ if (levels.length === 0) {
5656
+ await data.respond(`❌ ${this.backendNameForInstance(name)} has no reasoning-effort setting.`);
5657
+ return;
5658
+ }
5659
+ if (!data.respondChoices) {
5660
+ await data.respond(`Usage: /effort <${levels.join("|")}>`);
5661
+ return;
5662
+ }
5663
+ const current = this.resolveInstanceEffort(name).effort;
5664
+ const nonce = randomBytes(6).toString("hex");
5665
+ const choices = levels.map(l => ({
5666
+ id: `${EFFORT_SELECT_CALLBACK_PREFIX}${nonce}:${l}`,
5667
+ label: this.effortChoiceLabel(l, current),
5668
+ }));
5669
+ const timer = setTimeout(() => this.pendingEffortSelects.delete(nonce), CLASSIC_BACKEND_SELECTION_TIMEOUT_MS);
5670
+ timer.unref?.();
5671
+ this.pendingEffortSelects.set(nonce, { instanceName: name, userId: data.userId, channelId: data.channelId, timer, respond: data.respond });
5672
+ try {
5673
+ await data.respondChoices(`${this.effortMenuHeader(name)}\nSelect a new effort level:`, choices);
5674
+ }
5675
+ catch (err) {
5676
+ this.pendingEffortSelects.delete(nonce);
5677
+ clearTimeout(timer);
5678
+ this.logger.warn({ err, instanceName: name }, "effort menu failed");
5679
+ await data.respond(`Usage: /effort <${levels.join("|")}>`);
5680
+ }
5681
+ }
5682
+ /** TG inline-keyboard effort menu. Returns null on success, else a fallback string. */
5683
+ async promptEffortMenu(instanceName, userId, channelId, adapter, chatId, threadId) {
5684
+ const levels = this.effortLevelsFor(instanceName);
5685
+ if (levels.length === 0) {
5686
+ return `❌ ${this.backendNameForInstance(instanceName)} has no reasoning-effort setting.`;
5687
+ }
5688
+ const current = this.resolveInstanceEffort(instanceName).effort;
5689
+ const nonce = randomBytes(6).toString("hex");
5690
+ const choices = levels.map(l => ({
5691
+ id: `${EFFORT_SELECT_CALLBACK_PREFIX}${nonce}:${l}`,
5692
+ label: this.effortChoiceLabel(l, current),
5693
+ }));
5694
+ const respond = async (text) => {
5695
+ await adapter.sendText(chatId, text, { threadId });
5696
+ return undefined;
5697
+ };
5698
+ const timer = setTimeout(() => {
5699
+ const p = this.pendingEffortSelects.get(nonce);
5700
+ if (p) {
5701
+ this.pendingEffortSelects.delete(nonce);
5702
+ p.respond("⏰ Effort selection expired.").catch(() => { });
5703
+ }
5704
+ }, CLASSIC_BACKEND_SELECTION_TIMEOUT_MS);
5705
+ timer.unref?.();
5706
+ this.pendingEffortSelects.set(nonce, { instanceName, userId, channelId, timer, respond, adapter, adapterChatId: chatId, adapterThreadId: threadId });
5707
+ try {
5708
+ const menuMessageId = await adapter.promptUser(chatId, `${this.effortMenuHeader(instanceName)}\nSelect a new effort level:`, choices, { threadId });
5709
+ const pending = this.pendingEffortSelects.get(nonce);
5710
+ if (pending)
5711
+ pending.menuMessageId = menuMessageId;
5712
+ return null;
5713
+ }
5714
+ catch (err) {
5715
+ this.pendingEffortSelects.delete(nonce);
5716
+ clearTimeout(timer);
5717
+ this.logger.warn({ err, instanceName }, "TG effort menu failed");
5718
+ return `Usage: /effort <${levels.join("|")}>`;
5719
+ }
5720
+ }
5721
+ /** Consume an `/effort` selection callback. Mirrors handleModelSelection. */
5722
+ async handleEffortSelection(data) {
5723
+ if (!data.callbackData.startsWith(EFFORT_SELECT_CALLBACK_PREFIX))
5724
+ return false;
5725
+ const match = data.callbackData.match(/^effort-select:([0-9a-f]+):(.+)$/);
5726
+ if (!match)
5727
+ return true;
5728
+ const pending = this.pendingEffortSelects.get(match[1]);
5729
+ if (!pending)
5730
+ return true;
5731
+ if (data.userId && data.userId !== pending.userId)
5732
+ return true;
5733
+ const cbChannel = data.threadId ?? data.chatId;
5734
+ if (cbChannel !== pending.channelId && data.chatId !== pending.channelId)
5735
+ return true;
5736
+ this.pendingEffortSelects.delete(match[1]);
5737
+ clearTimeout(pending.timer);
5738
+ const level = match[2];
5739
+ const progressText = `⏳ Setting ${pending.instanceName} effort to \`${level}\`…`;
5740
+ let progressMsgId;
5741
+ if (pending.adapter && pending.adapterChatId) {
5742
+ const menuMessageId = pending.menuMessageId ?? data.messageId;
5743
+ if (menuMessageId && pending.adapter.editMessageRemoveButtons) {
5744
+ try {
5745
+ await pending.adapter.editMessageRemoveButtons(pending.adapterChatId, menuMessageId, progressText, pending.adapterThreadId);
5746
+ progressMsgId = menuMessageId;
5747
+ }
5748
+ catch { /* fall back to a new message */ }
5749
+ }
5750
+ if (!progressMsgId) {
5751
+ try {
5752
+ const sent = await pending.adapter.sendText(pending.adapterChatId, progressText, { threadId: pending.adapterThreadId });
5753
+ progressMsgId = sent.messageId;
5754
+ }
5755
+ catch { /* non-fatal */ }
5756
+ }
5757
+ }
5758
+ else {
5759
+ await pending.respond(progressText).catch(() => { });
5760
+ }
5761
+ // Background-applied and guarded for the same reason as the model path: a
5762
+ // restart backend respawns the instance here, and an unguarded rejection
5763
+ // from a menu click must not take the fleet down.
5764
+ void (async () => {
5765
+ let result;
5766
+ try {
5767
+ result = await this.applyEffort(pending.instanceName, level);
5768
+ }
5769
+ catch (err) {
5770
+ this.logger.error({ err, instance: pending.instanceName, level }, "Effort switch failed");
5771
+ result = `Effort switch to \`${level}\` failed: ${err instanceof Error ? err.message : String(err)}`;
5772
+ }
5773
+ if (pending.adapter && pending.adapterChatId) {
5774
+ if (progressMsgId) {
5775
+ pending.adapter.editMessage(pending.adapterChatId, progressMsgId, result, pending.adapterThreadId).catch(() => {
5776
+ pending.adapter.sendText(pending.adapterChatId, result, { threadId: pending.adapterThreadId }).catch(() => { });
5777
+ });
5778
+ }
5779
+ else {
5780
+ pending.adapter.sendText(pending.adapterChatId, result, { threadId: pending.adapterThreadId }).catch(() => { });
5781
+ }
5782
+ }
5783
+ else {
5784
+ await pending.respond(result).catch(() => { });
5785
+ }
5786
+ })();
5787
+ return true;
5788
+ }
4447
5789
  async handleModelSlash(data, adapterId) {
4448
5790
  if (!this.isModelAdmin(data.userId, data.channelId, adapterId)) {
4449
5791
  await data.respond(t("permission.denied"));
@@ -4580,9 +5922,19 @@ When users create specialized instances, suggest these configurations:
4580
5922
  // DC path: respond immediately with progress text
4581
5923
  await pending.respond(progressText).catch(() => { });
4582
5924
  }
4583
- // Apply model in background — don't await here (keeps callback handler fast)
5925
+ // Apply model in background — don't await here (keeps callback handler fast).
5926
+ // Guarded: applyModel() restarts the instance, and an unguarded rejection here
5927
+ // meant a user picking from the /model menu could take the whole fleet down.
5928
+ // On failure the user gets told, rather than the click silently doing nothing.
4584
5929
  void (async () => {
4585
- const result = await this.applyModel(pending.instanceName, model);
5930
+ let result;
5931
+ try {
5932
+ result = await this.applyModel(pending.instanceName, model);
5933
+ }
5934
+ catch (err) {
5935
+ this.logger.error({ err, instance: pending.instanceName, model }, "Model switch failed");
5936
+ result = `Model switch to \`${model}\` failed: ${err instanceof Error ? err.message : String(err)}`;
5937
+ }
4586
5938
  if (pending.adapter && pending.adapterChatId) {
4587
5939
  if (progressMsgId) {
4588
5940
  pending.adapter.editMessage(pending.adapterChatId, progressMsgId, result, pending.adapterThreadId).catch(() => {
@@ -4600,6 +5952,110 @@ When users create specialized instances, suggest these configurations:
4600
5952
  return true;
4601
5953
  }
4602
5954
  /** Apply a model to an instance: runtime paste (claude-code) or persist + restart (others). */
5955
+ /** AgEnD's canonical effort ladder, low → max. Backends expose a subset. */
5956
+ static EFFORT_LEVELS = ["low", "medium", "high", "xhigh", "max"];
5957
+ /** How this instance's backend applies an effort change. */
5958
+ effortStrategyFor(instanceName) {
5959
+ try {
5960
+ const backend = createBackend(this.backendNameForInstance(instanceName), this.getInstanceDir(instanceName));
5961
+ const strategy = backend.getEffortStrategy?.() ?? "unsupported";
5962
+ // A backend claiming support but listing no levels is unusable either way.
5963
+ return strategy !== "unsupported" && (backend.getEffortLevels?.() ?? []).length > 0
5964
+ ? strategy
5965
+ : "unsupported";
5966
+ }
5967
+ catch {
5968
+ return "unsupported";
5969
+ }
5970
+ }
5971
+ /** Effort levels this instance's backend actually accepts (empty = unsupported). */
5972
+ effortLevelsFor(instanceName) {
5973
+ try {
5974
+ const backend = createBackend(this.backendNameForInstance(instanceName), this.getInstanceDir(instanceName));
5975
+ if ((backend.getEffortStrategy?.() ?? "unsupported") === "unsupported")
5976
+ return [];
5977
+ return backend.getEffortLevels?.() ?? [];
5978
+ }
5979
+ catch {
5980
+ return [];
5981
+ }
5982
+ }
5983
+ /** Configured effort for an instance: per-instance, else fleet default, else none. */
5984
+ resolveInstanceEffort(instanceName) {
5985
+ const own = this.fleetConfig?.instances[instanceName]?.effort;
5986
+ if (own)
5987
+ return { effort: own, source: "instance" };
5988
+ const fallback = this.fleetConfig?.defaults?.effort;
5989
+ if (fallback)
5990
+ return { effort: fallback, source: "fleet-default" };
5991
+ return { effort: null, source: "unset" };
5992
+ }
5993
+ /**
5994
+ * Clamp a canonical level to the nearest one this backend supports.
5995
+ *
5996
+ * Clamping DOWN the ladder, never up: asking for `max` on a CLI that stops at
5997
+ * `high` should get high, not silently fall to low. The caller reports the
5998
+ * clamp — a user who asks for max and quietly receives high has been told the
5999
+ * request succeeded when it did not.
6000
+ */
6001
+ static clampEffort(level, supported) {
6002
+ if (supported.includes(level))
6003
+ return level;
6004
+ const ladder = FleetManager.EFFORT_LEVELS;
6005
+ const wanted = ladder.indexOf(level);
6006
+ if (wanted < 0)
6007
+ return null;
6008
+ for (let i = wanted - 1; i >= 0; i--) {
6009
+ if (supported.includes(ladder[i]))
6010
+ return ladder[i];
6011
+ }
6012
+ return supported[0] ?? null;
6013
+ }
6014
+ /**
6015
+ * Apply a reasoning-effort level, mirroring applyModel's shape.
6016
+ *
6017
+ * runtime backends take `/effort <level>` in the pane and keep working;
6018
+ * restart backends only read it at launch, so it is persisted and the
6019
+ * instance respawns.
6020
+ */
6021
+ async applyEffort(instanceName, requested) {
6022
+ const level = requested.trim().toLowerCase();
6023
+ const backendName = this.backendNameForInstance(instanceName);
6024
+ let strategy = "unsupported";
6025
+ let supported = [];
6026
+ try {
6027
+ const backend = createBackend(backendName, this.getInstanceDir(instanceName));
6028
+ strategy = backend.getEffortStrategy?.() ?? "unsupported";
6029
+ supported = backend.getEffortLevels?.() ?? [];
6030
+ }
6031
+ catch { /* treated as unsupported below */ }
6032
+ if (strategy === "unsupported" || supported.length === 0) {
6033
+ return `❌ ${backendName} has no reasoning-effort setting.`;
6034
+ }
6035
+ if (!FleetManager.EFFORT_LEVELS.includes(level)) {
6036
+ return `❌ Unknown effort level \`${level}\`. Use: ${FleetManager.EFFORT_LEVELS.join(", ")}.`;
6037
+ }
6038
+ const applied = FleetManager.clampEffort(level, supported);
6039
+ if (!applied)
6040
+ return `❌ ${backendName} accepts none of the canonical effort levels.`;
6041
+ const warn = applied === level
6042
+ ? ""
6043
+ : `⚠️ Clamped to \`${applied}\` (\`${level}\` not supported by ${backendName}).\n`;
6044
+ // Persist either way: a runtime switch must survive the next respawn too,
6045
+ // or the instance silently reverts on restart.
6046
+ if (this.fleetConfig?.instances[instanceName]) {
6047
+ this.fleetConfig.instances[instanceName].effort = applied;
6048
+ this.saveFleetConfig();
6049
+ }
6050
+ if (strategy === "runtime") {
6051
+ if (!this.instanceIpcClients.get(instanceName))
6052
+ return `${warn}❌ ${instanceName} is not running.`;
6053
+ this.pasteRawToClassicInstance(instanceName, `/effort ${applied}`);
6054
+ return `${warn}✅ Set ${instanceName} effort to \`${applied}\` (runtime).`;
6055
+ }
6056
+ await this.restartSingleInstance(instanceName);
6057
+ return `${warn}✅ Set ${instanceName} effort to \`${applied}\` and restarted.`;
6058
+ }
4603
6059
  async applyModel(instanceName, model) {
4604
6060
  const backendName = this.backendNameForInstance(instanceName);
4605
6061
  let strategy = "restart";
@@ -4612,7 +6068,7 @@ When users create specialized instances, suggest these configurations:
4612
6068
  if (!this.instanceIpcClients.get(instanceName))
4613
6069
  return `${warn}❌ ${instanceName} is not running.`;
4614
6070
  this.pasteRawToClassicInstance(instanceName, `/model ${model}`);
4615
- return `${warn}✅ Switched ${instanceName} to \`${model}\` (runtime).`;
6071
+ return `${warn}✅ Switched ${instanceName} to \`${model}\` (runtime).${this.effortSuffix(instanceName)}`;
4616
6072
  }
4617
6073
  // restart: persist the model so the respawned CLI launches with it.
4618
6074
  let persisted = false;
@@ -4627,7 +6083,24 @@ When users create specialized instances, suggest these configurations:
4627
6083
  if (!persisted)
4628
6084
  return `${warn}❌ Could not set model for ${instanceName}.`;
4629
6085
  await this.restartSingleInstance(instanceName);
4630
- return `${warn}✅ Set ${instanceName} to \`${model}\` and restarted.`;
6086
+ return `${warn}✅ Set ${instanceName} to \`${model}\` and restarted.${this.effortSuffix(instanceName)}`;
6087
+ }
6088
+ /**
6089
+ * The trailing "Current effort: …" line for a /model reply.
6090
+ *
6091
+ * Model and effort interact (a cheaper model at max effort is a different
6092
+ * trade than a bigger one at low), so showing the effort in force right after
6093
+ * a switch saves the round trip of asking. Empty when the backend has none.
6094
+ */
6095
+ effortSuffix(instanceName) {
6096
+ if (this.effortLevelsFor(instanceName).length === 0)
6097
+ return "";
6098
+ const { effort, source } = this.resolveInstanceEffort(instanceName);
6099
+ if (!effort)
6100
+ return "\nCurrent effort: (CLI default)";
6101
+ return source === "fleet-default"
6102
+ ? `\nCurrent effort: ${effort} (fleet default)`
6103
+ : `\nCurrent effort: ${effort}`;
4631
6104
  }
4632
6105
  /** Read recent chat log for agent context */
4633
6106
  getRecentChatLog(instanceName, maxLines = 10) {
@@ -4862,11 +6335,28 @@ When users create specialized instances, suggest these configurations:
4862
6335
  this.logger.info({ channelId, adapterId, instanceName: ch.instanceName }, "Classic channel stopped");
4863
6336
  return t("classic.stopped");
4864
6337
  }
4865
- async stopAll() {
6338
+ /**
6339
+ * Idempotent while in flight: SIGINT and SIGTERM share one handler and the
6340
+ * uncaughtException path calls this too, so overlapping runs were possible —
6341
+ * each snapshotting the daemon map and calling stop() on the same daemons
6342
+ * concurrently. Deliberately NOT `async`, so callers receive the same promise
6343
+ * object rather than a fresh wrapper around it. The latch clears when the run
6344
+ * settles, so a later genuine stop (after a restart) still does the work.
6345
+ */
6346
+ stopAll() {
6347
+ this.stopAllInFlight ??= this.doStopAll().finally(() => { this.stopAllInFlight = null; });
6348
+ return this.stopAllInFlight;
6349
+ }
6350
+ stopAllInFlight = null;
6351
+ async doStopAll() {
4866
6352
  this.startupComplete = false;
4867
6353
  this.reloadPending = false;
6354
+ // Before anything is stopped: everything that dies from here on dies
6355
+ // because we asked it to. Set synchronously — doStopAll runs to its first
6356
+ // await in the same tick as the signal handler, so no event can slip in.
6357
+ this.shuttingDown = true;
4868
6358
  this.ipcStoppingInstances.add("__fleet_stopping__");
4869
- sdNotify("STOPPING=1");
6359
+ sdNotifyBlocking("STOPPING=1");
4870
6360
  if (this.watchdogTimer) {
4871
6361
  clearInterval(this.watchdogTimer);
4872
6362
  this.watchdogTimer = null;
@@ -4886,6 +6376,29 @@ When users create specialized instances, suggest these configurations:
4886
6376
  clearInterval(this.updateCheckTimer);
4887
6377
  this.updateCheckTimer = null;
4888
6378
  }
6379
+ if (this.eventLogPruneTimer) {
6380
+ clearInterval(this.eventLogPruneTimer);
6381
+ this.eventLogPruneTimer = null;
6382
+ }
6383
+ if (this.logRotateTimer) {
6384
+ clearInterval(this.logRotateTimer);
6385
+ this.logRotateTimer = null;
6386
+ }
6387
+ // Cancel-button timers were never cleared here. The idle-check interval is not
6388
+ // unref'd, so it held the event loop open past shutdown and kept retrying
6389
+ // deletes against an adapter that was already gone.
6390
+ for (const entry of [...this.cancelButtons.values()]) {
6391
+ if (entry.retryTimer)
6392
+ clearTimeout(entry.retryTimer);
6393
+ if (entry.idleCheckTimer)
6394
+ clearInterval(entry.idleCheckTimer);
6395
+ if (entry.progressTimer)
6396
+ clearInterval(entry.progressTimer);
6397
+ }
6398
+ this.cancelButtons.clear();
6399
+ for (const timer of this.cancelButtonIdleRetireTimers.values())
6400
+ clearTimeout(timer);
6401
+ this.cancelButtonIdleRetireTimers.clear();
4889
6402
  if (this.topicCleanupTimer) {
4890
6403
  clearInterval(this.topicCleanupTimer);
4891
6404
  this.topicCleanupTimer = null;
@@ -4942,6 +6455,7 @@ When users create specialized instances, suggest these configurations:
4942
6455
  this.controlClient?.stop();
4943
6456
  this.controlClient = null;
4944
6457
  if (this.healthServer) {
6458
+ this.healthServerListening = false;
4945
6459
  this.healthServer.close();
4946
6460
  this.healthServer = null;
4947
6461
  }
@@ -4953,6 +6467,9 @@ When users create specialized instances, suggest these configurations:
4953
6467
  catch (e) {
4954
6468
  this.logger.debug({ err: e }, "Failed to remove fleet PID file");
4955
6469
  }
6470
+ // The lock contains a nonce, so an older/shutting-down process can never
6471
+ // remove a lock acquired by a newer fleet owner.
6472
+ releaseProcessFleetLock();
4956
6473
  }
4957
6474
  /**
4958
6475
  * Prune stale external sessions by re-querying each daemon for live sessions.
@@ -5104,6 +6621,15 @@ When users create specialized instances, suggest these configurations:
5104
6621
  removedRatio,
5105
6622
  validationErrors: validation.errors,
5106
6623
  }, "Refusing unsafe fleet config reload; running configuration was kept");
6624
+ // Tell the operator. A silently ignored config edit is the most confusing
6625
+ // possible outcome: they change fleet.yaml, send SIGHUP, and nothing happens
6626
+ // with no explanation anywhere they are looking.
6627
+ const why = !validation.valid
6628
+ ? `validation failed:\n${validation.errors.map(e => `• ${e.path}: ${e.message}`).join("\n")}`
6629
+ : unsafeEmpty
6630
+ ? `it removed every instance (${oldCount} → 0)`
6631
+ : `it removed more than half the instances (${oldCount} → ${newCount})`;
6632
+ this.notifyFleetError(`⚠️ fleet.yaml reload REJECTED — ${why}\nThe previous configuration is still running.`);
5107
6633
  return;
5108
6634
  }
5109
6635
  this.routing.rebuild(this.fleetConfig);
@@ -5187,6 +6713,10 @@ When users create specialized instances, suggest these configurations:
5187
6713
  clearTimeout(timeoutHandle);
5188
6714
  }
5189
6715
  this.logger.info("All instances idle — restarting...");
6716
+ const restartStartedAt = Date.now();
6717
+ // Capture the live adapter/topic before General's daemon is stopped. The
6718
+ // channel adapter remains connected throughout an in-process restart.
6719
+ const progressTarget = this.restartProgressTarget();
5190
6720
  this.clearStatuslineWatchers();
5191
6721
  for (const [, ipc] of this.instanceIpcClients) {
5192
6722
  await ipc.close();
@@ -5211,15 +6741,25 @@ When users create specialized instances, suggest these configurations:
5211
6741
  const fleet = this.loadConfig(this.configPath);
5212
6742
  this.fleetConfig = fleet;
5213
6743
  const topicMode = fleet.channel?.mode === "topic" || !!fleet.channels?.some(ch => ch.mode === "topic");
6744
+ const restartProgress = new RestartProgress(this.runnableStartupCount(fleet, topicMode), restartStartedAt, this.logger);
5214
6745
  // Phase 1: generals first
5215
6746
  const restartEntries = Object.entries(fleet.instances);
5216
6747
  const restartGenerals = restartEntries.filter(([_, cfg]) => cfg.general_topic);
5217
6748
  const restartOthers = restartEntries.filter(([_, cfg]) => !cfg.general_topic);
5218
6749
  for (const [name, cfg] of restartGenerals) {
5219
- await this.startInstance(name, cfg, topicMode).catch(err => this.logger.error({ err, name }, "Failed to start general instance"));
6750
+ try {
6751
+ await this.startInstance(name, cfg, topicMode);
6752
+ if (this.daemons.has(name))
6753
+ restartProgress.markReady();
6754
+ }
6755
+ catch (err) {
6756
+ this.logger.error({ err, name }, "Failed to start general instance");
6757
+ }
5220
6758
  }
6759
+ // General is ready again; now its topic can own the live progress message.
6760
+ await restartProgress.start(progressTarget);
5221
6761
  if (restartOthers.length > 0) {
5222
- await this.startInstancesWithConcurrency(restartOthers, topicMode);
6762
+ await this.startInstancesWithConcurrency(restartOthers, topicMode, () => restartProgress.markReady());
5223
6763
  }
5224
6764
  if (topicMode) {
5225
6765
  this.routing.rebuild(this.fleetConfig);
@@ -5228,12 +6768,22 @@ When users create specialized instances, suggest these configurations:
5228
6768
  // Restart classic channel instances (killed during orphan cleanup)
5229
6769
  if (this.classicChannels) {
5230
6770
  const fleetBackend = this.fleetConfig?.defaults?.backend;
5231
- const channels = this.classicChannels.getAll();
6771
+ const channels = this.classicChannels.getAll()
6772
+ .filter(ch => !this.lifecycle.isPaused(ch.instanceName));
5232
6773
  const concurrency = 3;
5233
6774
  let idx = 0;
5234
6775
  while (idx < channels.length) {
5235
6776
  const batch = channels.slice(idx, idx + concurrency);
5236
- await Promise.allSettled(batch.map(ch => this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after)).catch(err => this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance"))));
6777
+ await Promise.allSettled(batch.map(async (ch) => {
6778
+ try {
6779
+ await this.startClassicInstance(ch.instanceName, this.classicChannels.getBackendByInstance(ch.instanceName, fleetBackend), this.classicChannels.getPreTaskCommand(ch.channelId, ch.adapterId), this.classicChannels.getModel(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(ch.channelId, ch.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
6780
+ if (this.daemons.has(ch.instanceName))
6781
+ restartProgress.markReady();
6782
+ }
6783
+ catch (err) {
6784
+ this.logger.warn({ err, instanceName: ch.instanceName }, "Failed to start classic instance");
6785
+ }
6786
+ }));
5237
6787
  idx += concurrency;
5238
6788
  }
5239
6789
  }
@@ -5242,15 +6792,23 @@ When users create specialized instances, suggest these configurations:
5242
6792
  }
5243
6793
  }
5244
6794
  this.logger.info("Graceful restart complete");
6795
+ const configuredNames = this.configuredStartupInstanceNames(fleet, topicMode);
6796
+ const total = configuredNames.length;
6797
+ const started = configuredNames.filter(name => this.daemons.has(name)).length;
6798
+ const allNotRunning2 = configuredNames.filter(name => !this.daemons.has(name));
6799
+ const pausedNames2 = allNotRunning2.filter(n => this.lifecycle.isPaused(n));
6800
+ const failedNames = allNotRunning2.filter(n => !this.lifecycle.isPaused(n));
6801
+ const { createRequire } = await import("node:module");
6802
+ const _require2 = createRequire(import.meta.url);
6803
+ const agendVersion2 = _require2("../package.json").version ?? "unknown";
6804
+ const progressCompleted = await restartProgress.finish({
6805
+ running: started,
6806
+ total,
6807
+ version: agendVersion2,
6808
+ pausedNames: pausedNames2,
6809
+ failedNames,
6810
+ });
5245
6811
  if (groupId && this.adapter) {
5246
- const total = Object.keys(fleet.instances).length;
5247
- const started = this.daemons.size;
5248
- const allNotRunning2 = Object.keys(fleet.instances).filter(n => !this.daemons.has(n));
5249
- const pausedNames2 = allNotRunning2.filter(n => this.lifecycle.isPaused(n));
5250
- const failedNames = allNotRunning2.filter(n => !this.lifecycle.isPaused(n));
5251
- const { createRequire } = await import("node:module");
5252
- const _require2 = createRequire(import.meta.url);
5253
- const agendVersion2 = _require2("../package.json").version ?? "unknown";
5254
6812
  let restartText;
5255
6813
  if (failedNames.length === 0 && pausedNames2.length === 0) {
5256
6814
  restartText = t("fleet.ready", started, total, agendVersion2);
@@ -5262,8 +6820,10 @@ When users create specialized instances, suggest these configurations:
5262
6820
  restartText = t("fleet.ready_with_failed", started, total, agendVersion2, failedNames.join(", "))
5263
6821
  + (pausedNames2.length > 0 ? `\n⏸ Paused: ${pausedNames2.join(", ")}` : "");
5264
6822
  }
5265
- await this.adapter.sendText(String(groupId), restartText, notifyOpts)
5266
- .catch(e => this.logger.warn({ err: e }, "Failed to post restart completion notification"));
6823
+ if (!progressCompleted) {
6824
+ await this.adapter.sendText(String(groupId), restartText, notifyOpts)
6825
+ .catch(e => this.logger.warn({ err: e }, "Failed to post restart completion notification"));
6826
+ }
5267
6827
  // Notify each instance's channel — staggered to avoid rate limit storm
5268
6828
  const instances = Object.entries(this.fleetConfig?.instances ?? {});
5269
6829
  this.logger.info({ count: instances.length }, "Sending restart notification to instances (staggered)");
@@ -5303,10 +6863,20 @@ When users create specialized instances, suggest these configurations:
5303
6863
  // ── Update check ────────────────────────────────────────────────────
5304
6864
  async checkForUpdates() {
5305
6865
  try {
5306
- const { execSync } = await import("node:child_process");
6866
+ // Both npm lookups are async: as execSync they froze the fleet event loop for
6867
+ // up to 15s each, and on a beta build BOTH ran — 30s with no WATCHDOG ping,
6868
+ // past WatchdogSec's half-interval and enough for systemd to SIGABRT the fleet
6869
+ // for a background version check.
6870
+ const { execFile } = await import("node:child_process");
6871
+ const { promisify } = await import("node:util");
6872
+ const execFileP = promisify(execFile);
6873
+ const npmVersion = async (spec) => {
6874
+ const { stdout } = await execFileP("npm", ["view", spec, "version"], { timeout: 15_000 });
6875
+ return stdout.toString().trim();
6876
+ };
5307
6877
  const pkgPath = join(dirname(fileURLToPath(import.meta.url)), "..", "package.json");
5308
6878
  const currentVersion = JSON.parse(readFileSync(pkgPath, "utf-8")).version ?? "0.0.0";
5309
- const latest = execSync("npm view @songsid/agend version", { stdio: "pipe", timeout: 15_000 }).toString().trim();
6879
+ const latest = await npmVersion("@songsid/agend");
5310
6880
  let target = latest;
5311
6881
  if (currentVersion.includes("-beta")) {
5312
6882
  // Beta users track the @beta channel (never fall back to @latest, which is
@@ -5314,7 +6884,7 @@ When users create specialized instances, suggest these configurations:
5314
6884
  // of beta/latest is the newest.
5315
6885
  let beta = "";
5316
6886
  try {
5317
- beta = execSync("npm view @songsid/agend@beta version", { stdio: "pipe", timeout: 15_000 }).toString().trim();
6887
+ beta = await npmVersion("@songsid/agend@beta");
5318
6888
  }
5319
6889
  catch { /* no beta tag */ }
5320
6890
  target = beta || latest;
@@ -5388,28 +6958,22 @@ When users create specialized instances, suggest these configurations:
5388
6958
  // ── Health HTTP endpoint ─────────────────────────────────────────────
5389
6959
  startHealthServer(port) {
5390
6960
  this.startedAt = Date.now();
5391
- // Generate web token before server starts so auth is enforced from the first request.
5392
- this.webToken = randomBytes(24).toString("hex");
5393
- const tokenPath = join(this.dataDir, "web.token");
5394
- writeFileSync(tokenPath, this.webToken, { mode: 0o600 });
5395
- // Defensive: if file existed previously with looser perms, tighten it.
5396
- try {
5397
- chmodSync(tokenPath, 0o600);
5398
- }
5399
- catch {
5400
- // best-effort
5401
- }
5402
- // Separate read-only token for the /view page: grants terminal-view + profile
5403
- // read, but never write (POSTs still require the full web token).
5404
- this.viewToken = randomBytes(24).toString("hex");
5405
- const viewTokenPath = join(this.dataDir, "view.token");
5406
- writeFileSync(viewTokenPath, this.viewToken, { mode: 0o600 });
5407
- try {
5408
- chmodSync(viewTokenPath, 0o600);
5409
- }
5410
- catch { /* best-effort */ }
6961
+ this.healthServerListening = false;
6962
+ this.healthPortRetried = false;
6963
+ // Defensive for direct/unit callers; normal startup initializes these before adapters.
6964
+ if (!this.webToken || !this.viewToken)
6965
+ this.initializeWebAuthTokens();
5411
6966
  this.healthServer = createServer((req, res) => {
5412
6967
  res.setHeader("Content-Type", "application/json");
6968
+ const requestPath = new URL(req.url ?? "/", `http://localhost:${port}`).pathname;
6969
+ // Browsers request this automatically and AgEnD does not ship an icon.
6970
+ // It is neither user data nor an API route, so do not turn the harmless
6971
+ // probe into a noisy web-token 401 in the browser console.
6972
+ if (req.method === "GET" && requestPath === "/favicon.ico") {
6973
+ res.writeHead(204);
6974
+ res.end();
6975
+ return;
6976
+ }
5413
6977
  // Public health probe — no auth required.
5414
6978
  if (req.method === "GET" && req.url === "/health") {
5415
6979
  // fallthrough to existing handler below
@@ -5421,6 +6985,10 @@ When users create specialized instances, suggest these configurations:
5421
6985
  // /view routes accept the read-only view.token (or web.token) and do
5422
6986
  // their own per-method auth in view-api.ts — skip the web-token gate.
5423
6987
  }
6988
+ else if (isUsagePath(new URL(req.url ?? "/", `http://localhost:${port}`).pathname)) {
6989
+ // /api/ai-usage is read-only GET data for the /view Usage panel — open
6990
+ // like the other /view data routes (usage-api.ts rejects non-GET).
6991
+ }
5424
6992
  else {
5425
6993
  // All other endpoints require a valid token (query ?token= or X-Agend-Token header).
5426
6994
  // /ui/* will also re-check in web-api.ts, which is harmless.
@@ -5430,39 +6998,39 @@ When users create specialized instances, suggest these configurations:
5430
6998
  ?? (typeof headerToken === "string" ? headerToken : null);
5431
6999
  if (!this.webToken || providedToken !== this.webToken) {
5432
7000
  res.writeHead(401);
5433
- res.end(JSON.stringify({ error: "Unauthorized" }));
7001
+ res.end(JSON.stringify({ error: WEB_TOKEN_INVALID_MESSAGE }));
5434
7002
  return;
5435
7003
  }
5436
7004
  }
5437
7005
  if (req.method === "GET" && req.url === "/health") {
5438
- const instanceCount = this.fleetConfig?.instances
5439
- ? Object.keys(this.fleetConfig.instances).length
5440
- : 0;
5441
- res.writeHead(200);
5442
- res.end(JSON.stringify({
5443
- status: "ok",
5444
- instances: instanceCount,
5445
- uptime: Math.floor((Date.now() - this.startedAt) / 1000),
5446
- }));
7006
+ const health = this.getFleetHealth();
7007
+ // 503 when the fleet cannot do its job, so an external monitor sees it.
7008
+ // This used to always answer 200 "ok" with a count of CONFIGURED instances,
7009
+ // so every agent could be dead and every adapter down and it still looked
7010
+ // green.
7011
+ res.writeHead(health.status === "ok" ? 200 : 503);
7012
+ res.end(JSON.stringify(health));
5447
7013
  return;
5448
7014
  }
5449
7015
  if (req.method === "GET" && req.url === "/status") {
5450
7016
  const instances = Object.keys(this.fleetConfig?.instances ?? {}).map(name => {
5451
7017
  const statusFile = join(this.getInstanceDir(name), "statusline.json");
5452
- let context_pct = 0;
5453
7018
  let cost = 0;
5454
7019
  try {
5455
7020
  const data = JSON.parse(readFileSync(statusFile, "utf-8"));
5456
- context_pct = data.context_window?.used_percentage ?? 0;
5457
7021
  cost = data.cost?.total_cost_usd ?? 0;
5458
7022
  }
5459
7023
  catch (err) {
5460
7024
  this.logger.debug({ err, name }, "statusline.json read failed (/status)");
5461
7025
  }
7026
+ const backend = this.fleetConfig?.instances[name]?.backend
7027
+ ?? this.fleetConfig?.defaults?.backend
7028
+ ?? "claude-code";
7029
+ const { context } = resolveInstanceContext(this.dataDir, name, backend);
5462
7030
  return {
5463
7031
  name,
5464
7032
  status: this.getInstanceStatus(name),
5465
- context_pct,
7033
+ context_pct: context ?? 0,
5466
7034
  cost,
5467
7035
  };
5468
7036
  });
@@ -5568,7 +7136,7 @@ When users create specialized instances, suggest these configurations:
5568
7136
  (async () => {
5569
7137
  try {
5570
7138
  const topicMode = this.fleetConfig?.channel?.mode === "topic";
5571
- await this.startInstance(name, config, topicMode ?? false);
7139
+ await this.startInstance(name, config, topicMode ?? false, "fleet-topic", true);
5572
7140
  this.emitSseEvent("status", this.getUiStatus());
5573
7141
  res.writeHead(200);
5574
7142
  res.end(JSON.stringify({ ok: true }));
@@ -5577,7 +7145,10 @@ When users create specialized instances, suggest these configurations:
5577
7145
  res.writeHead(500);
5578
7146
  res.end(JSON.stringify({ error: `Start failed: ${err.message}` }));
5579
7147
  }
5580
- })();
7148
+ // The inner catch can itself throw (writeHead after a successful
7149
+ // writeHead is ERR_HTTP_HEADERS_SENT), and that rejection escapes the
7150
+ // IIFE. Same for the two handlers below.
7151
+ })().catch(err => this.logger.error({ err, name }, "HTTP start handler failed"));
5581
7152
  return;
5582
7153
  }
5583
7154
  // Instance restart (immediate, no idle wait)
@@ -5598,7 +7169,7 @@ When users create specialized instances, suggest these configurations:
5598
7169
  res.writeHead(status);
5599
7170
  res.end(JSON.stringify({ error: `Restart failed: ${err.message}` }));
5600
7171
  }
5601
- })();
7172
+ })().catch(err => this.logger.error({ err, name }, "HTTP restart handler failed"));
5602
7173
  return;
5603
7174
  }
5604
7175
  if (req.method === "POST" && req.url?.startsWith("/stop/")) {
@@ -5621,7 +7192,7 @@ When users create specialized instances, suggest these configurations:
5621
7192
  res.writeHead(500);
5622
7193
  res.end(JSON.stringify({ error: `Stop failed: ${err.message}` }));
5623
7194
  }
5624
- })();
7195
+ })().catch(err => this.logger.error({ err, name }, "HTTP stop handler failed"));
5625
7196
  return;
5626
7197
  }
5627
7198
  // ── Agent CLI endpoint ─────
@@ -5633,6 +7204,8 @@ When users create specialized instances, suggest these configurations:
5633
7204
  const url = new URL(req.url ?? "/", `http://localhost:${port}`);
5634
7205
  if (handleViewRequest(req, res, url, this))
5635
7206
  return;
7207
+ if (handleUsageRequest(req, res, url, this))
7208
+ return;
5636
7209
  if (handleSettingsRequest(req, res, url, this))
5637
7210
  return;
5638
7211
  if (handleWebRequest(req, res, url, this))
@@ -5640,10 +7213,20 @@ When users create specialized instances, suggest these configurations:
5640
7213
  res.writeHead(404);
5641
7214
  res.end(JSON.stringify({ error: "not found" }));
5642
7215
  });
7216
+ const markListening = (afterTakeover = false) => {
7217
+ this.healthServerListening = true;
7218
+ this.logger.info({ port }, afterTakeover
7219
+ ? "Health endpoint listening (after takeover)"
7220
+ : "Health endpoint listening");
7221
+ this.logger.info({ url: `http://localhost:${port}/ui?token=${this.webToken}` }, "Web UI available");
7222
+ this.logger.info({ url: `http://localhost:${port}/view?token=${this.viewToken}` }, "Web View available");
7223
+ };
5643
7224
  this.healthServer.on("error", (err) => {
7225
+ this.healthServerListening = false;
5644
7226
  if (err.code === "EADDRINUSE") {
5645
7227
  if (this.healthPortRetried) {
5646
- this.logger.debug({ port }, "Health port still in use after takeover — skipping health endpoint");
7228
+ this.logger.error({ err, port }, "Health port still in use after takeover — dashboard disabled");
7229
+ this.notifyFleetError(`⚠️ Dashboard unavailable — health port ${port} is already in use. Stop the conflicting process or configure a different health_port.`);
5647
7230
  return;
5648
7231
  }
5649
7232
  this.healthPortRetried = true;
@@ -5664,35 +7247,45 @@ When users create specialized instances, suggest these configurations:
5664
7247
  setTimeout(() => {
5665
7248
  if (!this.healthServer)
5666
7249
  return;
5667
- this.healthServer.listen(port, "127.0.0.1", () => {
5668
- this.logger.info({ port }, "Health endpoint listening (after takeover)");
5669
- });
7250
+ this.healthServer.listen(port, "127.0.0.1", () => markListening(true));
5670
7251
  }, 1500);
5671
7252
  return;
5672
7253
  }
5673
7254
  this.logger.error({ err, port }, "Health server error");
7255
+ this.notifyFleetError(`⚠️ Dashboard unavailable — health server failed: ${err.message}`);
5674
7256
  });
5675
- this.healthServer.listen(port, "127.0.0.1", () => {
5676
- this.logger.info({ port }, "Health endpoint listening");
5677
- });
5678
- this.logger.info({ url: `http://localhost:${port}/ui?token=${this.webToken}` }, "Web UI available");
5679
- this.logger.info({ url: `http://localhost:${port}/view?token=${this.viewToken}` }, "Web View available");
7257
+ this.healthServer.listen(port, "127.0.0.1", () => markListening());
5680
7258
  }
5681
7259
  getUiStatus() {
5682
- const instances = Object.keys(this.fleetConfig?.instances ?? {}).map(name => {
7260
+ const fleetNames = Object.keys(this.fleetConfig?.instances ?? {});
7261
+ // Classic rooms live only in classicBot.yaml — /api/profiles merges them into
7262
+ // the View roster, but previously getUiStatus skipped them so context_pct was
7263
+ // always 0 (live map miss → l?.context_pct ?? 0).
7264
+ const classicOnly = (this.classicChannels?.getAll() ?? [])
7265
+ .map(ch => ch.instanceName)
7266
+ .filter(name => !fleetNames.includes(name));
7267
+ const names = [...fleetNames, ...classicOnly];
7268
+ const instances = names.map(name => {
5683
7269
  const statusFile = join(this.getInstanceDir(name), "statusline.json");
5684
- let context_pct = 0;
5685
7270
  let cost = 0;
5686
7271
  let model = "";
5687
7272
  try {
5688
7273
  const data = JSON.parse(readFileSync(statusFile, "utf-8"));
5689
- context_pct = data.context_window?.used_percentage ?? 0;
5690
7274
  cost = data.cost?.total_cost_usd ?? 0;
5691
7275
  model = data.model?.display_name ?? "";
5692
7276
  }
5693
7277
  catch (err) {
5694
7278
  this.logger.debug({ err, name }, "statusline.json read failed (getUiStatus)");
5695
7279
  }
7280
+ // Align with /ctx: statusline for claude-code, pane scrape for kiro/grok/codex.
7281
+ const classic = classicOnly.includes(name);
7282
+ const backend = classic
7283
+ ? this.classicChannels.getBackendByInstance(name, this.fleetConfig?.defaults?.backend)
7284
+ : (this.fleetConfig?.instances[name]?.backend
7285
+ ?? this.fleetConfig?.defaults?.backend
7286
+ ?? "claude-code");
7287
+ const { context } = resolveInstanceContext(this.dataDir, name, backend);
7288
+ const context_pct = context ?? 0;
5696
7289
  return { name, status: this.getInstanceStatus(name), context_pct, cost, model };
5697
7290
  });
5698
7291
  return {