@songsid/agend 2.1.2-beta.2 → 2.1.2-beta.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/dist/access-path.js +15 -6
  2. package/dist/access-path.js.map +1 -1
  3. package/dist/agent-endpoint.d.ts +2 -0
  4. package/dist/agent-endpoint.js +17 -1
  5. package/dist/agent-endpoint.js.map +1 -1
  6. package/dist/backend/antigravity.d.ts +18 -0
  7. package/dist/backend/antigravity.js +32 -3
  8. package/dist/backend/antigravity.js.map +1 -1
  9. package/dist/backend/claude-code.d.ts +37 -0
  10. package/dist/backend/claude-code.js +43 -1
  11. package/dist/backend/claude-code.js.map +1 -1
  12. package/dist/backend/codex.d.ts +1 -0
  13. package/dist/backend/codex.js +25 -0
  14. package/dist/backend/codex.js.map +1 -1
  15. package/dist/backend/grok.d.ts +23 -0
  16. package/dist/backend/grok.js +45 -0
  17. package/dist/backend/grok.js.map +1 -1
  18. package/dist/backend/kiro.d.ts +20 -0
  19. package/dist/backend/kiro.js +77 -2
  20. package/dist/backend/kiro.js.map +1 -1
  21. package/dist/backend/types.d.ts +34 -0
  22. package/dist/backend/types.js +1 -1
  23. package/dist/backend/types.js.map +1 -1
  24. package/dist/channel/adapters/discord.d.ts +15 -0
  25. package/dist/channel/adapters/discord.js +91 -2
  26. package/dist/channel/adapters/discord.js.map +1 -1
  27. package/dist/channel/adapters/telegram.d.ts +21 -0
  28. package/dist/channel/adapters/telegram.js +82 -0
  29. package/dist/channel/adapters/telegram.js.map +1 -1
  30. package/dist/channel/ipc-bridge.d.ts +17 -1
  31. package/dist/channel/ipc-bridge.js +53 -17
  32. package/dist/channel/ipc-bridge.js.map +1 -1
  33. package/dist/channel/ipc-timeouts.d.ts +46 -0
  34. package/dist/channel/ipc-timeouts.js +65 -0
  35. package/dist/channel/ipc-timeouts.js.map +1 -0
  36. package/dist/channel/mcp-server.js +30 -14
  37. package/dist/channel/mcp-server.js.map +1 -1
  38. package/dist/channel/mcp-tools.js +28 -1
  39. package/dist/channel/mcp-tools.js.map +1 -1
  40. package/dist/channel/reconnect-backoff.d.ts +17 -0
  41. package/dist/channel/reconnect-backoff.js +21 -0
  42. package/dist/channel/reconnect-backoff.js.map +1 -0
  43. package/dist/channel/types.d.ts +29 -0
  44. package/dist/classic-channel-manager.js +1 -5
  45. package/dist/classic-channel-manager.js.map +1 -1
  46. package/dist/cli.js +132 -52
  47. package/dist/cli.js.map +1 -1
  48. package/dist/completion.d.ts +27 -0
  49. package/dist/completion.js +121 -0
  50. package/dist/completion.js.map +1 -0
  51. package/dist/config-validator.js +9 -0
  52. package/dist/config-validator.js.map +1 -1
  53. package/dist/config.js +1 -0
  54. package/dist/config.js.map +1 -1
  55. package/dist/cost-guard.d.ts +3 -1
  56. package/dist/cost-guard.js +3 -1
  57. package/dist/cost-guard.js.map +1 -1
  58. package/dist/daemon.d.ts +219 -13
  59. package/dist/daemon.js +919 -303
  60. package/dist/daemon.js.map +1 -1
  61. package/dist/event-log.d.ts +31 -0
  62. package/dist/event-log.js +96 -0
  63. package/dist/event-log.js.map +1 -1
  64. package/dist/fleet-manager.d.ts +238 -2
  65. package/dist/fleet-manager.js +1008 -90
  66. package/dist/fleet-manager.js.map +1 -1
  67. package/dist/general-knowledge/skills/cross-instance-messaging/SKILL.md +22 -0
  68. package/dist/general-knowledge/skills/model-discovery/SKILL.md +14 -23
  69. package/dist/general-knowledge/skills/session-management/SKILL.md +15 -34
  70. package/dist/hang-detector.d.ts +19 -13
  71. package/dist/hang-detector.js +19 -49
  72. package/dist/hang-detector.js.map +1 -1
  73. package/dist/instance-lifecycle.d.ts +13 -1
  74. package/dist/instance-lifecycle.js +84 -3
  75. package/dist/instance-lifecycle.js.map +1 -1
  76. package/dist/instructions.d.ts +5 -0
  77. package/dist/instructions.js +9 -11
  78. package/dist/instructions.js.map +1 -1
  79. package/dist/locale.js +2 -1
  80. package/dist/locale.js.map +1 -1
  81. package/dist/logger.js +14 -0
  82. package/dist/logger.js.map +1 -1
  83. package/dist/mcp-liveness.d.ts +21 -0
  84. package/dist/mcp-liveness.js +27 -0
  85. package/dist/mcp-liveness.js.map +1 -0
  86. package/dist/outbound-handlers.d.ts +16 -0
  87. package/dist/outbound-handlers.js +121 -27
  88. package/dist/outbound-handlers.js.map +1 -1
  89. package/dist/outbound-schemas.d.ts +8 -3
  90. package/dist/outbound-schemas.js +13 -2
  91. package/dist/outbound-schemas.js.map +1 -1
  92. package/dist/pane-write-lock.d.ts +48 -0
  93. package/dist/pane-write-lock.js +73 -0
  94. package/dist/pane-write-lock.js.map +1 -0
  95. package/dist/process-memory.d.ts +31 -0
  96. package/dist/process-memory.js +79 -0
  97. package/dist/process-memory.js.map +1 -0
  98. package/dist/quickstart.js +17 -16
  99. package/dist/quickstart.js.map +1 -1
  100. package/dist/reply-dedup.d.ts +41 -0
  101. package/dist/reply-dedup.js +0 -0
  102. package/dist/reply-dedup.js.map +1 -0
  103. package/dist/scheduler/db.js +3 -0
  104. package/dist/scheduler/db.js.map +1 -1
  105. package/dist/sd-notify.d.ts +27 -0
  106. package/dist/sd-notify.js +33 -1
  107. package/dist/sd-notify.js.map +1 -1
  108. package/dist/secret-file.d.ts +33 -0
  109. package/dist/secret-file.js +36 -0
  110. package/dist/secret-file.js.map +1 -0
  111. package/dist/setup-wizard.js +9 -7
  112. package/dist/setup-wizard.js.map +1 -1
  113. package/dist/tmux-control.d.ts +58 -5
  114. package/dist/tmux-control.js +102 -14
  115. package/dist/tmux-control.js.map +1 -1
  116. package/dist/tmux-manager.d.ts +31 -1
  117. package/dist/tmux-manager.js +40 -11
  118. package/dist/tmux-manager.js.map +1 -1
  119. package/dist/topic-commands.d.ts +50 -10
  120. package/dist/topic-commands.js +242 -106
  121. package/dist/topic-commands.js.map +1 -1
  122. package/dist/tui-glyphs.d.ts +25 -0
  123. package/dist/tui-glyphs.js +26 -0
  124. package/dist/tui-glyphs.js.map +1 -0
  125. package/dist/types.d.ts +6 -0
  126. package/dist/ui/view.html +123 -2
  127. package/dist/usage/format-rich.d.ts +11 -0
  128. package/dist/usage/format-rich.js +108 -0
  129. package/dist/usage/format-rich.js.map +1 -0
  130. package/dist/usage/providers.d.ts +97 -0
  131. package/dist/usage/providers.js +834 -0
  132. package/dist/usage/providers.js.map +1 -0
  133. package/dist/usage/usage-api.d.ts +50 -0
  134. package/dist/usage/usage-api.js +111 -0
  135. package/dist/usage/usage-api.js.map +1 -0
  136. package/dist/view-api.d.ts +3 -0
  137. package/dist/view-api.js +11 -2
  138. package/dist/view-api.js.map +1 -1
  139. package/dist/web-api.js +5 -2
  140. package/dist/web-api.js.map +1 -1
  141. package/package.json +4 -1
  142. package/dist/channel/tool-tracker.d.ts +0 -13
  143. package/dist/channel/tool-tracker.js +0 -58
  144. package/dist/channel/tool-tracker.js.map +0 -1
  145. package/dist/daemon-entry.d.ts +0 -1
  146. package/dist/daemon-entry.js +0 -30
  147. package/dist/daemon-entry.js.map +0 -1
@@ -4,7 +4,9 @@ import { createServer } from "node:http";
4
4
  import { join, dirname, basename } from "node:path";
5
5
  import { fileURLToPath } from "node:url";
6
6
  import { getAgendHome, ensureWorkspaceGit } from "./paths.js";
7
- import { sdNotify } from "./sd-notify.js";
7
+ import { sdNotify, sdNotifyBlocking } from "./sd-notify.js";
8
+ import { readFleetMemory } from "./process-memory.js";
9
+ import { ReplyDeduper } from "./reply-dedup.js";
8
10
  import { isScalar, parseDocument } from "yaml";
9
11
  const __filename = fileURLToPath(import.meta.url);
10
12
  const __dirname = dirname(__filename);
@@ -21,12 +23,12 @@ import { IpcClient } from "./channel/ipc-bridge.js";
21
23
  import { createAdapter } from "./channel/factory.js";
22
24
  import { createBackend } from "./backend/factory.js";
23
25
  import { isModelCompatible } from "./backend/types.js";
24
- import { createLogger } from "./logger.js";
26
+ import { createLogger, rotateLogIfNeeded } from "./logger.js";
25
27
  import { processAttachments } from "./channel/attachment-handler.js";
26
28
  import { routeToolCall } from "./channel/tool-router.js";
27
29
  import { Scheduler } from "./scheduler/index.js";
28
30
  import { DEFAULT_SCHEDULER_CONFIG } from "./scheduler/index.js";
29
- import { TopicCommands, saveCommandForBackend, parseSaveFilename, parsePauseWakeCommand, SAVE_FILENAME_RE, SAVE_UNSUPPORTED_MSG } from "./topic-commands.js";
31
+ import { TopicCommands, saveCommandForBackend, parseSaveFilename, parsePauseWakeCommand, SAVE_FILENAME_RE, SAVE_UNSUPPORTED_MSG, resolveInstanceContext, forgetInstanceContext } from "./topic-commands.js";
30
32
  import { DailySummary } from "./daily-summary.js";
31
33
  import { WebhookEmitter } from "./webhook-emitter.js";
32
34
  import { TmuxControlClient } from "./tmux-control.js";
@@ -38,6 +40,7 @@ import { StatuslineWatcher } from "./statusline-watcher.js";
38
40
  import { outboundHandlers } from "./outbound-handlers.js";
39
41
  import { handleWebRequest, broadcastSseEvent } from "./web-api.js";
40
42
  import { handleViewRequest, isViewPath } from "./view-api.js";
43
+ import { handleUsageRequest, isUsagePath } from "./usage/usage-api.js";
41
44
  import { handleSettingsRequest } from "./settings-api.js";
42
45
  import { setLocale, detectLocale, t } from "./locale.js";
43
46
  import { handleAgentRequest } from "./agent-endpoint.js";
@@ -83,6 +86,63 @@ const CANCEL_BTN_MAX_RETRIES = 3;
83
86
  * buttons no clear trigger reached (e.g. a scheduled/HTTP turn that never called
84
87
  * reply). 5min (not the old 2s idle-watch) so Thinking isn't misread as idle. */
85
88
  const CANCEL_BTN_IDLE_CHECK_INTERVAL_MS = 5 * 60_000;
89
+ /**
90
+ * How long after a reply an instance gets to resume working before its cancel
91
+ * button is retired. A short turn ends with a reply and never works again → the
92
+ * button disappears ~2 minutes after the answer. A multi-step run replies
93
+ * mid-flight and keeps going → the grace check sees "working" and leaves the
94
+ * button alone (the idle edge retires it when the run really ends).
95
+ */
96
+ const REPLY_RETIRE_GRACE_MS = 2 * 60_000;
97
+ /**
98
+ * The daemon only broadcasts execution state on TRANSITIONS, so a long
99
+ * single-state run sends nothing for hours. The idle backstop therefore pokes a
100
+ * query each tick; a live daemon answers within milliseconds and refreshes the
101
+ * cache. When nothing has refreshed it for this long despite those pokes, the
102
+ * reporting chain (daemon, IPC, or state monitor) is dead and a "working" state
103
+ * from 30 minutes ago proves nothing — the button may be retired.
104
+ */
105
+ const STATE_REPORT_STALE_MS = 30 * 60_000;
106
+ /**
107
+ * Unconditional ceiling on a cancel button's life. Deliberately far beyond any
108
+ * legitimate run (multi-hour tasks are normal on this fleet): everything below
109
+ * this is decided by real state; a button that somehow survives a full day is
110
+ * wreckage, stuck or not.
111
+ */
112
+ const CANCEL_BTN_MAX_LIFETIME_MS = 24 * 60 * 60_000;
113
+ /** Orphaned-button ledger, swept at startup. Lives in the fleet data dir. */
114
+ const CANCEL_BTN_LEDGER_FILE = "cancel-buttons.json";
115
+ /**
116
+ * How often the cancel button's text is refreshed with elapsed working time.
117
+ *
118
+ * One edit per working instance per interval — at 60s that is trivial for both
119
+ * platforms' rate limits, and it reads as a live counter rather than a stale
120
+ * snapshot. Nothing new is posted, so the channel is never spammed: there is
121
+ * exactly one progress message per turn, and it is the cancel button itself.
122
+ */
123
+ const PROGRESS_UPDATE_INTERVAL_MS = 60_000;
124
+ /** Elapsed time is only shown once work has clearly outlasted a quick answer. */
125
+ const PROGRESS_MIN_ELAPSED_MS = 2 * 60_000;
126
+ /** How much of a tool summary the progress line will show before eliding. */
127
+ const PROGRESS_ACTIVITY_MAX_CHARS = 48;
128
+ /**
129
+ * Emoji AgEnD itself stamps on messages as machine signals: the delivery-status
130
+ * ladder (⏳ queued, 👀 delivered, ✅ confirmed, ❌ failed) and the 🫡 ack. They
131
+ * are indicators, not opinions, so they never belong in the reactions queue —
132
+ * regardless of who they arrive from, because the adapter-level bot filter can
133
+ * miss (Discord partial users may not carry the `bot` flag). The cost is that a
134
+ * human genuinely reacting ✅ is also dropped; with four bot-stamped meanings
135
+ * already attached to these glyphs, that reading is ambiguous anyway.
136
+ */
137
+ const DELIVERY_STATUS_EMOJIS = new Set(["👀", "⏳", "✅", "❌", "🫡"]);
138
+ /**
139
+ * How long a delivery waits out a disconnected instance IPC before giving up.
140
+ *
141
+ * Sized for a daemon restart (socket close → respawn → CLI ready), which is the
142
+ * event this exists for. Past it the delivery fails loudly as it always did.
143
+ */
144
+ const IPC_RECONNECT_GRACE_MS = 30_000;
145
+ const IPC_RECONNECT_POLL_MS = 250;
86
146
  const CLASSIC_BACKEND_SELECTION_TIMEOUT_MS = 60_000;
87
147
  const CLASSIC_BACKEND_CALLBACK_PREFIX = "classic-backend:";
88
148
  const MODEL_SELECT_CALLBACK_PREFIX = "model-select:";
@@ -93,7 +153,7 @@ export class FleetManager {
93
153
  static sighupHandlerInstalled = false;
94
154
  children = new Map();
95
155
  lifecycle;
96
- /** @deprecated Use lifecycle.daemons — kept for backward compat */
156
+ /** Live view of lifecycle.daemons — used throughout; not deprecated. */
97
157
  get daemons() { return this.lifecycle.daemons; }
98
158
  fleetConfig = null;
99
159
  rawFleetConfig = {};
@@ -151,6 +211,12 @@ export class FleetManager {
151
211
  // reply, on cancel, or when a newer button supersedes it for the same
152
212
  // instance. Per-button tracking means a failed delete never strands a button.
153
213
  cancelButtons = new Map();
214
+ /** Duplicate-reply suppression across both the MCP and HTTP reply paths. */
215
+ replyDeduper = new ReplyDeduper();
216
+ /** instanceName → what it is doing right now, when the backend can tell us. */
217
+ instanceActivity = new Map();
218
+ /** instanceName → tail of deliveries waiting for its IPC to come back. */
219
+ ipcWaitTails = new Map();
154
220
  // Last user message delivered to each instance — used to react ✅ on completion.
155
221
  lastInboundMsg = new Map();
156
222
  topicArchiver;
@@ -163,6 +229,10 @@ export class FleetManager {
163
229
  failoverActive = new Map(); // instance → current failover model
164
230
  // IPC reconnect: tracks instances being intentionally stopped (skip reconnect)
165
231
  ipcStoppingInstances = new Set();
232
+ /** Coalesce concurrent connection attempts for the same daemon socket. */
233
+ ipcConnectInFlight = new Map();
234
+ /** At most one reconnect/backoff loop may exist per instance. */
235
+ ipcReconnectInFlight = new Map();
166
236
  // Adapter restart: prevents re-entrant restart attempts
167
237
  adapterRestarting = new Set();
168
238
  // Adapter isolation: track state per adapter for retry + visibility
@@ -172,6 +242,10 @@ export class FleetManager {
172
242
  healthServer = null;
173
243
  healthPortRetried = false;
174
244
  updateCheckTimer = null;
245
+ eventLogPruneTimer = null;
246
+ logRotateTimer = null;
247
+ /** Days of event/activity history to keep. */
248
+ static EVENT_LOG_RETENTION_DAYS = 30;
175
249
  watchdogTimer = null;
176
250
  startedAt = 0;
177
251
  // Mirror topic: buffer cross-instance messages, flush every 3s
@@ -210,7 +284,13 @@ export class FleetManager {
210
284
  }
211
285
  this.reloadPending = false;
212
286
  this.reconcileInFlight = this.reconcileInstances()
213
- .catch(err => this.logger.error({ err }, "SIGHUP config reload failed"))
287
+ .catch(err => {
288
+ // Almost always a YAML parse error. Log-only meant the user edited
289
+ // fleet.yaml, sent SIGHUP, and got no reaction and no explanation.
290
+ this.logger.error({ err }, "SIGHUP config reload failed");
291
+ const message = err instanceof Error ? err.message : String(err);
292
+ this.notifyFleetError(`⚠️ fleet.yaml reload FAILED — ${message}\nThe previous configuration is still running.`);
293
+ })
214
294
  .finally(() => {
215
295
  this.reconcileInFlight = null;
216
296
  if (this.reloadPending && this.startupComplete) {
@@ -222,12 +302,35 @@ export class FleetManager {
222
302
  this.startupComplete = true;
223
303
  if (this.reloadPending)
224
304
  this.scheduleReconcile();
305
+ void this.sweepOrphanedCancelButtons();
225
306
  }
226
307
  // ── ArchiverContext bridge ────────────────────────────────────────────
227
308
  lastActivityMs(name) {
228
309
  return this.lastActivity.get(name) ?? 0;
229
310
  }
311
+ /**
312
+ * Is the instance between turns?
313
+ *
314
+ * Prefers the daemon's pane state machine (debounced, busy-pattern aware) over
315
+ * the control client's raw 2-second output-silence heuristic. The raw heuristic
316
+ * reads every >2s output lull as idle — and long silent tools (a build, a test
317
+ * run) or an LLM pause produce those constantly mid-turn. That misreading is
318
+ * what retired cancel buttons in the middle of long work (the 5-minute backstop
319
+ * fired during a lull) and froze their progress text (ticker skipped "idle"
320
+ * ticks). The silence heuristic remains only as the fallback for instances
321
+ * whose daemon has not reported a state yet.
322
+ */
230
323
  getInstanceIdle(name) {
324
+ // A daemon that is not running cannot be mid-turn. This is what a stale
325
+ // "working" cache after a hard daemon kill (SIGKILL/OOM — no IPC crash
326
+ // report ever arrives) must not override.
327
+ if (this.getInstanceStatus(name) !== "running")
328
+ return true;
329
+ const state = this.getInstanceExecutionState(name);
330
+ if (state === "working" || state === "stuck")
331
+ return false;
332
+ if (state === "idle")
333
+ return true;
231
334
  try {
232
335
  const widFile = join(this.getInstanceDir(name), "window-id");
233
336
  if (!existsSync(widFile))
@@ -239,6 +342,18 @@ export class FleetManager {
239
342
  return true;
240
343
  }
241
344
  }
345
+ /**
346
+ * True when the instance claims working/stuck but nothing has refreshed that
347
+ * claim for STATE_REPORT_STALE_MS despite the backstop's per-tick queries.
348
+ * Measures the CACHE's age, not the button's — a healthy multi-hour run
349
+ * answers every query and never trips this.
350
+ */
351
+ stateReportDead(name) {
352
+ const cached = this.instanceStateCache.get(name);
353
+ if (!cached)
354
+ return false; // no claim to distrust — getInstanceIdle owns this case
355
+ return Date.now() - cached.receivedAt > STATE_REPORT_STALE_MS;
356
+ }
242
357
  // ── LifecycleContext bridge methods ──────────────────────────────────────
243
358
  webhookEmit(event, name, data) {
244
359
  this.webhookEmitter?.emit(event, name, data);
@@ -467,14 +582,21 @@ export class FleetManager {
467
582
  unchangedForMs: numberOr(msg.unchangedForMs, previous?.unchangedForMs ?? 0),
468
583
  observedAt: numberOr(msg.observedAt, now),
469
584
  stateChangedAt: numberOr(msg.stateChangedAt, previous?.state === state ? previous.stateChangedAt : now),
585
+ // Fleet-manager receipt time, NOT the daemon's observation time: staleness
586
+ // asks "is anyone still reporting", which only the receiver can date.
587
+ receivedAt: now,
470
588
  });
471
589
  for (const check of this.instanceIdleWaiters.get(name) ?? [])
472
590
  check();
473
591
  // warm_cap: a fresh transition into idle may free this instance for eviction,
474
592
  // or (more usefully) reveal that the fleet is now over cap. Only fire on the
475
593
  // edge into idle, not on every idle heartbeat.
476
- if (state === "idle" && previous?.state !== "idle")
594
+ if (state === "idle" && previous?.state !== "idle") {
477
595
  this.enforceWarmCap();
596
+ // The turn is genuinely over — retire the cancel/progress button now rather
597
+ // than waiting for the 5-minute idle backstop to notice.
598
+ this.retireInstanceButtons(name);
599
+ }
478
600
  }
479
601
  cacheInstanceProcessStatus(name, status) {
480
602
  if (status === "running") {
@@ -579,12 +701,70 @@ export class FleetManager {
579
701
  if (!idle) {
580
702
  this.logger.warn({ instanceName, timeoutMs }, "Idle gate timed out; forcing delivery");
581
703
  }
582
- const ipc = this.instanceIpcClients.get(instanceName);
583
- if (!ipc?.connected)
584
- throw new Error(`Instance '${instanceName}' IPC is unavailable`);
585
- ipc.send(payload);
704
+ await this.sendWhenConnected(instanceName, payload);
586
705
  this.lastDeliveryAt.set(instanceName, Date.now());
587
706
  }
707
+ /**
708
+ * Hand a payload to an instance's IPC, waiting out a *transient* disconnect.
709
+ *
710
+ * A daemon that is restarting — `/restart`, crash recovery, a model switch —
711
+ * drops its socket for a few seconds. Any message arriving in that window used
712
+ * to fail instantly: the caller logged a warning, put ❌ on the user's message,
713
+ * and the message was gone. The user had to notice the ❌ and retype it. That is
714
+ * the "instance 訊息不容易掉" goal failing on the most predictable event there is.
715
+ *
716
+ * The wait is bounded. If the instance is genuinely down, this still throws and
717
+ * the ❌ still appears — just for a real failure rather than a restart.
718
+ *
719
+ * Ordering is preserved by serialising behind any waiter already queued for this
720
+ * instance, *including* when the socket happens to be up: otherwise a message
721
+ * arriving after the reconnect could overtake one that has been waiting for it.
722
+ */
723
+ async sendWhenConnected(instanceName, payload) {
724
+ const queued = this.ipcWaitTails.get(instanceName);
725
+ if (!queued) {
726
+ const ipc = this.instanceIpcClients.get(instanceName);
727
+ if (ipc?.connected && ipc.send(payload))
728
+ return;
729
+ }
730
+ const attempt = (queued ?? Promise.resolve())
731
+ .catch(() => { })
732
+ .then(() => this.sendAfterIpcReturns(instanceName, payload));
733
+ // The chain stores a settled-either-way promise so one failed delivery cannot
734
+ // wedge every later one, and so `queued` above is safe to await unguarded.
735
+ const tail = attempt.catch(() => { });
736
+ this.ipcWaitTails.set(instanceName, tail);
737
+ try {
738
+ await attempt;
739
+ }
740
+ finally {
741
+ // Only the last waiter clears the chain; while a queue is still draining the
742
+ // map must keep pointing at it or ordering is lost.
743
+ if (this.ipcWaitTails.get(instanceName) === tail) {
744
+ this.ipcWaitTails.delete(instanceName);
745
+ }
746
+ }
747
+ }
748
+ /** Poll for the instance's IPC to come back, then send. Throws if it does not. */
749
+ async sendAfterIpcReturns(instanceName, payload) {
750
+ const deadline = Date.now() + IPC_RECONNECT_GRACE_MS;
751
+ let warned = false;
752
+ for (;;) {
753
+ // Re-read every round: a reconnect replaces the IpcClient object entirely,
754
+ // so a cached reference would stay dead forever.
755
+ const ipc = this.instanceIpcClients.get(instanceName);
756
+ if (ipc?.connected && ipc.send(payload))
757
+ return;
758
+ if (Date.now() >= deadline) {
759
+ throw new Error(`Instance '${instanceName}' IPC is unavailable`);
760
+ }
761
+ if (!warned) {
762
+ warned = true;
763
+ this.logger.info({ instanceName }, "Instance IPC is down — holding delivery until it reconnects");
764
+ }
765
+ await new Promise(resolve => setTimeout(resolve, IPC_RECONNECT_POLL_MS));
766
+ }
767
+ }
588
768
  /** Single delivery facade: wake paused CLIs and serialize non-user work behind idle. */
589
769
  async deliverToInstance(instanceName, payload, options = {}) {
590
770
  const meta = payload.meta && typeof payload.meta === "object"
@@ -600,10 +780,7 @@ export class FleetManager {
600
780
  await this.lifecycle.wake(instanceName, 30_000);
601
781
  this.enforceWarmCap(instanceName); // woke one → evict a different LRU idle if over cap
602
782
  }
603
- const ipc = this.instanceIpcClients.get(instanceName);
604
- if (!ipc?.connected)
605
- throw new Error(`Instance '${instanceName}' IPC is unavailable`);
606
- ipc.send(payload);
783
+ await this.sendWhenConnected(instanceName, payload);
607
784
  // A cross-instance item arriving before the daemon observes this turn as
608
785
  // working must not trust the stale idle snapshot from before the send.
609
786
  this.lastDeliveryAt.set(instanceName, Date.now());
@@ -651,7 +828,7 @@ export class FleetManager {
651
828
  await new Promise(resolve => setTimeout(resolve, 250));
652
829
  await this.startClassicInstance(instanceName, this.classicChannels.getBackendByInstance(instanceName, this.fleetConfig?.defaults?.backend), this.classicChannels.getPreTaskCommand(channel.channelId, channel.adapterId), this.classicChannels.getModel(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.model), this.classicChannels.getAutoPauseAfter(channel.channelId, channel.adapterId, this.fleetConfig?.defaults?.auto_pause_after));
653
830
  }
654
- async startInstance(name, config, topicMode) {
831
+ async startInstance(name, config, topicMode, kind = "fleet-topic") {
655
832
  if (this.lifecycle.isPaused(name)) {
656
833
  this.logger.info({ name }, "Persisted paused instance — skipping startup");
657
834
  return;
@@ -667,7 +844,11 @@ export class FleetManager {
667
844
  this.ensureGeneralInstructions(config.working_directory, config.backend);
668
845
  }
669
846
  this.instanceProcessStatus.delete(name);
670
- await this.lifecycle.start(name, config, topicMode);
847
+ await this.lifecycle.start(name, config, topicMode, {
848
+ kind,
849
+ backend: config.backend ?? this.fleetConfig?.defaults?.backend ?? "claude-code",
850
+ model: this.resolveInstanceModel(name).display,
851
+ });
671
852
  // Auto-connect IPC — daemon.start() ensures socket is ready before resolving
672
853
  await this.connectIpcToInstance(name);
673
854
  }
@@ -845,7 +1026,6 @@ export class FleetManager {
845
1026
  this.configPath = configPath;
846
1027
  this.loadEnvFile();
847
1028
  // Rotate fleet.log if oversized (before any logging)
848
- const { rotateLogIfNeeded } = await import("./logger.js");
849
1029
  rotateLogIfNeeded(join(this.dataDir, "fleet.log"));
850
1030
  const fleet = this.loadConfig(configPath);
851
1031
  setLocale(detectLocale(fleet)); // user-facing text language (fleet.yaml defaults.locale / timezone)
@@ -889,7 +1069,7 @@ export class FleetManager {
889
1069
  }
890
1070
  const pidPath = join(this.dataDir, "fleet.pid");
891
1071
  writeFileSync(pidPath, String(process.pid), "utf-8");
892
- this.eventLog = new EventLog(join(this.dataDir, "events.db"));
1072
+ this.eventLog = this.openEventLog();
893
1073
  // Initialize classic channel manager. The primary adapter (channels[0])
894
1074
  // migrates legacy single-bot entries and names without a suffix. Classic
895
1075
  // routing does NOT go through the routing engine (single-key, can't hold two
@@ -978,6 +1158,10 @@ export class FleetManager {
978
1158
  // Rotate fleet.log daily too (besides the startup size check above), so a
979
1159
  // long-running fleet doesn't accumulate an unbounded log.
980
1160
  rotateLogIfNeeded(join(this.dataDir, "fleet.log"));
1161
+ // Instance output.log is pipe-pane (TUI ANSI). Daemon health ticks rotate a
1162
+ // running instance's own log; this sweep is the safety net for every other
1163
+ // kind. One implementation, so the two cannot cover different sets.
1164
+ this.rotateAllInstanceLogs();
981
1165
  }, () => {
982
1166
  const instances = Object.keys(this.fleetConfig?.instances ?? {});
983
1167
  const costMap = new Map();
@@ -1117,9 +1301,29 @@ export class FleetManager {
1117
1301
  }
1118
1302
  }
1119
1303
  }
1120
- // Signal systemd: generals ready
1121
- sdNotify("READY=1");
1304
+ // The systemd watchdog answers exactly one question: is this process still
1305
+ // turning its event loop? Pinging from a timer proves that, and after the
1306
+ // blocking child-process calls were made async it is a meaningful signal —
1307
+ // a deadlocked or frozen fleet stops pinging and systemd restarts it.
1308
+ //
1309
+ // It deliberately does NOT gate on fleet health. "No adapter connected" or
1310
+ // "an instance crashed" must not kill the process: the fleet would be restarted
1311
+ // into the same broken state, and a user who has legitimately stopped every
1312
+ // instance would get a restart loop. Those conditions surface through /health
1313
+ // (which now returns 503) and through the General-topic notifications instead.
1122
1314
  this.watchdogTimer = setInterval(() => sdNotify("WATCHDOG=1"), 30_000);
1315
+ // EventLog.prune() existed but was never called, so `events` and `activity`
1316
+ // grew without bound for the life of the install. Prune once at startup and
1317
+ // daily after that; the timer is unref'd so it never holds the loop open.
1318
+ this.pruneEventLog();
1319
+ this.eventLogPruneTimer = setInterval(() => this.pruneEventLog(), 24 * 60 * 60_000);
1320
+ this.eventLogPruneTimer.unref?.();
1321
+ // Same shape for pipe-pane logs, and for the same reason: the only sweep that
1322
+ // covered them lived inside the daily-summary callback, so it did not run at
1323
+ // all when summaries were off.
1324
+ this.rotateAllInstanceLogs();
1325
+ this.logRotateTimer = setInterval(() => this.rotateAllInstanceLogs(), 24 * 60 * 60_000);
1326
+ this.logRotateTimer.unref?.();
1123
1327
  // Phase 2: Start remaining instances with staggered concurrency
1124
1328
  if (others.length > 0) {
1125
1329
  await this.startInstancesWithConcurrency(others, topicMode);
@@ -1255,6 +1459,15 @@ export class FleetManager {
1255
1459
  // rest of startup finishes. Replay one coalesced reload only after all
1256
1460
  // startup-owned lifecycle work and signal handlers are in place.
1257
1461
  this.finishStartup();
1462
+ // Tell systemd we are ready only now. This used to fire right after the
1463
+ // generals started — before adapters, classic instances, topic creation and the
1464
+ // health server — so `systemctl start` returned success while the fleet was
1465
+ // still deaf: no path existed for a user message to arrive.
1466
+ sdNotify("READY=1");
1467
+ const health = this.getFleetHealth();
1468
+ if (health.status !== "ok") {
1469
+ this.logger.warn({ health }, "Fleet started with problems — see /health");
1470
+ }
1258
1471
  }
1259
1472
  /**
1260
1473
  * Delete inbox files older than retentionDays (by mtime). Cleans the shared
@@ -1371,6 +1584,67 @@ export class FleetManager {
1371
1584
  getAdapterStates() {
1372
1585
  return this.adapterState;
1373
1586
  }
1587
+ /**
1588
+ * Real, checkable fleet health for `/health` and the operator.
1589
+ *
1590
+ * `status` is:
1591
+ * - `ok` — at least one adapter connected and every configured instance
1592
+ * that should be running is running
1593
+ * - `degraded` — reachable, but something the operator should look at (an
1594
+ * adapter retrying, an instance crashed or stopped)
1595
+ * - `down` — the fleet cannot do its job: no adapter is connected, so no
1596
+ * message can arrive or be answered
1597
+ *
1598
+ * Deliberately does NOT gate the systemd watchdog — see the comment at the
1599
+ * WATCHDOG timer for why.
1600
+ */
1601
+ getFleetHealth() {
1602
+ const names = Object.keys(this.fleetConfig?.instances ?? {});
1603
+ const counts = { configured: names.length, running: 0, crashed: 0, paused: 0, stopped: 0 };
1604
+ for (const name of names) {
1605
+ const state = this.getInstanceStatus(name);
1606
+ if (state === "running")
1607
+ counts.running++;
1608
+ else if (state === "crashed")
1609
+ counts.crashed++;
1610
+ else if (state === "paused")
1611
+ counts.paused++;
1612
+ else
1613
+ counts.stopped++;
1614
+ }
1615
+ const states = {};
1616
+ let connected = 0;
1617
+ for (const [id, state] of this.adapterState) {
1618
+ states[id] = state.status;
1619
+ if (state.status === "connected")
1620
+ connected++;
1621
+ }
1622
+ const problems = [];
1623
+ if (this.adapterState.size > 0 && connected === 0)
1624
+ problems.push("no channel adapter is connected");
1625
+ if (counts.crashed > 0)
1626
+ problems.push(`${counts.crashed} instance(s) crashed`);
1627
+ for (const [id, state] of this.adapterState) {
1628
+ if (state.status !== "connected")
1629
+ problems.push(`adapter ${id} is ${state.status}`);
1630
+ }
1631
+ if (!this.startupComplete)
1632
+ problems.push("startup has not completed");
1633
+ // "down" is reserved for "cannot receive or answer a message at all". A fleet
1634
+ // with adapters configured but none connected is exactly that.
1635
+ const status = this.adapterState.size > 0 && connected === 0
1636
+ ? "down"
1637
+ : problems.length > 0 ? "degraded" : "ok";
1638
+ return {
1639
+ status,
1640
+ uptime: Math.floor((Date.now() - this.startedAt) / 1000),
1641
+ instances: counts,
1642
+ adapters: { total: this.adapterState.size, connected, states },
1643
+ startupComplete: this.startupComplete,
1644
+ memory: readFleetMemory(),
1645
+ problems,
1646
+ };
1647
+ }
1374
1648
  /** Start the primary adapter (backward-compatible, sets this.adapter) */
1375
1649
  async startSingleAdapter(fleet, channelConfig) {
1376
1650
  const botToken = process.env[channelConfig.bot_token_env];
@@ -1397,6 +1671,9 @@ export class FleetManager {
1397
1671
  this.adapter.on("message", safeHandler(async (msg) => {
1398
1672
  await this.handleInboundMessage(msg);
1399
1673
  }, this.logger, "adapter.message"));
1674
+ this.adapter.on("reaction", safeHandler(async (r) => {
1675
+ await this.handleInboundReaction(r);
1676
+ }, this.logger, "adapter.reaction"));
1400
1677
  this.adapter.on("callback_query", safeHandler(async (data) => {
1401
1678
  if (await this.handleClassicBackendSelection(data))
1402
1679
  return;
@@ -1571,19 +1848,29 @@ export class FleetManager {
1571
1848
  await data.respond(t("not_authorized"));
1572
1849
  return;
1573
1850
  }
1851
+ await data.respond(await this.runBackendDoctor());
1852
+ }
1853
+ else if (data.command === "usage") {
1854
+ // Same permission level as /ctx (none). The reply is still ephemeral —
1855
+ // the adapter defers non-chat commands that way — so it never spams the
1856
+ // channel either way.
1574
1857
  try {
1575
- const { execSync } = await import("node:child_process");
1576
- const backend = this.fleetConfig?.defaults?.backend || "claude-code";
1577
- const result = execSync(`agend backend doctor ${backend}`, { timeout: 30_000, encoding: "utf-8" });
1578
- const clean = result.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
1579
- await data.respond(clean || "No output");
1858
+ const { getUsageSnapshot } = await import("./usage/usage-api.js");
1859
+ const { renderUsageMarkdown } = await import("./usage/format-rich.js");
1860
+ // slash_command is Discord-only; editReply renders Markdown natively.
1861
+ await data.respond(renderUsageMarkdown(await getUsageSnapshot()));
1580
1862
  }
1581
1863
  catch (err) {
1582
- const output = (err.stdout ?? err.message ?? "Doctor failed").replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
1583
- await data.respond(output);
1864
+ await data.respond(`⚠️ Usage fetch failed: ${err.message}`);
1584
1865
  }
1585
1866
  }
1586
1867
  else if (data.command === "status") {
1868
+ // Admin-gated (like the topic path): the merged table shows every
1869
+ // instance's cost and IPC health.
1870
+ if (!this.isFleetAdmin(data.userId, adapterId)) {
1871
+ await data.respond(t("cmd.admin_required", "/status"));
1872
+ return;
1873
+ }
1587
1874
  const text = await this.topicCommands.getStatusText();
1588
1875
  await data.respond(text);
1589
1876
  }
@@ -1692,6 +1979,9 @@ export class FleetManager {
1692
1979
  adapter.on("message", safeHandler(async (msg) => {
1693
1980
  await this.handleInboundMessage(msg);
1694
1981
  }, this.logger, `adapter[${adapterId}].message`));
1982
+ adapter.on("reaction", safeHandler(async (r) => {
1983
+ await this.handleInboundReaction(r);
1984
+ }, this.logger, `adapter[${adapterId}].reaction`));
1695
1985
  adapter.on("callback_query", safeHandler(async (data) => {
1696
1986
  if (await this.handleClassicBackendSelection(data))
1697
1987
  return;
@@ -1853,19 +2143,29 @@ export class FleetManager {
1853
2143
  await data.respond(t("not_authorized"));
1854
2144
  return;
1855
2145
  }
2146
+ await data.respond(await this.runBackendDoctor());
2147
+ }
2148
+ else if (data.command === "usage") {
2149
+ // Same permission level as /ctx (none). The reply is still ephemeral —
2150
+ // the adapter defers non-chat commands that way — so it never spams the
2151
+ // channel either way.
1856
2152
  try {
1857
- const { execSync } = await import("node:child_process");
1858
- const backend = this.fleetConfig?.defaults?.backend || "claude-code";
1859
- const result = execSync(`agend backend doctor ${backend}`, { timeout: 30_000, encoding: "utf-8" });
1860
- const clean = result.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
1861
- await data.respond(clean || "No output");
2153
+ const { getUsageSnapshot } = await import("./usage/usage-api.js");
2154
+ const { renderUsageMarkdown } = await import("./usage/format-rich.js");
2155
+ // slash_command is Discord-only; editReply renders Markdown natively.
2156
+ await data.respond(renderUsageMarkdown(await getUsageSnapshot()));
1862
2157
  }
1863
2158
  catch (err) {
1864
- const output = (err.stdout ?? err.message ?? "Doctor failed").replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
1865
- await data.respond(output);
2159
+ await data.respond(`⚠️ Usage fetch failed: ${err.message}`);
1866
2160
  }
1867
2161
  }
1868
2162
  else if (data.command === "status") {
2163
+ // Admin-gated (like the topic path): the merged table shows every
2164
+ // instance's cost and IPC health.
2165
+ if (!this.isFleetAdmin(data.userId, adapterId)) {
2166
+ await data.respond(t("cmd.admin_required", "/status"));
2167
+ return;
2168
+ }
1869
2169
  const text = await this.topicCommands.getStatusText();
1870
2170
  await data.respond(text);
1871
2171
  }
@@ -1931,19 +2231,38 @@ export class FleetManager {
1931
2231
  this.logger.info({ adapterId, type: channelConfig.type }, "Additional adapter started");
1932
2232
  }
1933
2233
  /** Connect IPC to a single instance with all handlers */
1934
- async connectIpcToInstance(name) {
2234
+ connectIpcToInstance(name) {
2235
+ const inFlight = this.ipcConnectInFlight.get(name);
2236
+ if (inFlight)
2237
+ return inFlight;
2238
+ const connection = this.connectIpcToInstanceInternal(name)
2239
+ .finally(() => {
2240
+ if (this.ipcConnectInFlight.get(name) === connection) {
2241
+ this.ipcConnectInFlight.delete(name);
2242
+ }
2243
+ });
2244
+ this.ipcConnectInFlight.set(name, connection);
2245
+ return connection;
2246
+ }
2247
+ async connectIpcToInstanceInternal(name) {
1935
2248
  // Close existing client to prevent socket leak on reconnect
1936
2249
  const existing = this.instanceIpcClients.get(name);
1937
2250
  if (existing) {
1938
- this.ipcStoppingInstances.add(name);
2251
+ // Remove application listeners before destroying the socket. Even if a
2252
+ // future regression creates two clients, the replaced one cannot keep
2253
+ // handling fleet_outbound messages as an orphan.
2254
+ existing.removeAllListeners();
1939
2255
  try {
1940
- existing.close();
2256
+ await existing.close();
1941
2257
  }
1942
2258
  catch (err) {
1943
2259
  this.logger.debug({ err, name }, "IPC client close failed (likely already closed)");
1944
2260
  }
1945
- this.instanceIpcClients.delete(name);
1946
- this.ipcStoppingInstances.delete(name);
2261
+ finally {
2262
+ if (this.instanceIpcClients.get(name) === existing) {
2263
+ this.instanceIpcClients.delete(name);
2264
+ }
2265
+ }
1947
2266
  }
1948
2267
  const sockPath = join(this.getInstanceDir(name), "channel.sock");
1949
2268
  if (!existsSync(sockPath))
@@ -2002,6 +2321,9 @@ export class FleetManager {
2002
2321
  else if (msg.type === "instance_process_state") {
2003
2322
  this.cacheInstanceProcessStatus(name, msg.status);
2004
2323
  }
2324
+ else if (msg.type === "instance_activity") {
2325
+ this.cacheInstanceActivity(name, msg.activity);
2326
+ }
2005
2327
  else if (msg.type === "instance_state" || msg.type === "instance_state_response") {
2006
2328
  this.cacheInstanceExecutionState(name, msg);
2007
2329
  if (msg.type === "instance_state_response") {
@@ -2021,6 +2343,10 @@ export class FleetManager {
2021
2343
  }
2022
2344
  // Auto-reconnect on disconnect (unless intentionally stopping)
2023
2345
  ipc.on("disconnect", () => {
2346
+ // A delayed event from a replaced/stale client must never delete the
2347
+ // current connection or start another reconnect loop.
2348
+ if (this.instanceIpcClients.get(name) !== ipc)
2349
+ return;
2024
2350
  this.instanceIpcClients.delete(name);
2025
2351
  if (this.ipcStoppingInstances.has(name))
2026
2352
  return;
@@ -2032,7 +2358,20 @@ export class FleetManager {
2032
2358
  }
2033
2359
  }
2034
2360
  /** Attempt IPC reconnection with exponential backoff */
2035
- async ipcReconnect(name) {
2361
+ ipcReconnect(name) {
2362
+ const inFlight = this.ipcReconnectInFlight.get(name);
2363
+ if (inFlight)
2364
+ return inFlight;
2365
+ const reconnect = this.runIpcReconnect(name)
2366
+ .finally(() => {
2367
+ if (this.ipcReconnectInFlight.get(name) === reconnect) {
2368
+ this.ipcReconnectInFlight.delete(name);
2369
+ }
2370
+ });
2371
+ this.ipcReconnectInFlight.set(name, reconnect);
2372
+ return reconnect;
2373
+ }
2374
+ async runIpcReconnect(name) {
2036
2375
  for (let attempt = 1;; attempt++) {
2037
2376
  if (this.ipcStoppingInstances.has(name) || !this.daemons.has(name))
2038
2377
  return;
@@ -2055,9 +2394,22 @@ export class FleetManager {
2055
2394
  if (existsSync(windowIdPath)) {
2056
2395
  const windowId = readFileSync(windowIdPath, "utf-8").trim();
2057
2396
  if (windowId) {
2397
+ // Async with an explicit timeout: this was execSync with NO timeout at
2398
+ // all, so a wedged tmux server blocked the whole fleet event loop
2399
+ // indefinitely — while we were here to diagnose a lost connection.
2400
+ // A timeout is also the correct signal: an unresponsive tmux server
2401
+ // means we cannot verify the pane, which is treated as dead (the same
2402
+ // conclusion the old code reached only by throwing).
2058
2403
  try {
2059
- const { execSync } = await import("node:child_process");
2060
- execSync(`tmux list-panes -t "${windowId}"`, { stdio: "ignore" });
2404
+ const { execFile } = await import("node:child_process");
2405
+ const { promisify } = await import("node:util");
2406
+ const { getTmuxSocketName } = await import("./paths.js");
2407
+ // Honour socket isolation: without -L this queried the user's default
2408
+ // tmux server instead of the fleet's, so under a custom AGEND_HOME the
2409
+ // check was meaningless (it reported every pane dead).
2410
+ const socket = getTmuxSocketName();
2411
+ const args = socket ? ["-L", socket, "list-panes", "-t", windowId] : ["list-panes", "-t", windowId];
2412
+ await promisify(execFile)("tmux", args, { timeout: 5_000 });
2061
2413
  }
2062
2414
  catch {
2063
2415
  // Pane dead — respawn
@@ -2078,6 +2430,12 @@ export class FleetManager {
2078
2430
  if (this.adapterRestarting.has(id))
2079
2431
  return;
2080
2432
  this.adapterRestarting.add(id);
2433
+ // Reflect reality in adapterState throughout. This loop used to leave the state
2434
+ // untouched, so getAdapterStates() — and therefore /health and the dashboard —
2435
+ // kept reporting "connected" for an adapter that had been down for hours. An
2436
+ // adapter's true status was simply not knowable from inside the process.
2437
+ const previous = this.adapterState.get(id);
2438
+ this.adapterState.set(id, { status: "retrying", retryCount: 0, lastError: previous?.lastError });
2081
2439
  try {
2082
2440
  for (let attempt = 1;; attempt++) {
2083
2441
  if (this.ipcStoppingInstances.has("__fleet_stopping__"))
@@ -2090,9 +2448,16 @@ export class FleetManager {
2090
2448
  await adapter.stop().catch(() => { });
2091
2449
  await adapter.start();
2092
2450
  this.logger.info({ id, attempt }, "Adapter restarted successfully");
2451
+ this.adapterState.set(id, { status: "connected", retryCount: 0 });
2093
2452
  return;
2094
2453
  }
2095
- catch { /* retry */ }
2454
+ catch (err) {
2455
+ this.adapterState.set(id, {
2456
+ status: "retrying",
2457
+ retryCount: attempt,
2458
+ lastError: err?.message ?? String(err),
2459
+ });
2460
+ }
2096
2461
  if (attempt % 10 === 0) {
2097
2462
  this.logger.warn({ id, attempt }, "Adapter restart still failing");
2098
2463
  }
@@ -2128,6 +2493,49 @@ export class FleetManager {
2128
2493
  }
2129
2494
  return generals[0];
2130
2495
  }
2496
+ /**
2497
+ * A user reacted to one of the bot's messages (#408).
2498
+ *
2499
+ * A reaction is context, not a message (#432, reworking #413): it never triggers
2500
+ * an agent turn and never wakes anything. It is queued in the event log and rides
2501
+ * into the instance's NEXT real message as one compact leading line —
2502
+ * `[Recent reactions: 👍×2 from hanhanv]` — after which it is marked consumed.
2503
+ * No pending reactions → no line → zero context spent, which is the common case.
2504
+ */
2505
+ async handleInboundReaction(r) {
2506
+ const instanceName = this.resolveSlashTarget(r.threadId ?? r.chatId, r.adapterId);
2507
+ if (!instanceName) {
2508
+ this.logger.debug({ emoji: r.emoji, chatId: r.chatId }, "Reaction in an unrouted channel — ignoring");
2509
+ return;
2510
+ }
2511
+ if (DELIVERY_STATUS_EMOJIS.has(r.emoji)) {
2512
+ this.logger.debug({ emoji: r.emoji, user: r.username }, "Ignoring delivery-status emoji as a reaction");
2513
+ return;
2514
+ }
2515
+ this.eventLog?.logActivity("reaction", r.username, `${r.emoji} ${r.action}`, instanceName);
2516
+ if (r.action === "add") {
2517
+ this.eventLog?.addReaction(instanceName, r.messageId, r.username, r.emoji);
2518
+ }
2519
+ else {
2520
+ // Withdrawn before anyone saw it → it never happened. See removeReaction.
2521
+ this.eventLog?.removeReaction(instanceName, r.messageId, r.username, r.emoji);
2522
+ }
2523
+ }
2524
+ /**
2525
+ * The queued-reaction summary for an instance's next real message, or {} when
2526
+ * nothing is pending (the common case must add zero context). The consume
2527
+ * callback is separate from the fetch so reactions are only marked once the
2528
+ * message actually went out — a failed delivery keeps them queued.
2529
+ */
2530
+ pendingReactionsMeta(instanceName) {
2531
+ const pending = this.eventLog?.pendingReactions(instanceName);
2532
+ if (!pending)
2533
+ return { meta: {}, consume: () => { } };
2534
+ return {
2535
+ meta: { pending_reactions: pending.summary },
2536
+ consume: () => this.eventLog?.markReactionsConsumed(instanceName, pending.maxId),
2537
+ };
2538
+ }
2131
2539
  async handleInboundMessage(msg) {
2132
2540
  const threadId = msg.threadId || undefined;
2133
2541
  this.logger.debug({ source: msg.source, chatId: msg.chatId, threadId, userId: msg.userId, isBotMessage: msg.isBotMessage, textLen: (msg.text ?? "").length, text: (msg.text ?? "").slice(0, 80) }, "handleInboundMessage entry");
@@ -2465,6 +2873,7 @@ export class FleetManager {
2465
2873
  }
2466
2874
  this.warnIfRateLimited(generalInstance, msg);
2467
2875
  const { text, extraMeta } = await processAttachments(msg, inboundAdapter, this.logger, generalInstance);
2876
+ const generalReactions = this.pendingReactionsMeta(generalInstance);
2468
2877
  try {
2469
2878
  await this.deliverToInstance(generalInstance, {
2470
2879
  type: "fleet_inbound",
@@ -2480,9 +2889,11 @@ export class FleetManager {
2480
2889
  adapter_id: msg.adapterId,
2481
2890
  source: msg.source,
2482
2891
  ...(msg.replyToText ? { reply_to_text: msg.replyToText } : {}),
2892
+ ...generalReactions.meta,
2483
2893
  ...extraMeta,
2484
2894
  },
2485
2895
  });
2896
+ generalReactions.consume();
2486
2897
  this.lastInboundUser.set(generalInstance, msg.username);
2487
2898
  this.logger.info(`${msg.username} → ${generalInstance}: ${(text ?? "").slice(0, 100)}`);
2488
2899
  this.eventLog?.logActivity("message", msg.username, (text ?? "").slice(0, 200), generalInstance);
@@ -2558,6 +2969,7 @@ export class FleetManager {
2558
2969
  this.setTopicIcon(instanceName, "blue");
2559
2970
  this.warnIfRateLimited(instanceName, msg);
2560
2971
  const { text, extraMeta } = await processAttachments(msg, inboundAdapter, this.logger, instanceName);
2972
+ const reactions = this.pendingReactionsMeta(instanceName);
2561
2973
  try {
2562
2974
  await this.deliverToInstance(instanceName, {
2563
2975
  type: "fleet_inbound",
@@ -2573,9 +2985,13 @@ export class FleetManager {
2573
2985
  adapter_id: msg.adapterId,
2574
2986
  source: msg.source,
2575
2987
  ...(msg.replyToText ? { reply_to_text: msg.replyToText } : {}),
2988
+ ...reactions.meta,
2576
2989
  ...extraMeta,
2577
2990
  },
2578
2991
  });
2992
+ // Only after the message actually went out. A failed delivery keeps the
2993
+ // reactions queued for the retry / the next message.
2994
+ reactions.consume();
2579
2995
  }
2580
2996
  catch (err) {
2581
2997
  this.logger.warn({ err: err.message, instanceName }, "Wake/delivery failed");
@@ -2664,25 +3080,33 @@ export class FleetManager {
2664
3080
  delete args.thread_id;
2665
3081
  threadId = undefined;
2666
3082
  }
3083
+ // Reply dedup: retries land here when the agent was told a send failed
3084
+ // (daemon budget elapsed, shell tool killed) while the adapter send was
3085
+ // still in flight and about to succeed. One real send, everyone gets its
3086
+ // outcome; a genuinely failed send clears the entry so a retry passes.
3087
+ if (tool === "reply") {
3088
+ const ticket = this.replyDeduper.begin(instanceName, String(args.text ?? ""), Array.isArray(args.files) ? args.files : []);
3089
+ if (ticket.duplicate) {
3090
+ this.logger.info({ instanceName }, "Duplicate reply suppressed — replaying the original send's outcome");
3091
+ ticket.subscribe(respond);
3092
+ return;
3093
+ }
3094
+ const original = respond;
3095
+ const respondAndRecord = (result, error) => {
3096
+ ticket.complete(result, error);
3097
+ original(result, error);
3098
+ };
3099
+ if (routeToolCall(outAdapter, tool, args, threadId, respondAndRecord)) {
3100
+ this.afterReplyRouted(instanceName, args, senderSessionName);
3101
+ return;
3102
+ }
3103
+ // routeToolCall knows "reply"; not handling it means the world changed.
3104
+ ticket.complete(null, "reply not handled");
3105
+ original(null, "reply not handled");
3106
+ return;
3107
+ }
2667
3108
  // Route standard channel tools (reply, react, edit_message, download_attachment)
2668
3109
  if (routeToolCall(outAdapter, tool, args, threadId, respond)) {
2669
- if (tool === "reply") {
2670
- // Agent answered — retire its pending cancel button and mark ✅ done.
2671
- this.clearCancelButton(instanceName);
2672
- this.reactDone(instanceName);
2673
- const replyTo = this.lastInboundUser.get(instanceName) ?? "user";
2674
- this.logger.info(`${instanceName} → ${replyTo}: ${(args.text ?? "").slice(0, 100)}`);
2675
- this.emitSseEvent("message", {
2676
- instance: instanceName, sender: senderSessionName ?? instanceName,
2677
- text: (args.text ?? "").slice(0, 2000),
2678
- ts: new Date().toISOString(),
2679
- });
2680
- // Log bot reply to classic instance chat-log
2681
- const isClassic = this.classicChannels?.getChannelIdByInstance(instanceName) !== undefined;
2682
- if (isClassic) {
2683
- ClassicChannelManager.logMessage(instanceName, "bot", args.text ?? "", new Date());
2684
- }
2685
- }
2686
3110
  return;
2687
3111
  }
2688
3112
  // Log tool calls for activity visualization
@@ -2697,6 +3121,35 @@ export class FleetManager {
2697
3121
  respond(null, `Unknown tool: ${tool}`);
2698
3122
  }
2699
3123
  }
3124
+ /** Side effects of a routed reply: cancel-button lifecycle, logs, SSE, chat log. */
3125
+ afterReplyRouted(instanceName, args, senderSessionName) {
3126
+ // A reply is NOT proof the turn is over (#410) — but it is not proof of
3127
+ // more work either. Split the difference: an instance that is clearly
3128
+ // idle loses the button now; one that looks busy keeps it (re-posted
3129
+ // below the reply so it stays last in the channel), with a 2-minute
3130
+ // grace check — if it has NOT resumed working by then, the reply was the
3131
+ // end of the turn and the button goes. A multi-step run that keeps
3132
+ // working sails through the check and keeps its button.
3133
+ if (this.getInstanceIdle(instanceName)) {
3134
+ this.clearCancelButton(instanceName);
3135
+ }
3136
+ else {
3137
+ void this.sendCancelButton(instanceName).then(() => this.armReplyGrace(instanceName));
3138
+ }
3139
+ this.reactDone(instanceName);
3140
+ const replyTo = this.lastInboundUser.get(instanceName) ?? "user";
3141
+ this.logger.info(`${instanceName} → ${replyTo}: ${(args.text ?? "").slice(0, 100)}`);
3142
+ this.emitSseEvent("message", {
3143
+ instance: instanceName, sender: senderSessionName ?? instanceName,
3144
+ text: (args.text ?? "").slice(0, 2000),
3145
+ ts: new Date().toISOString(),
3146
+ });
3147
+ // Log bot reply to classic instance chat-log
3148
+ const isClassic = this.classicChannels?.getChannelIdByInstance(instanceName) !== undefined;
3149
+ if (isClassic) {
3150
+ ClassicChannelManager.logMessage(instanceName, "bot", args.text ?? "", new Date());
3151
+ }
3152
+ }
2700
3153
  /** Handle tool status update from a daemon instance */
2701
3154
  handleToolStatusFromInstance(instanceName, msg) {
2702
3155
  const statusAdapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
@@ -3299,6 +3752,9 @@ export class FleetManager {
3299
3752
  }
3300
3753
  }
3301
3754
  async removeInstance(name) {
3755
+ // Drop cached pane context — the map is keyed by instance name and nothing
3756
+ // else evicted deleted entries, so it grew for the life of the process.
3757
+ forgetInstanceContext(name);
3302
3758
  // Clean up schedules (scheduler is fleet-level, not lifecycle-level)
3303
3759
  const config = this.fleetConfig?.instances[name];
3304
3760
  if (this.scheduler && config?.topic_id) {
@@ -3404,6 +3860,171 @@ export class FleetManager {
3404
3860
  this.collabInstances.add(instanceName);
3405
3861
  return true;
3406
3862
  }
3863
+ /**
3864
+ * Open the event log, tolerating a corrupt file.
3865
+ *
3866
+ * `events.db` holds history only — event rows and the activity feed. Nothing the
3867
+ * fleet needs to run depends on it, and every consumer already uses
3868
+ * `this.eventLog?.`. An unguarded `new EventLog(...)` here meant a corrupt or
3869
+ * unreadable history file (a truncated WAL after a hard kill, a full disk)
3870
+ * threw during startAll and the WHOLE FLEET FAILED TO BOOT — trading every
3871
+ * running agent for a file whose only job is reporting.
3872
+ *
3873
+ * So: try, move a bad file aside and retry once with a fresh one, and if even
3874
+ * that fails carry on without an event log.
3875
+ */
3876
+ /**
3877
+ * Run `agend backend doctor` for the fleet's default backend and return its
3878
+ * cleaned output.
3879
+ *
3880
+ * Async on purpose: this was `execSync` with a 30s timeout, reachable by any
3881
+ * allowlisted user through `/doctor`. While it ran, the entire fleet event loop
3882
+ * was frozen — no IPC, no adapter, no message delivery, no health responses,
3883
+ * and critically no WATCHDOG ping, so a slow doctor could push past
3884
+ * WatchdogSec and have systemd SIGABRT the fleet.
3885
+ */
3886
+ async runBackendDoctor() {
3887
+ const stripAnsi = (s) => s.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
3888
+ const backend = this.fleetConfig?.defaults?.backend || "claude-code";
3889
+ try {
3890
+ const { execFile } = await import("node:child_process");
3891
+ const { promisify } = await import("node:util");
3892
+ // execFile with an argv array — no shell, so the backend name cannot be
3893
+ // interpreted as a command even if config is malformed.
3894
+ const { stdout } = await promisify(execFile)("agend", ["backend", "doctor", backend], {
3895
+ timeout: 30_000,
3896
+ encoding: "utf-8",
3897
+ });
3898
+ return stripAnsi(stdout) || "No output";
3899
+ }
3900
+ catch (err) {
3901
+ const e = err;
3902
+ return stripAnsi(e.stdout ?? e.message ?? "Doctor failed");
3903
+ }
3904
+ }
3905
+ /** Drop event/activity rows older than the retention window. Best-effort. */
3906
+ /**
3907
+ * Cap every instance's pipe-pane log, walking the instances **directory** rather
3908
+ * than the config.
3909
+ *
3910
+ * A running instance rotates its own log on each health tick, so the ones that
3911
+ * need this are the ones nothing else looks at:
3912
+ *
3913
+ * - deleted instances, whose directory outlives the config entry. Nothing ever
3914
+ * touched these again. On the machine this was found on, one held 122 MB and
3915
+ * another 74 MB, out of 622 MB of pipe-pane logs in total.
3916
+ * - classic instances, which live in classicChannels, not fleetConfig.instances,
3917
+ * and so were never in the old config-driven loop at all.
3918
+ * - stopped instances, which have no health tick running.
3919
+ *
3920
+ * pipe-pane writes raw TUI output, so a wedged splash screen can emit ANSI frames
3921
+ * at animation rate. Unbounded growth here fills the disk, which takes the whole
3922
+ * fleet down rather than one instance.
3923
+ */
3924
+ rotateAllInstanceLogs() {
3925
+ const root = join(this.dataDir, "instances");
3926
+ let entries;
3927
+ try {
3928
+ entries = readdirSync(root, { withFileTypes: true });
3929
+ }
3930
+ catch {
3931
+ return; // no instances directory yet
3932
+ }
3933
+ for (const entry of entries) {
3934
+ if (!entry.isDirectory())
3935
+ continue;
3936
+ // rotateLogIfNeeded is already best-effort and returns early on a missing
3937
+ // file, so a directory without a pipe-pane log costs one stat.
3938
+ rotateLogIfNeeded(join(root, entry.name, "output.log"));
3939
+ }
3940
+ }
3941
+ pruneEventLog() {
3942
+ try {
3943
+ this.eventLog?.prune(FleetManager.EVENT_LOG_RETENTION_DAYS);
3944
+ }
3945
+ catch (err) {
3946
+ this.logger.warn({ err }, "Event log prune failed");
3947
+ }
3948
+ }
3949
+ openEventLog() {
3950
+ const dbPath = join(this.dataDir, "events.db");
3951
+ try {
3952
+ return new EventLog(dbPath);
3953
+ }
3954
+ catch (err) {
3955
+ this.logger.error({ err, dbPath }, "events.db unusable — moving it aside and starting a fresh one");
3956
+ const stamp = new Date().toISOString().replace(/[:.]/g, "-");
3957
+ for (const suffix of ["", "-wal", "-shm"]) {
3958
+ try {
3959
+ renameSync(`${dbPath}${suffix}`, `${dbPath}${suffix}.corrupt-${stamp}`);
3960
+ }
3961
+ catch { /* may not exist */ }
3962
+ }
3963
+ try {
3964
+ return new EventLog(dbPath);
3965
+ }
3966
+ catch (retryErr) {
3967
+ // History is worth losing; a fleet that won't start is not.
3968
+ this.logger.error({ err: retryErr, dbPath }, "Could not open a fresh events.db — continuing without event logging");
3969
+ return null;
3970
+ }
3971
+ }
3972
+ }
3973
+ /**
3974
+ * Report a fleet-level fault (not attributable to one instance) to the General
3975
+ * topic, so the operator learns about it without reading daemon.log.
3976
+ *
3977
+ * Throttled per distinct message: an unhandled rejection typically comes from a
3978
+ * loop (a poller, a repeating timer), and one channel message per occurrence
3979
+ * would bury the topic — which is worse than silence. First occurrence goes out
3980
+ * immediately, repeats are suppressed for THROTTLE_MS and then re-sent with a
3981
+ * count.
3982
+ *
3983
+ * The log line is written by the caller regardless: if every adapter is down,
3984
+ * the only notification path is the one that is broken.
3985
+ */
3986
+ notifyFleetError(text) {
3987
+ const now = Date.now();
3988
+ const key = text.slice(0, 200);
3989
+ const seen = this.fleetErrorNotices.get(key);
3990
+ if (seen && now - seen.at < FleetManager.FLEET_ERROR_THROTTLE_MS) {
3991
+ seen.suppressed++;
3992
+ return;
3993
+ }
3994
+ const suppressed = seen?.suppressed ?? 0;
3995
+ this.fleetErrorNotices.set(key, { at: now, suppressed: 0 });
3996
+ // Bound the map: it is keyed by message text, and a message with a varying
3997
+ // suffix (a path, an id) would otherwise grow it without limit.
3998
+ if (this.fleetErrorNotices.size > 100) {
3999
+ const oldest = this.fleetErrorNotices.keys().next().value;
4000
+ if (oldest !== undefined)
4001
+ this.fleetErrorNotices.delete(oldest);
4002
+ }
4003
+ const body = suppressed > 0
4004
+ ? `${text}\n(plus ${suppressed} more in the last ${Math.round(FleetManager.FLEET_ERROR_THROTTLE_MS / 60_000)}m)`
4005
+ : text;
4006
+ // Resolved from config, NOT findGeneralInstance(): that requires a live daemon,
4007
+ // and a fleet-level fault is exactly when the General may be down. The topic
4008
+ // itself still exists, and notifyInstanceTopic only needs adapter + group +
4009
+ // topic_id to post into it.
4010
+ const general = Object.entries(this.fleetConfig?.instances ?? {})
4011
+ .find(([, config]) => config.general_topic === true)?.[0];
4012
+ if (general) {
4013
+ this.notifyInstanceTopic(general, body);
4014
+ return;
4015
+ }
4016
+ // No General instance — fall back to the primary channel's group.
4017
+ const channelCfg = this.getChannelConfig();
4018
+ const groupId = channelCfg?.group_id;
4019
+ if (this.adapter && groupId) {
4020
+ this.adapter.sendText(String(groupId), body)
4021
+ .catch(err => this.logger.warn({ err }, "Failed to send fleet error notification"));
4022
+ return;
4023
+ }
4024
+ this.logger.warn({ text: body }, "Fleet error had no notification target (no General instance, no adapter)");
4025
+ }
4026
+ static FLEET_ERROR_THROTTLE_MS = 10 * 60_000;
4027
+ fleetErrorNotices = new Map();
3407
4028
  notifyInstanceTopic(instanceName, text, extraOpts) {
3408
4029
  const adapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
3409
4030
  if (!adapter)
@@ -3492,8 +4113,13 @@ export class FleetManager {
3492
4113
  const adapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
3493
4114
  if (!adapter)
3494
4115
  return;
4116
+ // Resolve the group through the world fallback (first world when unbound),
4117
+ // NOT through getChannelConfig(binding)?.group_id: on a fleet configured with
4118
+ // `channels:` worlds the primary `channel:` block is empty, so an instance
4119
+ // with no world binding yet (fresh restart, cross-instance delegation)
4120
+ // resolved group_id to undefined and the button silently never appeared.
3495
4121
  const adapterId = this.instanceWorldBinding.get(instanceName);
3496
- const groupId = this.getChannelConfig(adapterId)?.group_id;
4122
+ const groupId = this.getGroupIdForInstance(instanceName) || undefined;
3497
4123
  const topicId = this.fleetConfig?.instances[instanceName]?.topic_id;
3498
4124
  let chatId;
3499
4125
  let threadId;
@@ -3509,8 +4135,12 @@ export class FleetManager {
3509
4135
  if (!chatId && groupId)
3510
4136
  chatId = String(groupId);
3511
4137
  }
3512
- if (!chatId)
4138
+ if (!chatId) {
4139
+ // A button that cannot be addressed must say so — this exact silence is how
4140
+ // "the cancel button sometimes never appears" stayed unreported-in-logs.
4141
+ this.logger.warn({ instanceName, topicId, groupId }, "Cannot address cancel button (no chat id resolved)");
3513
4142
  return;
4143
+ }
3514
4144
  try {
3515
4145
  const sent = await adapter.notifyAlert(chatId, {
3516
4146
  type: "cancel",
@@ -3533,7 +4163,16 @@ export class FleetManager {
3533
4163
  threadId: sent.threadId ?? threadId,
3534
4164
  correlationId,
3535
4165
  retryCount: 0,
4166
+ // Elapsed time is measured from when this button was posted — i.e. from
4167
+ // when the work was handed over — not from the pane's working transition,
4168
+ // which resets if the CLI blips idle mid-turn.
4169
+ startedAt: Date.now(),
4170
+ // Matches the text notifyAlert just posted, so the first 60s tick does
4171
+ // not re-edit identical text — which put a "(edited)" mark on Discord
4172
+ // with nothing visibly changed.
4173
+ lastProgressText: "👀 處理中…",
3536
4174
  };
4175
+ this.startProgressTicker(entry);
3537
4176
  // Idle-check backstop: every 5min, if the instance is idle, retire the
3538
4177
  // button. Covers turns that end without hitting a clear trigger (reply /
3539
4178
  // cancel / correlation). Cleared in discardButton when the entry is removed.
@@ -3542,18 +4181,141 @@ export class FleetManager {
3542
4181
  clearInterval(entry.idleCheckTimer);
3543
4182
  return;
3544
4183
  }
3545
- if (this.getInstanceIdle(instanceName)) {
3546
- this.logger.info({ instanceName, messageId: entry.messageId }, "Cancel button idle backstop retiring");
4184
+ const reason = this.getInstanceIdle(instanceName) ? "idle"
4185
+ : this.stateReportDead(instanceName) ? "state reports stopped"
4186
+ : Date.now() - (entry.startedAt ?? 0) > CANCEL_BTN_MAX_LIFETIME_MS ? "24h ceiling"
4187
+ : null;
4188
+ if (reason) {
4189
+ this.logger.info({ instanceName, messageId: entry.messageId, reason }, "Cancel button backstop retiring");
3547
4190
  this.retireButton(entry);
4191
+ return;
3548
4192
  }
4193
+ // Still looks busy. The daemon only broadcasts on transitions, so ask for
4194
+ // a fresh snapshot — a live daemon's answer refreshes receivedAt and keeps
4195
+ // the staleness check honest; a dead one's silence is the evidence.
4196
+ this.instanceIpcClients.get(instanceName)?.send({
4197
+ type: "query_instance_state", requestId: `cancel-btn-${Date.now()}`,
4198
+ });
3549
4199
  }, CANCEL_BTN_IDLE_CHECK_INTERVAL_MS);
3550
4200
  this.cancelButtons.set(sent.messageId, entry);
4201
+ this.persistCancelButtons();
3551
4202
  this.logger.info({ instanceName, messageId: sent.messageId }, "Cancel button sent");
3552
4203
  }
3553
4204
  catch (e) {
3554
4205
  this.logger.warn({ err: e.message, instanceName }, "Failed to send cancel button");
3555
4206
  }
3556
4207
  }
4208
+ /**
4209
+ * The cancel button's text for a given elapsed time.
4210
+ *
4211
+ * Below the threshold it keeps the original wording, so a normal quick answer
4212
+ * looks exactly as it did before. Past it, the button doubles as the live
4213
+ * progress indicator (#409) — the channel showed nothing at all during long work,
4214
+ * and once the agent had replied once there was no sign it was still going.
4215
+ */
4216
+ static progressText(elapsedMs, activity) {
4217
+ if (elapsedMs < PROGRESS_MIN_ELAPSED_MS)
4218
+ return "👀 處理中…";
4219
+ const totalSeconds = Math.floor(elapsedMs / 1000);
4220
+ const minutes = Math.floor(totalSeconds / 60);
4221
+ const seconds = totalSeconds % 60;
4222
+ const elapsed = minutes >= 60
4223
+ ? `${Math.floor(minutes / 60)}h ${minutes % 60}m`
4224
+ : `${minutes}m ${String(seconds).padStart(2, "0")}s`;
4225
+ const detail = FleetManager.sanitizeActivity(activity);
4226
+ return detail
4227
+ ? `⏳ 處理中… (已進行 ${elapsed} · ${detail})`
4228
+ : `⏳ 處理中… (已進行 ${elapsed})`;
4229
+ }
4230
+ /**
4231
+ * Make a tool summary safe to paste into a channel message.
4232
+ *
4233
+ * The text is agent-controlled (it is built from tool inputs — file paths,
4234
+ * shell commands), so it gets flattened to one line, capped, and stripped of
4235
+ * the two Discord mass-mention triggers. Neither channel renders it with a
4236
+ * parse mode, so no markup escaping is needed beyond that.
4237
+ */
4238
+ static sanitizeActivity(activity) {
4239
+ if (!activity)
4240
+ return null;
4241
+ const flat = activity.replace(/\s+/g, " ").replace(/@(everyone|here)/g, "@​$1").trim();
4242
+ if (!flat)
4243
+ return null;
4244
+ return flat.length > PROGRESS_ACTIVITY_MAX_CHARS
4245
+ ? `${flat.slice(0, PROGRESS_ACTIVITY_MAX_CHARS - 1)}…`
4246
+ : flat;
4247
+ }
4248
+ /**
4249
+ * Remember what an instance is currently doing, for the progress line.
4250
+ *
4251
+ * Best-effort by design: only backends that expose a live activity feed report
4252
+ * anything, and the progress line simply omits the detail for the rest. It is
4253
+ * never used to decide anything — purely what the user is shown.
4254
+ */
4255
+ cacheInstanceActivity(name, activity) {
4256
+ if (activity)
4257
+ this.instanceActivity.set(name, activity);
4258
+ else
4259
+ this.instanceActivity.delete(name);
4260
+ }
4261
+ /**
4262
+ * Refresh the button's text in place while the instance keeps working.
4263
+ *
4264
+ * Uses `editAlert`, NOT `editMessage`: on Telegram the latter omits reply_markup,
4265
+ * and the Bot API treats that as "clear the keyboard" — so editing with it would
4266
+ * delete the very cancel button this is trying to keep alive.
4267
+ */
4268
+ startProgressTicker(entry) {
4269
+ entry.progressTimer = setInterval(() => {
4270
+ if (!this.cancelButtons.has(entry.messageId)) {
4271
+ clearInterval(entry.progressTimer);
4272
+ return;
4273
+ }
4274
+ const text = FleetManager.progressText(Date.now() - (entry.startedAt ?? Date.now()), this.instanceActivity.get(entry.instanceName));
4275
+ if (text === entry.lastProgressText)
4276
+ return; // nothing changed — skip the API call
4277
+ const adapter = this.getAdapterForInstance(entry.instanceName) ?? this.adapter;
4278
+ if (!adapter?.editAlert)
4279
+ return;
4280
+ entry.lastProgressText = text;
4281
+ adapter.editAlert(entry.chatId, entry.messageId, {
4282
+ type: "cancel",
4283
+ instanceName: entry.instanceName,
4284
+ message: text,
4285
+ choices: [{ id: `cancel:${entry.instanceName}`, label: t("cancel.button") }],
4286
+ }, entry.threadId ? { threadId: entry.threadId } : undefined)
4287
+ .catch(err => {
4288
+ // A failed progress edit must never escalate: the button still works and
4289
+ // the next tick retries. Common causes are a deleted message or a
4290
+ // rate limit.
4291
+ this.logger.debug({ err, instanceName: entry.instanceName }, "Progress edit failed");
4292
+ });
4293
+ }, PROGRESS_UPDATE_INTERVAL_MS);
4294
+ entry.progressTimer.unref?.();
4295
+ }
4296
+ /**
4297
+ * After a reply: give the instance REPLY_RETIRE_GRACE_MS to resume working; if
4298
+ * it has not, retire its button. Re-arming replaces the previous timer, so a
4299
+ * burst of replies ends with exactly one pending check.
4300
+ */
4301
+ armReplyGrace(instanceName) {
4302
+ for (const entry of this.cancelButtons.values()) {
4303
+ if (entry.instanceName !== instanceName)
4304
+ continue;
4305
+ if (entry.replyGraceTimer)
4306
+ clearTimeout(entry.replyGraceTimer);
4307
+ entry.replyGraceTimer = setTimeout(() => {
4308
+ entry.replyGraceTimer = undefined;
4309
+ if (!this.cancelButtons.has(entry.messageId))
4310
+ return;
4311
+ if (!this.getInstanceIdle(instanceName))
4312
+ return; // resumed — a long run keeps its button
4313
+ this.logger.info({ instanceName, messageId: entry.messageId }, "Cancel button retired — no work resumed after reply");
4314
+ this.retireButton(entry);
4315
+ }, REPLY_RETIRE_GRACE_MS);
4316
+ entry.replyGraceTimer.unref?.();
4317
+ }
4318
+ }
3557
4319
  /** Retire (delete) every cancel button belonging to an instance. */
3558
4320
  retireInstanceButtons(instanceName) {
3559
4321
  // Snapshot first — retireButton may delete entries from the map on success.
@@ -3585,7 +4347,70 @@ export class FleetManager {
3585
4347
  clearTimeout(entry.retryTimer);
3586
4348
  if (entry.idleCheckTimer)
3587
4349
  clearInterval(entry.idleCheckTimer);
4350
+ if (entry.progressTimer)
4351
+ clearInterval(entry.progressTimer);
4352
+ if (entry.replyGraceTimer)
4353
+ clearTimeout(entry.replyGraceTimer);
3588
4354
  this.cancelButtons.delete(entry.messageId);
4355
+ this.persistCancelButtons();
4356
+ }
4357
+ /**
4358
+ * Mirror the live buttons to disk. The map is memory-only, so before this a
4359
+ * fleet restart orphaned every button on screen: frozen "處理中…" text and a
4360
+ * click that did nothing, forever. The ledger is tiny (a handful of rows) and
4361
+ * written on every add/remove — no debounce needed at that rate.
4362
+ */
4363
+ persistCancelButtons() {
4364
+ try {
4365
+ const rows = [...this.cancelButtons.values()].map(e => ({
4366
+ instanceName: e.instanceName,
4367
+ adapterId: e.adapterId,
4368
+ chatId: e.chatId,
4369
+ messageId: e.messageId,
4370
+ threadId: e.threadId,
4371
+ }));
4372
+ writeFileSync(join(this.dataDir, CANCEL_BTN_LEDGER_FILE), JSON.stringify(rows));
4373
+ }
4374
+ catch (err) {
4375
+ this.logger.debug({ err }, "Cancel button ledger write failed");
4376
+ }
4377
+ }
4378
+ /**
4379
+ * Delete the previous process's buttons. Runs once adapters are up: nothing
4380
+ * from a previous fleet process can still be mid-turn from this process's
4381
+ * point of view, so every ledger row is an orphan by definition.
4382
+ */
4383
+ async sweepOrphanedCancelButtons() {
4384
+ const ledgerPath = join(this.dataDir, CANCEL_BTN_LEDGER_FILE);
4385
+ let rows;
4386
+ try {
4387
+ if (!existsSync(ledgerPath))
4388
+ return;
4389
+ rows = JSON.parse(readFileSync(ledgerPath, "utf-8"));
4390
+ }
4391
+ catch {
4392
+ try {
4393
+ unlinkSync(ledgerPath);
4394
+ }
4395
+ catch { /* corrupt ledger — drop it */ }
4396
+ return;
4397
+ }
4398
+ for (const row of rows) {
4399
+ const adapter = (row.adapterId ? this.worlds.get(row.adapterId)?.adapter : undefined)
4400
+ ?? this.getAdapterForInstance?.(row.instanceName) ?? this.adapter;
4401
+ if (!adapter?.deleteMessage)
4402
+ continue;
4403
+ try {
4404
+ await adapter.deleteMessage(row.chatId, row.messageId, row.threadId);
4405
+ this.logger.info({ instanceName: row.instanceName, messageId: row.messageId }, "Swept orphaned cancel button from previous run");
4406
+ }
4407
+ catch (err) {
4408
+ // Best effort: the message may already be gone, or too old to delete.
4409
+ this.logger.debug({ err, messageId: row.messageId }, "Orphaned cancel button sweep failed");
4410
+ }
4411
+ }
4412
+ // The current process owns the ledger from here on.
4413
+ this.persistCancelButtons();
3589
4414
  }
3590
4415
  /** Re-attempt a failed button delete up to CANCEL_BTN_MAX_RETRIES times. */
3591
4416
  scheduleButtonRetry(entry, err) {
@@ -4250,6 +5075,12 @@ When users create specialized instances, suggest these configurations:
4250
5075
  meta.image_path = saves[saves.length - 1][1].split(",")[0].trim();
4251
5076
  }
4252
5077
  }
5078
+ // Classic channels queue reactions like everyone else (#432 stored them, but
5079
+ // this path never attached them — reactions in a ClassicBot channel went into
5080
+ // the DB and were never seen again). Same contract as the topic paths:
5081
+ // consumed only after the delivery succeeded.
5082
+ const reactions = this.pendingReactionsMeta(instanceName);
5083
+ Object.assign(meta, reactions.meta);
4253
5084
  try {
4254
5085
  await this.deliverToInstance(instanceName, {
4255
5086
  type: "fleet_inbound",
@@ -4257,6 +5088,7 @@ When users create specialized instances, suggest these configurations:
4257
5088
  targetSession: instanceName,
4258
5089
  meta,
4259
5090
  });
5091
+ reactions.consume();
4260
5092
  }
4261
5093
  catch (err) {
4262
5094
  this.logger.warn({ err: err.message, instanceName }, "Classic wake/delivery failed");
@@ -4531,9 +5363,19 @@ When users create specialized instances, suggest these configurations:
4531
5363
  // DC path: respond immediately with progress text
4532
5364
  await pending.respond(progressText).catch(() => { });
4533
5365
  }
4534
- // Apply model in background — don't await here (keeps callback handler fast)
5366
+ // Apply model in background — don't await here (keeps callback handler fast).
5367
+ // Guarded: applyModel() restarts the instance, and an unguarded rejection here
5368
+ // meant a user picking from the /model menu could take the whole fleet down.
5369
+ // On failure the user gets told, rather than the click silently doing nothing.
4535
5370
  void (async () => {
4536
- const result = await this.applyModel(pending.instanceName, model);
5371
+ let result;
5372
+ try {
5373
+ result = await this.applyModel(pending.instanceName, model);
5374
+ }
5375
+ catch (err) {
5376
+ this.logger.error({ err, instance: pending.instanceName, model }, "Model switch failed");
5377
+ result = `Model switch to \`${model}\` failed: ${err instanceof Error ? err.message : String(err)}`;
5378
+ }
4537
5379
  if (pending.adapter && pending.adapterChatId) {
4538
5380
  if (progressMsgId) {
4539
5381
  pending.adapter.editMessage(pending.adapterChatId, progressMsgId, result, pending.adapterThreadId).catch(() => {
@@ -4771,7 +5613,7 @@ When users create specialized instances, suggest these configurations:
4771
5613
  ...(preTaskCommand ? { pre_task_command: preTaskCommand } : {}),
4772
5614
  };
4773
5615
  const topicMode = this.fleetConfig?.channel?.mode === "topic";
4774
- await this.startInstance(instanceName, config, topicMode);
5616
+ await this.startInstance(instanceName, config, topicMode, "classic");
4775
5617
  }
4776
5618
  /** Handle /start slash command — register classic channel */
4777
5619
  async handleClassicStart(channelId, channelName, userId, guildId, adapterId, backend) {
@@ -4813,11 +5655,24 @@ When users create specialized instances, suggest these configurations:
4813
5655
  this.logger.info({ channelId, adapterId, instanceName: ch.instanceName }, "Classic channel stopped");
4814
5656
  return t("classic.stopped");
4815
5657
  }
4816
- async stopAll() {
5658
+ /**
5659
+ * Idempotent while in flight: SIGINT and SIGTERM share one handler and the
5660
+ * uncaughtException path calls this too, so overlapping runs were possible —
5661
+ * each snapshotting the daemon map and calling stop() on the same daemons
5662
+ * concurrently. Deliberately NOT `async`, so callers receive the same promise
5663
+ * object rather than a fresh wrapper around it. The latch clears when the run
5664
+ * settles, so a later genuine stop (after a restart) still does the work.
5665
+ */
5666
+ stopAll() {
5667
+ this.stopAllInFlight ??= this.doStopAll().finally(() => { this.stopAllInFlight = null; });
5668
+ return this.stopAllInFlight;
5669
+ }
5670
+ stopAllInFlight = null;
5671
+ async doStopAll() {
4817
5672
  this.startupComplete = false;
4818
5673
  this.reloadPending = false;
4819
5674
  this.ipcStoppingInstances.add("__fleet_stopping__");
4820
- sdNotify("STOPPING=1");
5675
+ sdNotifyBlocking("STOPPING=1");
4821
5676
  if (this.watchdogTimer) {
4822
5677
  clearInterval(this.watchdogTimer);
4823
5678
  this.watchdogTimer = null;
@@ -4837,6 +5692,26 @@ When users create specialized instances, suggest these configurations:
4837
5692
  clearInterval(this.updateCheckTimer);
4838
5693
  this.updateCheckTimer = null;
4839
5694
  }
5695
+ if (this.eventLogPruneTimer) {
5696
+ clearInterval(this.eventLogPruneTimer);
5697
+ this.eventLogPruneTimer = null;
5698
+ }
5699
+ if (this.logRotateTimer) {
5700
+ clearInterval(this.logRotateTimer);
5701
+ this.logRotateTimer = null;
5702
+ }
5703
+ // Cancel-button timers were never cleared here. The idle-check interval is not
5704
+ // unref'd, so it held the event loop open past shutdown and kept retrying
5705
+ // deletes against an adapter that was already gone.
5706
+ for (const entry of [...this.cancelButtons.values()]) {
5707
+ if (entry.retryTimer)
5708
+ clearTimeout(entry.retryTimer);
5709
+ if (entry.idleCheckTimer)
5710
+ clearInterval(entry.idleCheckTimer);
5711
+ if (entry.progressTimer)
5712
+ clearInterval(entry.progressTimer);
5713
+ }
5714
+ this.cancelButtons.clear();
4840
5715
  if (this.topicCleanupTimer) {
4841
5716
  clearInterval(this.topicCleanupTimer);
4842
5717
  this.topicCleanupTimer = null;
@@ -5055,6 +5930,15 @@ When users create specialized instances, suggest these configurations:
5055
5930
  removedRatio,
5056
5931
  validationErrors: validation.errors,
5057
5932
  }, "Refusing unsafe fleet config reload; running configuration was kept");
5933
+ // Tell the operator. A silently ignored config edit is the most confusing
5934
+ // possible outcome: they change fleet.yaml, send SIGHUP, and nothing happens
5935
+ // with no explanation anywhere they are looking.
5936
+ const why = !validation.valid
5937
+ ? `validation failed:\n${validation.errors.map(e => `• ${e.path}: ${e.message}`).join("\n")}`
5938
+ : unsafeEmpty
5939
+ ? `it removed every instance (${oldCount} → 0)`
5940
+ : `it removed more than half the instances (${oldCount} → ${newCount})`;
5941
+ this.notifyFleetError(`⚠️ fleet.yaml reload REJECTED — ${why}\nThe previous configuration is still running.`);
5058
5942
  return;
5059
5943
  }
5060
5944
  this.routing.rebuild(this.fleetConfig);
@@ -5254,10 +6138,20 @@ When users create specialized instances, suggest these configurations:
5254
6138
  // ── Update check ────────────────────────────────────────────────────
5255
6139
  async checkForUpdates() {
5256
6140
  try {
5257
- const { execSync } = await import("node:child_process");
6141
+ // Both npm lookups are async: as execSync they froze the fleet event loop for
6142
+ // up to 15s each, and on a beta build BOTH ran — 30s with no WATCHDOG ping,
6143
+ // past WatchdogSec's half-interval and enough for systemd to SIGABRT the fleet
6144
+ // for a background version check.
6145
+ const { execFile } = await import("node:child_process");
6146
+ const { promisify } = await import("node:util");
6147
+ const execFileP = promisify(execFile);
6148
+ const npmVersion = async (spec) => {
6149
+ const { stdout } = await execFileP("npm", ["view", spec, "version"], { timeout: 15_000 });
6150
+ return stdout.toString().trim();
6151
+ };
5258
6152
  const pkgPath = join(dirname(fileURLToPath(import.meta.url)), "..", "package.json");
5259
6153
  const currentVersion = JSON.parse(readFileSync(pkgPath, "utf-8")).version ?? "0.0.0";
5260
- const latest = execSync("npm view @songsid/agend version", { stdio: "pipe", timeout: 15_000 }).toString().trim();
6154
+ const latest = await npmVersion("@songsid/agend");
5261
6155
  let target = latest;
5262
6156
  if (currentVersion.includes("-beta")) {
5263
6157
  // Beta users track the @beta channel (never fall back to @latest, which is
@@ -5265,7 +6159,7 @@ When users create specialized instances, suggest these configurations:
5265
6159
  // of beta/latest is the newest.
5266
6160
  let beta = "";
5267
6161
  try {
5268
- beta = execSync("npm view @songsid/agend@beta version", { stdio: "pipe", timeout: 15_000 }).toString().trim();
6162
+ beta = await npmVersion("@songsid/agend@beta");
5269
6163
  }
5270
6164
  catch { /* no beta tag */ }
5271
6165
  target = beta || latest;
@@ -5372,6 +6266,10 @@ When users create specialized instances, suggest these configurations:
5372
6266
  // /view routes accept the read-only view.token (or web.token) and do
5373
6267
  // their own per-method auth in view-api.ts — skip the web-token gate.
5374
6268
  }
6269
+ else if (isUsagePath(new URL(req.url ?? "/", `http://localhost:${port}`).pathname)) {
6270
+ // /api/ai-usage is read-only GET data for the /view Usage panel — open
6271
+ // like the other /view data routes (usage-api.ts rejects non-GET).
6272
+ }
5375
6273
  else {
5376
6274
  // All other endpoints require a valid token (query ?token= or X-Agend-Token header).
5377
6275
  // /ui/* will also re-check in web-api.ts, which is harmless.
@@ -5386,34 +6284,34 @@ When users create specialized instances, suggest these configurations:
5386
6284
  }
5387
6285
  }
5388
6286
  if (req.method === "GET" && req.url === "/health") {
5389
- const instanceCount = this.fleetConfig?.instances
5390
- ? Object.keys(this.fleetConfig.instances).length
5391
- : 0;
5392
- res.writeHead(200);
5393
- res.end(JSON.stringify({
5394
- status: "ok",
5395
- instances: instanceCount,
5396
- uptime: Math.floor((Date.now() - this.startedAt) / 1000),
5397
- }));
6287
+ const health = this.getFleetHealth();
6288
+ // 503 when the fleet cannot do its job, so an external monitor sees it.
6289
+ // This used to always answer 200 "ok" with a count of CONFIGURED instances,
6290
+ // so every agent could be dead and every adapter down and it still looked
6291
+ // green.
6292
+ res.writeHead(health.status === "ok" ? 200 : 503);
6293
+ res.end(JSON.stringify(health));
5398
6294
  return;
5399
6295
  }
5400
6296
  if (req.method === "GET" && req.url === "/status") {
5401
6297
  const instances = Object.keys(this.fleetConfig?.instances ?? {}).map(name => {
5402
6298
  const statusFile = join(this.getInstanceDir(name), "statusline.json");
5403
- let context_pct = 0;
5404
6299
  let cost = 0;
5405
6300
  try {
5406
6301
  const data = JSON.parse(readFileSync(statusFile, "utf-8"));
5407
- context_pct = data.context_window?.used_percentage ?? 0;
5408
6302
  cost = data.cost?.total_cost_usd ?? 0;
5409
6303
  }
5410
6304
  catch (err) {
5411
6305
  this.logger.debug({ err, name }, "statusline.json read failed (/status)");
5412
6306
  }
6307
+ const backend = this.fleetConfig?.instances[name]?.backend
6308
+ ?? this.fleetConfig?.defaults?.backend
6309
+ ?? "claude-code";
6310
+ const { context } = resolveInstanceContext(this.dataDir, name, backend);
5413
6311
  return {
5414
6312
  name,
5415
6313
  status: this.getInstanceStatus(name),
5416
- context_pct,
6314
+ context_pct: context ?? 0,
5417
6315
  cost,
5418
6316
  };
5419
6317
  });
@@ -5528,7 +6426,10 @@ When users create specialized instances, suggest these configurations:
5528
6426
  res.writeHead(500);
5529
6427
  res.end(JSON.stringify({ error: `Start failed: ${err.message}` }));
5530
6428
  }
5531
- })();
6429
+ // The inner catch can itself throw (writeHead after a successful
6430
+ // writeHead is ERR_HTTP_HEADERS_SENT), and that rejection escapes the
6431
+ // IIFE. Same for the two handlers below.
6432
+ })().catch(err => this.logger.error({ err, name }, "HTTP start handler failed"));
5532
6433
  return;
5533
6434
  }
5534
6435
  // Instance restart (immediate, no idle wait)
@@ -5549,7 +6450,7 @@ When users create specialized instances, suggest these configurations:
5549
6450
  res.writeHead(status);
5550
6451
  res.end(JSON.stringify({ error: `Restart failed: ${err.message}` }));
5551
6452
  }
5552
- })();
6453
+ })().catch(err => this.logger.error({ err, name }, "HTTP restart handler failed"));
5553
6454
  return;
5554
6455
  }
5555
6456
  if (req.method === "POST" && req.url?.startsWith("/stop/")) {
@@ -5572,7 +6473,7 @@ When users create specialized instances, suggest these configurations:
5572
6473
  res.writeHead(500);
5573
6474
  res.end(JSON.stringify({ error: `Stop failed: ${err.message}` }));
5574
6475
  }
5575
- })();
6476
+ })().catch(err => this.logger.error({ err, name }, "HTTP stop handler failed"));
5576
6477
  return;
5577
6478
  }
5578
6479
  // ── Agent CLI endpoint ─────
@@ -5584,6 +6485,8 @@ When users create specialized instances, suggest these configurations:
5584
6485
  const url = new URL(req.url ?? "/", `http://localhost:${port}`);
5585
6486
  if (handleViewRequest(req, res, url, this))
5586
6487
  return;
6488
+ if (handleUsageRequest(req, res, url, this))
6489
+ return;
5587
6490
  if (handleSettingsRequest(req, res, url, this))
5588
6491
  return;
5589
6492
  if (handleWebRequest(req, res, url, this))
@@ -5630,20 +6533,35 @@ When users create specialized instances, suggest these configurations:
5630
6533
  this.logger.info({ url: `http://localhost:${port}/view?token=${this.viewToken}` }, "Web View available");
5631
6534
  }
5632
6535
  getUiStatus() {
5633
- const instances = Object.keys(this.fleetConfig?.instances ?? {}).map(name => {
6536
+ const fleetNames = Object.keys(this.fleetConfig?.instances ?? {});
6537
+ // Classic rooms live only in classicBot.yaml — /api/profiles merges them into
6538
+ // the View roster, but previously getUiStatus skipped them so context_pct was
6539
+ // always 0 (live map miss → l?.context_pct ?? 0).
6540
+ const classicOnly = (this.classicChannels?.getAll() ?? [])
6541
+ .map(ch => ch.instanceName)
6542
+ .filter(name => !fleetNames.includes(name));
6543
+ const names = [...fleetNames, ...classicOnly];
6544
+ const instances = names.map(name => {
5634
6545
  const statusFile = join(this.getInstanceDir(name), "statusline.json");
5635
- let context_pct = 0;
5636
6546
  let cost = 0;
5637
6547
  let model = "";
5638
6548
  try {
5639
6549
  const data = JSON.parse(readFileSync(statusFile, "utf-8"));
5640
- context_pct = data.context_window?.used_percentage ?? 0;
5641
6550
  cost = data.cost?.total_cost_usd ?? 0;
5642
6551
  model = data.model?.display_name ?? "";
5643
6552
  }
5644
6553
  catch (err) {
5645
6554
  this.logger.debug({ err, name }, "statusline.json read failed (getUiStatus)");
5646
6555
  }
6556
+ // Align with /ctx: statusline for claude-code, pane scrape for kiro/grok/codex.
6557
+ const classic = classicOnly.includes(name);
6558
+ const backend = classic
6559
+ ? this.classicChannels.getBackendByInstance(name, this.fleetConfig?.defaults?.backend)
6560
+ : (this.fleetConfig?.instances[name]?.backend
6561
+ ?? this.fleetConfig?.defaults?.backend
6562
+ ?? "claude-code");
6563
+ const { context } = resolveInstanceContext(this.dataDir, name, backend);
6564
+ const context_pct = context ?? 0;
5647
6565
  return { name, status: this.getInstanceStatus(name), context_pct, cost, model };
5648
6566
  });
5649
6567
  return {