@songsid/agend 2.1.2-beta.3 → 2.1.2-beta.30

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/dist/access-path.js +15 -6
  2. package/dist/access-path.js.map +1 -1
  3. package/dist/agent-cli-instructions.md +1 -0
  4. package/dist/agent-cli.js +14 -0
  5. package/dist/agent-cli.js.map +1 -1
  6. package/dist/agent-endpoint.d.ts +2 -0
  7. package/dist/agent-endpoint.js +25 -1
  8. package/dist/agent-endpoint.js.map +1 -1
  9. package/dist/backend/antigravity.d.ts +20 -0
  10. package/dist/backend/antigravity.js +37 -3
  11. package/dist/backend/antigravity.js.map +1 -1
  12. package/dist/backend/claude-code.d.ts +59 -0
  13. package/dist/backend/claude-code.js +67 -1
  14. package/dist/backend/claude-code.js.map +1 -1
  15. package/dist/backend/codex.d.ts +3 -0
  16. package/dist/backend/codex.js +30 -0
  17. package/dist/backend/codex.js.map +1 -1
  18. package/dist/backend/grok.d.ts +25 -0
  19. package/dist/backend/grok.js +51 -0
  20. package/dist/backend/grok.js.map +1 -1
  21. package/dist/backend/kiro.d.ts +22 -0
  22. package/dist/backend/kiro.js +85 -3
  23. package/dist/backend/kiro.js.map +1 -1
  24. package/dist/backend/types.d.ts +58 -0
  25. package/dist/backend/types.js +13 -1
  26. package/dist/backend/types.js.map +1 -1
  27. package/dist/channel/adapters/discord.d.ts +15 -0
  28. package/dist/channel/adapters/discord.js +100 -2
  29. package/dist/channel/adapters/discord.js.map +1 -1
  30. package/dist/channel/adapters/telegram.d.ts +21 -0
  31. package/dist/channel/adapters/telegram.js +86 -0
  32. package/dist/channel/adapters/telegram.js.map +1 -1
  33. package/dist/channel/ipc-bridge.d.ts +9 -1
  34. package/dist/channel/ipc-bridge.js +12 -3
  35. package/dist/channel/ipc-bridge.js.map +1 -1
  36. package/dist/channel/ipc-timeouts.d.ts +46 -0
  37. package/dist/channel/ipc-timeouts.js +65 -0
  38. package/dist/channel/ipc-timeouts.js.map +1 -0
  39. package/dist/channel/mcp-server.js +30 -14
  40. package/dist/channel/mcp-server.js.map +1 -1
  41. package/dist/channel/mcp-tools.js +30 -1
  42. package/dist/channel/mcp-tools.js.map +1 -1
  43. package/dist/channel/reconnect-backoff.d.ts +17 -0
  44. package/dist/channel/reconnect-backoff.js +21 -0
  45. package/dist/channel/reconnect-backoff.js.map +1 -0
  46. package/dist/channel/types.d.ts +29 -0
  47. package/dist/classic-channel-manager.js +1 -5
  48. package/dist/classic-channel-manager.js.map +1 -1
  49. package/dist/cli.js +155 -63
  50. package/dist/cli.js.map +1 -1
  51. package/dist/completion.d.ts +27 -0
  52. package/dist/completion.js +121 -0
  53. package/dist/completion.js.map +1 -0
  54. package/dist/config-validator.js +21 -0
  55. package/dist/config-validator.js.map +1 -1
  56. package/dist/config.js +1 -0
  57. package/dist/config.js.map +1 -1
  58. package/dist/cost-guard.d.ts +3 -1
  59. package/dist/cost-guard.js +3 -1
  60. package/dist/cost-guard.js.map +1 -1
  61. package/dist/daemon.d.ts +240 -14
  62. package/dist/daemon.js +956 -308
  63. package/dist/daemon.js.map +1 -1
  64. package/dist/event-log.d.ts +31 -0
  65. package/dist/event-log.js +96 -0
  66. package/dist/event-log.js.map +1 -1
  67. package/dist/fleet-context.d.ts +9 -0
  68. package/dist/fleet-manager.d.ts +311 -2
  69. package/dist/fleet-manager.js +1369 -108
  70. package/dist/fleet-manager.js.map +1 -1
  71. package/dist/general-knowledge/skills/cross-instance-messaging/SKILL.md +22 -0
  72. package/dist/general-knowledge/skills/model-discovery/SKILL.md +14 -23
  73. package/dist/general-knowledge/skills/session-management/SKILL.md +15 -34
  74. package/dist/hang-detector.d.ts +19 -13
  75. package/dist/hang-detector.js +19 -49
  76. package/dist/hang-detector.js.map +1 -1
  77. package/dist/instance-lifecycle.d.ts +43 -1
  78. package/dist/instance-lifecycle.js +155 -43
  79. package/dist/instance-lifecycle.js.map +1 -1
  80. package/dist/instructions.d.ts +5 -0
  81. package/dist/instructions.js +9 -11
  82. package/dist/instructions.js.map +1 -1
  83. package/dist/locale.js +5 -1
  84. package/dist/locale.js.map +1 -1
  85. package/dist/logger.js +14 -0
  86. package/dist/logger.js.map +1 -1
  87. package/dist/mcp-liveness.d.ts +21 -0
  88. package/dist/mcp-liveness.js +27 -0
  89. package/dist/mcp-liveness.js.map +1 -0
  90. package/dist/outbound-handlers.d.ts +16 -0
  91. package/dist/outbound-handlers.js +151 -27
  92. package/dist/outbound-handlers.js.map +1 -1
  93. package/dist/outbound-schemas.d.ts +11 -3
  94. package/dist/outbound-schemas.js +16 -2
  95. package/dist/outbound-schemas.js.map +1 -1
  96. package/dist/pane-write-lock.d.ts +48 -0
  97. package/dist/pane-write-lock.js +73 -0
  98. package/dist/pane-write-lock.js.map +1 -0
  99. package/dist/process-memory.d.ts +31 -0
  100. package/dist/process-memory.js +79 -0
  101. package/dist/process-memory.js.map +1 -0
  102. package/dist/quickstart.js +17 -16
  103. package/dist/quickstart.js.map +1 -1
  104. package/dist/reply-dedup.d.ts +41 -0
  105. package/dist/reply-dedup.js +0 -0
  106. package/dist/reply-dedup.js.map +1 -0
  107. package/dist/scheduler/db.js +3 -0
  108. package/dist/scheduler/db.js.map +1 -1
  109. package/dist/sd-notify.d.ts +27 -0
  110. package/dist/sd-notify.js +33 -1
  111. package/dist/sd-notify.js.map +1 -1
  112. package/dist/secret-file.d.ts +33 -0
  113. package/dist/secret-file.js +36 -0
  114. package/dist/secret-file.js.map +1 -0
  115. package/dist/setup-wizard.js +9 -7
  116. package/dist/setup-wizard.js.map +1 -1
  117. package/dist/tmux-control.d.ts +58 -5
  118. package/dist/tmux-control.js +102 -14
  119. package/dist/tmux-control.js.map +1 -1
  120. package/dist/tmux-manager.d.ts +31 -1
  121. package/dist/tmux-manager.js +40 -11
  122. package/dist/tmux-manager.js.map +1 -1
  123. package/dist/topic-commands.d.ts +50 -10
  124. package/dist/topic-commands.js +269 -106
  125. package/dist/topic-commands.js.map +1 -1
  126. package/dist/tui-glyphs.d.ts +25 -0
  127. package/dist/tui-glyphs.js +26 -0
  128. package/dist/tui-glyphs.js.map +1 -0
  129. package/dist/types.d.ts +12 -2
  130. package/dist/ui/view.html +123 -2
  131. package/dist/update-marker.d.ts +15 -0
  132. package/dist/update-marker.js +69 -0
  133. package/dist/update-marker.js.map +1 -0
  134. package/dist/usage/format-rich.d.ts +11 -0
  135. package/dist/usage/format-rich.js +121 -0
  136. package/dist/usage/format-rich.js.map +1 -0
  137. package/dist/usage/providers.d.ts +130 -0
  138. package/dist/usage/providers.js +1030 -0
  139. package/dist/usage/providers.js.map +1 -0
  140. package/dist/usage/statusline-usage.d.ts +20 -0
  141. package/dist/usage/statusline-usage.js +120 -0
  142. package/dist/usage/statusline-usage.js.map +1 -0
  143. package/dist/usage/usage-api.d.ts +50 -0
  144. package/dist/usage/usage-api.js +167 -0
  145. package/dist/usage/usage-api.js.map +1 -0
  146. package/dist/view-api.d.ts +3 -0
  147. package/dist/view-api.js +11 -2
  148. package/dist/view-api.js.map +1 -1
  149. package/dist/web-api.js +5 -2
  150. package/dist/web-api.js.map +1 -1
  151. package/package.json +4 -1
  152. package/dist/channel/tool-tracker.d.ts +0 -13
  153. package/dist/channel/tool-tracker.js +0 -58
  154. package/dist/channel/tool-tracker.js.map +0 -1
  155. package/dist/daemon-entry.d.ts +0 -1
  156. package/dist/daemon-entry.js +0 -30
  157. package/dist/daemon-entry.js.map +0 -1
package/dist/daemon.js CHANGED
@@ -1,18 +1,23 @@
1
1
  import { join, dirname, basename, resolve } from "node:path";
2
- import { mkdirSync, writeFileSync, readFileSync, existsSync, unlinkSync, rmSync, appendFileSync, statSync, chmodSync, renameSync } from "node:fs";
2
+ import { mkdirSync, writeFileSync, readFileSync, existsSync, unlinkSync, rmSync, appendFileSync, statSync, renameSync } from "node:fs";
3
3
  import { fileURLToPath } from "node:url";
4
4
  import { createHash, randomBytes } from "node:crypto";
5
5
  import { EventEmitter } from "node:events";
6
+ import { rotateLogIfNeeded } from "./logger.js";
7
+ import { mcpServerState } from "./mcp-liveness.js";
6
8
  import { clearPausedMarker, writePausedMarker } from "./pause-marker.js";
7
9
  import { TmuxManager, resolveTmuxLogicalSize } from "./tmux-manager.js";
8
10
  import { TranscriptMonitor } from "./transcript-monitor.js";
9
11
  import { ContextGuardian } from "./context-guardian.js";
10
12
  import { IpcServer } from "./channel/ipc-bridge.js";
13
+ import { daemonBudgetMs } from "./channel/ipc-timeouts.js";
11
14
  import { MessageBus } from "./channel/message-bus.js";
12
15
  import { shellQuote } from "./backend/types.js";
13
16
  import { getTmuxSession } from "./config.js";
14
17
  import { routeToolCall } from "./channel/tool-router.js";
15
18
  import { HangDetector } from "./hang-detector.js";
19
+ import { writeSecretFile } from "./secret-file.js";
20
+ import { PaneWriteLock } from "./pane-write-lock.js";
16
21
  import { buildFleetInstructions } from "./instructions.js";
17
22
  const __filename = fileURLToPath(import.meta.url);
18
23
  const __dirname = dirname(__filename);
@@ -37,9 +42,54 @@ export function buildInstructionReloadNotice(binaryName, instanceName, instanceD
37
42
  export const DEFAULT_STUCK_TIMEOUT_MS = 10 * 60_000;
38
43
  export const DEFAULT_STATE_IDLE_DEBOUNCE_MS = 2_000;
39
44
  export const DEFAULT_STATE_SAFETY_SWEEP_MS = 60_000;
40
- /** @deprecated State detection is event-driven; this now aliases the safety sweep. */
41
- export const DEFAULT_STATE_POLL_INTERVAL_MS = DEFAULT_STATE_SAFETY_SWEEP_MS;
42
45
  const LAST_INBOUND_FILE = "last-inbound-at";
46
+ /** Minimum gap between "health check is failing" notifications for one instance. */
47
+ const HEALTH_ERROR_NOTIFY_INTERVAL_MS = 10 * 60_000;
48
+ /**
49
+ * Whether two working directories belong to the same project, so a fleet-scoped
50
+ * decision recorded in one reaches the other. Covers the worktree/checkout
51
+ * layout AgEnD fleets actually use — `AgEnD`, `AgEnD-dev1`, `AgEnD-dev2`,
52
+ * `AgEnD-reviewer`, `AgEnD-main` are one project; `DouPo_Server` is not.
53
+ * Nesting counts too (a subdirectory of a project belongs to it).
54
+ */
55
+ export function sameProjectFamily(a, b) {
56
+ const norm = (p) => p.replace(/\/+$/, "");
57
+ const x = norm(a), y = norm(b);
58
+ if (x === y)
59
+ return true;
60
+ if (x.startsWith(y + "/") || y.startsWith(x + "/"))
61
+ return true;
62
+ // Strip a trailing worktree/role suffix: "AgEnD-dev2" → "agend".
63
+ const family = (p) => {
64
+ const base = p.slice(p.lastIndexOf("/") + 1).toLowerCase();
65
+ return base.replace(/[-_](dev\d*|main|reviewer|tester|leader|sol|codex|worktree|wt\d*)$/i, "");
66
+ };
67
+ const fx = family(x), fy = family(y);
68
+ return fx.length > 0 && fx === fy;
69
+ }
70
+ /**
71
+ * Pick the decisions worth injecting into one instance's system prompt.
72
+ *
73
+ * `scope: "fleet"` previously bypassed the project check outright, so every
74
+ * instance carried every other project's playbook — measured at 8 of 14 here,
75
+ * resent on every API call. A fleet decision must now also be relevant: global
76
+ * (no project_root), the same project, or the same project family.
77
+ *
78
+ * `isDispatcher` (a general) opts out of that narrowing on purpose: it routes
79
+ * work across all projects, so cross-project rules ("X 文件操作由 Y 負責") are
80
+ * precisely what it must know. Narrowing a general would cause misrouting.
81
+ */
82
+ export function selectRelevantDecisions(all, workDir, isDispatcher = false) {
83
+ return all.filter(d => {
84
+ if (d.project_root === workDir)
85
+ return true; // own project, any scope
86
+ if (d.scope !== "fleet")
87
+ return false; // project-scoped, elsewhere
88
+ if (!d.project_root)
89
+ return true; // truly global
90
+ return isDispatcher || sameProjectFamily(d.project_root, workDir);
91
+ });
92
+ }
43
93
  /** Read the last real channel inbound timestamp persisted across daemon restarts. */
44
94
  export function readLastInboundAt(instanceDir, now = Date.now()) {
45
95
  try {
@@ -58,6 +108,40 @@ export function writeLastInboundAt(instanceDir, timestamp) {
58
108
  writeFileSync(temp, String(timestamp));
59
109
  renameSync(temp, target);
60
110
  }
111
+ /**
112
+ * Render the handoff/routing metadata that rides along with an inbound message
113
+ * into the block actually pasted into the agent's pane.
114
+ *
115
+ * These fields were all populated by the sender (outbound-handlers builds them
116
+ * into `ipcMeta`) and delivered over IPC, but never rendered — so the receiving
117
+ * agent could not see them, and several documented flows could not work:
118
+ *
119
+ * - `report_result` requires a `correlation_id` the agent had no way to know, so
120
+ * the "correlation_id not recognized" warning fired on essentially every call.
121
+ * - Delegation cancel buttons are retired by correlation id, so they never retired.
122
+ * - `react`, `edit_message` and `reply.reply_to` need `message_id`, which the tool
123
+ * descriptions tell the agent to take "from the inbound block".
124
+ * - `download_attachment` needs `attachment_file_id`.
125
+ * - `requires_reply` was invisible, so a delegated task looked like an FYI.
126
+ *
127
+ * Only non-empty fields are emitted, so a plain user message gains at most a
128
+ * message_id line.
129
+ */
130
+ export function renderHandoffMetadata(meta) {
131
+ const rows = [];
132
+ const add = (label, value) => {
133
+ if (value && value.trim())
134
+ rows.push(`${label}: ${value.trim()}`);
135
+ };
136
+ add("message_id", meta.message_id);
137
+ add("correlation_id", meta.correlation_id);
138
+ add("request_kind", meta.request_kind);
139
+ add("task_summary", meta.task_summary);
140
+ add("working_directory", meta.working_directory);
141
+ add("branch", meta.branch);
142
+ add("attachment_file_id", meta.attachment_file_id);
143
+ return rows.length ? `\n(${rows.join(" | ")})` : "";
144
+ }
61
145
  /** Headless inactivity timer used by the daemon and unit tests. */
62
146
  export class AutoPauseController {
63
147
  thresholdMs;
@@ -102,37 +186,78 @@ export class AutoPauseController {
102
186
  * Pane motion wins over a ready match because several backends keep their ready
103
187
  * marker in a persistent header/footer while generating. A stable ready pane is
104
188
  * idle; changing content is working; stable non-ready content eventually sticks.
189
+ *
190
+ * ## Why an optional busy pattern exists
191
+ *
192
+ * "Motion wins" degrades badly when a backend's ready marker is *always* on
193
+ * screen. `stuck` requires `!ready`, so for such a backend the state can only ever
194
+ * be idle or working — the stuck edge, and with it `handleStuckTransition` and the
195
+ * hang notification, is unreachable. A frozen CLI is then reported as idle, which
196
+ * also clears pending work: the message the user sent is quietly booked as done.
197
+ *
198
+ * claude-code was exactly this case (see ClaudeCodeBackend.getReadyPattern). A
199
+ * backend that can point at a marker meaning "generating right now" supplies
200
+ * `getBusyPattern()`, and that match vetoes ready regardless of what the ready
201
+ * pattern says.
105
202
  */
106
203
  export class PaneStateMachine {
107
204
  stuckTimeoutMs;
108
205
  readyPattern;
206
+ busyPattern;
109
207
  lastPaneHash = null;
110
208
  lastPaneChangeAt;
111
209
  lastObservedAt;
112
210
  stateChangedAt;
113
211
  currentState = "idle";
114
- constructor(readyPattern, stuckTimeoutMs = DEFAULT_STUCK_TIMEOUT_MS, now = Date.now()) {
212
+ constructor(readyPattern, stuckTimeoutMs = DEFAULT_STUCK_TIMEOUT_MS, now = Date.now(), busyPattern) {
115
213
  this.stuckTimeoutMs = stuckTimeoutMs;
116
214
  // Stateful g/y regexes mutate lastIndex and can alternate true/false across
117
215
  // polls. State detection must be deterministic for identical pane content.
118
216
  this.readyPattern = new RegExp(readyPattern.source, readyPattern.flags.replace(/[gy]/g, ""));
217
+ this.busyPattern = busyPattern
218
+ ? new RegExp(busyPattern.source, busyPattern.flags.replace(/[gy]/g, ""))
219
+ : null;
119
220
  this.lastPaneChangeAt = now;
120
221
  this.lastObservedAt = now;
121
222
  this.stateChangedAt = now;
122
223
  }
123
- observe(pane, now = Date.now()) {
224
+ /** Ready, unless the backend can positively see that it is still generating. */
225
+ isReady(pane) {
226
+ if (this.busyPattern?.test(pane))
227
+ return false;
228
+ return this.readyPattern.test(pane);
229
+ }
230
+ /**
231
+ * @param now when this observation is being evaluated — drives the idle and
232
+ * stuck decisions.
233
+ * @param opts.changeAt when the content is believed to have changed, if that
234
+ * is earlier than `now` (the tmux output timestamp). Only the elapsed-time
235
+ * clock uses it; defaults to `now`.
236
+ * @param opts.settled the caller knows no output has arrived for the debounce
237
+ * window, so "the content differs from the last capture" says nothing about
238
+ * whether the CLI is still generating — decide on the patterns instead.
239
+ *
240
+ * Captures are event-driven, not periodic: the one taken 2s after output
241
+ * stops is compared against a capture that may be a minute old, so it
242
+ * *always* differs. Without this flag that difference reads as "still
243
+ * working" and a finished turn would not be seen as idle until the next 60s
244
+ * safety sweep. Output timestamps are the honest liveness signal here, and
245
+ * the caller has them.
246
+ */
247
+ observe(pane, now = Date.now(), opts = {}) {
248
+ const { settled = false, changeAt = now } = opts;
124
249
  const paneHash = createHash("sha256").update(pane).digest("hex");
125
250
  const firstObservation = this.lastPaneHash === null;
126
251
  const paneChanged = this.lastPaneHash !== paneHash;
127
252
  if (paneChanged) {
128
253
  this.lastPaneHash = paneHash;
129
- this.lastPaneChangeAt = now;
254
+ this.lastPaneChangeAt = changeAt;
130
255
  }
131
256
  this.lastObservedAt = now;
132
- const ready = this.readyPattern.test(pane);
257
+ const ready = this.isReady(pane);
133
258
  const nextState = firstObservation
134
259
  ? ready ? "idle" : "working"
135
- : paneChanged
260
+ : paneChanged && !settled
136
261
  ? "working"
137
262
  : ready
138
263
  ? "idle"
@@ -193,6 +318,23 @@ export class PendingWorkTracker {
193
318
  const NORMAL_ENTER_SETTLE_MS = 500;
194
319
  const FIRST_ENTER_SETTLE_MS = 1_750;
195
320
  const FIRST_DELIVERY_WINDOW_MS = 5_000;
321
+ /** After busy native-queue paste+Enter, wait before checking the pane for silent loss. */
322
+ const NATIVE_QUEUE_PASTE_VERIFY_MS = 2_000;
323
+ /** Enter-confirmation poll: 10 × 200ms ≈ 2s of observed silence before giving up. */
324
+ const CONFIRM_BUSY_POLLS = 10;
325
+ const CONFIRM_BUSY_POLL_MS = 200;
326
+ /**
327
+ * Ceiling on the whole confirmation, including time re-earned after a control-mode
328
+ * reconnect. Without it a reconnect loop could restart the poll indefinitely and
329
+ * hold the pane write lock along with it.
330
+ */
331
+ const CONFIRM_BUSY_MAX_WAIT_MS = 10_000;
332
+ /**
333
+ * How long a delivery waits for an in-flight spawn. Comfortably past the default
334
+ * 25s startup timeout plus dialog dismissal; past it we fall back to the old
335
+ * behaviour rather than holding a message indefinitely.
336
+ */
337
+ const SPAWN_SETTLE_MAX_WAIT_MS = 60_000;
196
338
  /**
197
339
  * One-shot timing gate for the first paste after a CLI reaches its ready
198
340
  * prompt. Recent tmux versions expose a short interval where the prompt is
@@ -244,13 +386,13 @@ export class Daemon extends EventEmitter {
244
386
  topicMode;
245
387
  backend;
246
388
  controlClient;
389
+ runtimeIdentity;
247
390
  logger;
248
391
  tmuxSessionName;
249
392
  tmux = null;
250
393
  ipcServer = null;
251
394
  messageBus;
252
395
  transcriptMonitor = null;
253
- toolTracker = null;
254
396
  guardian = null;
255
397
  adapter = null;
256
398
  pendingIpcRequests = new Map();
@@ -261,6 +403,8 @@ export class Daemon extends EventEmitter {
261
403
  // Pending ack: react 🫡 on first transcript activity after receiving a message
262
404
  pendingAckMessage = null;
263
405
  // Tool status tracking for channel adapter
406
+ /** Last activity published to the fleet manager; null when nothing is running. */
407
+ currentActivity = null;
264
408
  toolStatusMessageId = null;
265
409
  toolStatusLines = [];
266
410
  toolStatusDebounce = null;
@@ -276,9 +420,19 @@ export class Daemon extends EventEmitter {
276
420
  lastSpawnAt = 0;
277
421
  crashTimestamps = [];
278
422
  healthCheckPaused = false;
423
+ lastHealthErrorNotifyAt = 0;
279
424
  /** CLI pane availability, independent from the daemon process and tri-state. */
280
425
  processStatus = "running";
281
426
  spawning = false;
427
+ /**
428
+ * Resolves when the in-flight spawn finishes; null when none is running.
429
+ *
430
+ * `spawning` alone can only be polled. Delivery needs to *wait*, and a boolean
431
+ * poll would either busy-loop or race the flag being cleared.
432
+ */
433
+ spawnSettled = null;
434
+ resolveSpawnSettled = null;
435
+ spawnDepth = 0;
282
436
  skipResume = false;
283
437
  backgroundSessionRecoveryAttempted = false;
284
438
  /** Whether the last spawn started a fresh session (not resumed). */
@@ -301,10 +455,18 @@ export class Daemon extends EventEmitter {
301
455
  instanceStateIdleDebounceMs = DEFAULT_STATE_IDLE_DEBOUNCE_MS;
302
456
  instanceStateStuckTimeoutMs = DEFAULT_STUCK_TIMEOUT_MS;
303
457
  instanceStateReadyPattern = null;
458
+ instanceStateBusyPattern = null;
304
459
  instanceStateMonitorActive = false;
305
460
  statePollInFlight = false;
306
461
  autoPauseController;
307
462
  pauseRequested = false;
463
+ /**
464
+ * Sticky "pause as soon as possible" (auth failure). Unlike pauseRequested —
465
+ * which the state monitor clears whenever the pane is not idle — this survives
466
+ * busy/stuck states, because an auth error fires mid-turn and a plain pause()
467
+ * would silently no-op exactly when we most need to stop feeding the CLI.
468
+ */
469
+ pausePending = false;
308
470
  pauseWakeState = "active";
309
471
  pauseWakeTransition = null;
310
472
  // Model failover: override model on next spawn when rate-limited
@@ -314,7 +476,13 @@ export class Daemon extends EventEmitter {
314
476
  recentEvents = [];
315
477
  recentToolActivity = [];
316
478
  snapshotConsumed = false;
479
+ /** Orders inbound channel messages against each other (queue-depth accounting
480
+ * and ⏳/👀/✅ reactions live here). It does NOT cover the other writers that
481
+ * reach the pane — {@link paneWriteLock} does. */
317
482
  pasteLock = Promise.resolve();
483
+ /** Mutual exclusion for *every* write into the pane, whichever subsystem it
484
+ * comes from. See PaneWriteLock for why interleaving is destructive. */
485
+ paneWriteLock = new PaneWriteLock();
318
486
  pendingInstructionsUpdate;
319
487
  pendingInstructionsNotice = false;
320
488
  // Whether the warmup steering-reload notice should be injected after spawn.
@@ -327,6 +495,10 @@ export class Daemon extends EventEmitter {
327
495
  firstDeliveryDelay = new FirstDeliveryDelay();
328
496
  // PTY error pattern monitoring
329
497
  errorMonitorTimer = null;
498
+ /** Same 5-min gate the error monitor uses, so a dead MCP server alerts once. */
499
+ static MCP_DEATH_COOLDOWN_MS = 5 * 60_000;
500
+ lastMcpDeathNotifiedAt = 0;
501
+ mcpDeathNotifiedForPid = null;
330
502
  /** Prevent in-flight monitor callbacks from re-arming after a pause. */
331
503
  runtimeMonitorsFrozen = false;
332
504
  errorWaitingForRecovery = false; // true = error detected, waiting for ready pattern
@@ -371,7 +543,7 @@ export class Daemon extends EventEmitter {
371
543
  this.errorRecoveryDeadlineAt = 0;
372
544
  this.activeErrorPatternKey = null;
373
545
  }
374
- constructor(name, config, instanceDir, topicMode = false, backend, controlClient, rootLogger) {
546
+ constructor(name, config, instanceDir, topicMode = false, backend, controlClient, rootLogger, runtimeIdentity) {
375
547
  super();
376
548
  this.name = name;
377
549
  this.config = config;
@@ -379,8 +551,14 @@ export class Daemon extends EventEmitter {
379
551
  this.topicMode = topicMode;
380
552
  this.backend = backend;
381
553
  this.controlClient = controlClient;
554
+ this.runtimeIdentity = runtimeIdentity;
382
555
  if (!rootLogger)
383
556
  throw new Error("Daemon requires a shared root logger");
557
+ this.runtimeIdentity ??= {
558
+ kind: "fleet-topic",
559
+ backend: config.backend ?? backend?.binaryName ?? "unknown",
560
+ model: config.model ?? "default",
561
+ };
384
562
  this.logger = rootLogger.child({ instance: name }, { level: config.log_level });
385
563
  this.tmuxSessionName = getTmuxSession();
386
564
  this.messageBus = new MessageBus();
@@ -447,9 +625,6 @@ export class Daemon extends EventEmitter {
447
625
  if ((type === "fleet_schedule_response" || type === "fleet_outbound_response" || type === "fleet_decision_response" || type === "fleet_task_response" || type === "fleet_display_name_response" || type === "fleet_description_response") && msg.fleetRequestId) {
448
626
  key = String(msg.fleetRequestId);
449
627
  }
450
- else if (type === "fleet_outbound_response" && msg.requestId != null) {
451
- key = `fleet_out_${msg.requestId}`;
452
- }
453
628
  if (key && this.pendingIpcRequests.has(key)) {
454
629
  const handler = this.pendingIpcRequests.get(key);
455
630
  this.pendingIpcRequests.delete(key);
@@ -600,7 +775,12 @@ export class Daemon extends EventEmitter {
600
775
  else {
601
776
  await new Promise(r => setTimeout(r, 5000));
602
777
  }
603
- await this.tmux?.pasteText(buildInstructionReloadNotice(this.backend?.binaryName ?? "unknown", this.name, this.instanceDir));
778
+ // This path only runs when pasteQueueDepth > 0 — i.e. exactly when a real
779
+ // delivery is already in flight or queued. Without the lock the notice and
780
+ // that delivery race into the same pane.
781
+ await this.paneWriteLock.run(async () => {
782
+ await this.tmux?.pasteText(buildInstructionReloadNotice(this.backend?.binaryName ?? "unknown", this.name, this.instanceDir), this.systemPasteOptions());
783
+ });
604
784
  // Record the value the agent has now been told about so the next
605
785
  // unchanged restart skips the reload.
606
786
  try {
@@ -612,8 +792,12 @@ export class Daemon extends EventEmitter {
612
792
  catch { /* non-fatal */ }
613
793
  })();
614
794
  if (!this.config.lightweight) {
615
- // 3. Pipe-pane for prompt detection
795
+ // 3. Pipe-pane for prompt detection. Rotate first so a ballooned log from a
796
+ // previous stuck splash (hundreds of MB of ANSI frames) is truncated before
797
+ // we attach — pipe-pane uses `cat >>` on the same inode, so copytruncate
798
+ // keeps the writer attached after size resets.
616
799
  const outputLog = join(this.instanceDir, "output.log");
800
+ rotateLogIfNeeded(outputLog);
617
801
  await this.tmux.pipeOutput(outputLog).catch(() => { });
618
802
  // 4. Transcript monitor
619
803
  this.transcriptMonitor = new TranscriptMonitor(this.instanceDir, this.logger);
@@ -629,27 +813,31 @@ export class Daemon extends EventEmitter {
629
813
  this.transcriptMonitor.on("tool_use", (name, input) => {
630
814
  this.logger.debug({ tool: name }, "Tool use");
631
815
  ackIfPending();
632
- this.hangDetector?.recordActivity();
633
816
  this.recordRecentEvent({ type: "tool_use", name, preview: this.summarizeTool(name, input) });
634
817
  this.recordRecentToolActivity(this.summarizeTool(name, input));
818
+ this.publishActivity(this.summarizeTool(name, input));
635
819
  });
636
820
  this.transcriptMonitor.on("tool_result", (name, _output) => {
637
- this.hangDetector?.recordActivity();
638
821
  this.recordRecentEvent({ type: "tool_result", name });
822
+ // The tool finished; whatever comes next has not started yet. Better to
823
+ // show only elapsed time than to leave a stale "Bash: npm test" on screen.
824
+ this.publishActivity(null);
639
825
  });
640
826
  this.transcriptMonitor.on("assistant_text", (text) => {
641
827
  this.logger.debug({ text: text.slice(0, 200) }, "Claude response");
642
828
  ackIfPending();
643
- this.hangDetector?.recordActivity();
644
829
  this.recordRecentEvent({ type: "assistant_text", preview: text.slice(0, 100) });
645
830
  });
646
831
  this.transcriptMonitor.startPolling();
647
- // HangDetector remains the fleet-manager notification event bridge. Its
648
- // legacy silence timer is intentionally not started: pane state transitions
649
- // below are now the sole source of hang events.
832
+ // HangDetector is the notification bridge to the fleet manager; pane state
833
+ // transitions below are the sole source of hang events (see hang-detector.ts).
834
+ // `hang_detector.enabled: false` opts out of the notification entirely.
835
+ // NOTE: `hang_detector.timeout_minutes` is NOT read here — the stuck timeout
836
+ // the pane monitor actually uses is resolved separately below. The two have
837
+ // long been separate; documented rather than silently ignored.
650
838
  const hangConfig = this.config.hang_detector;
651
839
  if (hangConfig?.enabled !== false) {
652
- this.hangDetector = new HangDetector(hangConfig?.timeout_minutes ?? 10);
840
+ this.hangDetector = new HangDetector();
653
841
  }
654
842
  // 8. Context guardian
655
843
  const statusFile = join(this.instanceDir, "statusline.json");
@@ -657,7 +845,6 @@ export class Daemon extends EventEmitter {
657
845
  this.guardian.startWatching();
658
846
  this.guardian.on("status_update", () => {
659
847
  this.saveSessionId();
660
- this.hangDetector?.recordStatuslineUpdate();
661
848
  });
662
849
  // Context rotation removed: all CLI backends have built-in auto-compact.
663
850
  // Crash recovery (health check + respawn with snapshot) is retained below.
@@ -677,6 +864,40 @@ export class Daemon extends EventEmitter {
677
864
  this.startInstanceStateMonitor();
678
865
  this.logger.info(`${this.name} ready`);
679
866
  }
867
+ /**
868
+ * Detect a CRASHED MCP server and report it once.
869
+ *
870
+ * The MCP server writes its pid to channel.mcp.pid at startup and unlinks it on
871
+ * a clean exit, so "file present + pid dead" specifically means it died without
872
+ * cleaning up (crash / OOM / SIGKILL). A missing file is either "not started
873
+ * yet" or "exited cleanly" — neither is an incident, and an orphan exit implies
874
+ * the CLI itself died, which the crash detector already covers.
875
+ *
876
+ * The daemon deliberately does NOT try to respawn it: MCP is a stdio protocol
877
+ * where the CLI spawns the server and owns its pipes, so a daemon-spawned
878
+ * process would have no client reading it. Only the CLI can restore its own
879
+ * tools — hence notify, and let the operator decide about restarting.
880
+ */
881
+ checkMcpServerAlive() {
882
+ if (this.isPaused)
883
+ return;
884
+ const status = mcpServerState(this.instanceDir);
885
+ if (status.state === "unknown")
886
+ return;
887
+ if (status.state === "alive") {
888
+ this.mcpDeathNotifiedForPid = null; // the CLI respawned it — re-arm
889
+ return;
890
+ }
891
+ // Dead: report once per pid, and at most once per cooldown window.
892
+ if (this.mcpDeathNotifiedForPid === status.pid)
893
+ return;
894
+ if (Date.now() - this.lastMcpDeathNotifiedAt < Daemon.MCP_DEATH_COOLDOWN_MS)
895
+ return;
896
+ this.mcpDeathNotifiedForPid = status.pid;
897
+ this.lastMcpDeathNotifiedAt = Date.now();
898
+ this.logger.error({ pid: status.pid }, "MCP server process is gone — instance has no agend tools");
899
+ this.emit("mcp_died", { name: this.name, pid: status.pid });
900
+ }
680
901
  startHealthCheck() {
681
902
  if (this.runtimeMonitorsFrozen || this.healthCheckTimer)
682
903
  return;
@@ -692,249 +913,285 @@ export class Daemon extends EventEmitter {
692
913
  this.healthCheckTimer = null;
693
914
  if (this.runtimeMonitorsFrozen)
694
915
  return;
695
- // Instance directory removed externally (e.g. `rm -rf ~/.agend/instances/<name>`).
696
- // Stop the loop permanently otherwise every tick triggers a respawn, whose
697
- // writeRotationSnapshot fails with ENOENT and gets caught as "Failed to respawn",
698
- // spamming errors every ~30s forever.
699
- if (!existsSync(this.instanceDir)) {
700
- this.logger.warn({ instanceDir: this.instanceDir }, "Instance directory missing — stopping health check");
701
- this.healthCheckPaused = true;
702
- this.healthCheckTimer = null;
703
- return;
704
- }
705
- if (!this.tmux || this.spawning || this.healthCheckPaused || Daemon.tmuxServerPaused) {
706
- scheduleNext();
707
- return;
708
- }
709
- // Human-readable backend label for logs (e.g. "claude", "kiro-cli")
710
- const cliLabel = this.backend?.binaryName ?? "CLI";
711
- let paneStatus = await this.tmux.getPaneStatus();
712
- // Auto-pause intentionally exits the pane process. A health tick that
713
- // began just before pause must not classify that exit as a crash.
714
- if (this.isPaused || this.pauseWakeState === "waking") {
715
- scheduleNext();
716
- return;
717
- }
718
- if (paneStatus?.alive) {
719
- scheduleNext();
720
- return;
721
- }
722
- // A null status is ambiguous: it can be a transient `tmux list-panes`
723
- // failure (e.g. tmux busy during a fleet-restart storm) rather than a
724
- // real exit. Re-confirm once after a short delay before treating it as
725
- // a crash. A non-null {alive:false} is a definite dead pane (real exit)
726
- // and needs no recheck.
727
- if (paneStatus === null) {
728
- await new Promise(r => setTimeout(r, 1500));
729
- paneStatus = await this.tmux.getPaneStatus();
916
+ // The whole tick is guarded: an unguarded throw in here (a tmux hiccup,
917
+ // ENOSPC on the crash-history write) became an unhandled rejection, which
918
+ // the CLI turned into stopAll() + exit(1) for the ENTIRE fleet, and it also
919
+ // ended this instance's health loop because scheduleNext() sat after the
920
+ // throwing code. Catch, log, and keep checking.
921
+ try {
922
+ // Instance directory removed externally (e.g. `rm -rf ~/.agend/instances/<name>`).
923
+ // Stop the loop permanently — otherwise every tick triggers a respawn, whose
924
+ // writeRotationSnapshot fails with ENOENT and gets caught as "Failed to respawn",
925
+ // spamming errors every ~30s forever.
926
+ if (!existsSync(this.instanceDir)) {
927
+ this.logger.warn({ instanceDir: this.instanceDir }, "Instance directory missing — stopping health check");
928
+ this.healthCheckPaused = true;
929
+ this.healthCheckTimer = null;
930
+ this.emitSupervisionEnded("instance directory was removed from disk", "Recreate the instance, or delete it from fleet.yaml.");
931
+ return;
932
+ }
933
+ if (!this.tmux || this.spawning || this.healthCheckPaused || Daemon.tmuxServerPaused) {
934
+ scheduleNext();
935
+ return;
936
+ }
937
+ // The CLI owns the MCP server process, so the daemon can only observe it.
938
+ this.checkMcpServerAlive();
939
+ // Human-readable backend label for logs (e.g. "claude", "kiro-cli")
940
+ const cliLabel = this.backend?.binaryName ?? "CLI";
941
+ let paneStatus = await this.tmux.getPaneStatus();
942
+ // Auto-pause intentionally exits the pane process. A health tick that
943
+ // began just before pause must not classify that exit as a crash.
944
+ if (this.isPaused || this.pauseWakeState === "waking") {
945
+ scheduleNext();
946
+ return;
947
+ }
730
948
  if (paneStatus?.alive) {
731
- this.logger.debug(`[health] ${cliLabel} pane reported gone then alive on recheck transient query failure, ignoring`);
949
+ // Instance output.log is fed by tmux pipe-pane and was previously never
950
+ // rotated (only fleet.log / daemon.log were). Cap growth every tick.
951
+ if (!this.config.lightweight) {
952
+ rotateLogIfNeeded(join(this.instanceDir, "output.log"));
953
+ }
732
954
  scheduleNext();
733
955
  return;
734
956
  }
735
- }
736
- // paneStatus === null → window gone entirely (e.g. tmux server crash)
737
- // paneStatus.alive === false pane dead, exit code available
738
- const exitCode = paneStatus?.exitCode;
739
- this.logger.debug({ exitCode }, `[health] pane exited with code: ${exitCode}`);
740
- // Normal exit (e.g. user Ctrl+C or /exit) — no crash, no respawn
741
- if (paneStatus && exitCode === 0) {
742
- this.setProcessStatus("stopped");
743
- this.logger.info("CLI exited normally (code 0) — pausing health check");
744
- await this.tmux.killWindow();
745
- this.healthCheckPaused = true;
746
- return;
747
- }
748
- this.setProcessStatus("crashed");
749
- // Distinguish tmux server crash from single window crash.
750
- // nullReason records *why* getPaneStatus returned null (for diagnosing
751
- // whether this was a real window loss or a transient query failure).
752
- let crashType = "window";
753
- let nullReason;
754
- if (!paneStatus) {
755
- const serverAlive = await TmuxManager.sessionExists(this.tmuxSessionName);
756
- if (!serverAlive) {
757
- crashType = "server";
758
- nullReason = "server_gone";
759
- this.logger.error(`tmux server died — all ${cliLabel} windows lost`);
760
- // Fleet-level circuit breaker: pause all instances on repeated tmux server crashes
761
- Daemon.tmuxServerCrashTimestamps.push(Date.now());
762
- const cutoff = Date.now() - 5 * 60_000;
763
- Daemon.tmuxServerCrashTimestamps = Daemon.tmuxServerCrashTimestamps.filter(t => t > cutoff);
764
- if (Daemon.tmuxServerCrashTimestamps.length >= 2 && !Daemon.tmuxServerPaused) {
765
- Daemon.tmuxServerPaused = true;
766
- this.logger.error("Fleet-level tmux server circuit breaker triggered — pausing all respawns for 30s");
767
- this.emit("tmux_server_crash", this.name);
768
- if (!Daemon.tmuxServerRecoveryTimer) {
769
- Daemon.tmuxServerRecoveryTimer = setTimeout(() => {
770
- Daemon.tmuxServerRecoveryTimer = null;
771
- Daemon.tmuxServerPaused = false;
772
- }, 30_000);
773
- }
957
+ // A null status is ambiguous: it can be a transient `tmux list-panes`
958
+ // failure (e.g. tmux busy during a fleet-restart storm) rather than a
959
+ // real exit. Re-confirm once after a short delay before treating it as
960
+ // a crash. A non-null {alive:false} is a definite dead pane (real exit)
961
+ // and needs no recheck.
962
+ if (paneStatus === null) {
963
+ await new Promise(r => setTimeout(r, 1500));
964
+ paneStatus = await this.tmux.getPaneStatus();
965
+ if (paneStatus?.alive) {
966
+ this.logger.debug(`[health] ${cliLabel} pane reported gone then alive on recheck — transient query failure, ignoring`);
774
967
  scheduleNext();
775
968
  return;
776
969
  }
777
- await new Promise(r => setTimeout(r, 2_000)); // let session stabilize
778
970
  }
779
- else {
780
- // null but server alive: window-level disappearance. Probe whether
781
- // the window truly no longer exists vs a transient query glitch.
782
- nullReason = "no_window";
783
- try {
784
- const windows = await TmuxManager.listWindows(this.tmuxSessionName);
785
- if (windows.some(w => w.name === this.name))
786
- nullReason = "window_present_query_glitch";
971
+ // paneStatus === null → window gone entirely (e.g. tmux server crash)
972
+ // paneStatus.alive === false pane dead, exit code available
973
+ const exitCode = paneStatus?.exitCode;
974
+ this.logger.debug({ exitCode }, `[health] pane exited with code: ${exitCode}`);
975
+ // Normal exit (e.g. user Ctrl+C or /exit) — no crash, no respawn
976
+ if (paneStatus && exitCode === 0) {
977
+ this.setProcessStatus("stopped");
978
+ this.logger.info("CLI exited normally (code 0) — pausing health check");
979
+ await this.tmux.killWindow();
980
+ this.healthCheckPaused = true;
981
+ this.emitSupervisionEnded("the CLI exited normally (code 0)", "Nothing crashed — start it again when you need it.");
982
+ return;
983
+ }
984
+ this.setProcessStatus("crashed");
985
+ // Distinguish tmux server crash from single window crash.
986
+ // nullReason records *why* getPaneStatus returned null (for diagnosing
987
+ // whether this was a real window loss or a transient query failure).
988
+ let crashType = "window";
989
+ let nullReason;
990
+ if (!paneStatus) {
991
+ const serverAlive = await TmuxManager.sessionExists(this.tmuxSessionName);
992
+ if (!serverAlive) {
993
+ crashType = "server";
994
+ nullReason = "server_gone";
995
+ this.logger.error(`tmux server died — all ${cliLabel} windows lost`);
996
+ // Fleet-level circuit breaker: pause all instances on repeated tmux server crashes
997
+ Daemon.tmuxServerCrashTimestamps.push(Date.now());
998
+ const cutoff = Date.now() - 5 * 60_000;
999
+ Daemon.tmuxServerCrashTimestamps = Daemon.tmuxServerCrashTimestamps.filter(t => t > cutoff);
1000
+ if (Daemon.tmuxServerCrashTimestamps.length >= 2 && !Daemon.tmuxServerPaused) {
1001
+ Daemon.tmuxServerPaused = true;
1002
+ this.logger.error("Fleet-level tmux server circuit breaker triggered — pausing all respawns for 30s");
1003
+ this.emit("tmux_server_crash", this.name);
1004
+ if (!Daemon.tmuxServerRecoveryTimer) {
1005
+ Daemon.tmuxServerRecoveryTimer = setTimeout(() => {
1006
+ Daemon.tmuxServerRecoveryTimer = null;
1007
+ Daemon.tmuxServerPaused = false;
1008
+ }, 30_000);
1009
+ }
1010
+ scheduleNext();
1011
+ return;
1012
+ }
1013
+ await new Promise(r => setTimeout(r, 2_000)); // let session stabilize
787
1014
  }
788
- catch {
789
- nullReason = "query_error";
1015
+ else {
1016
+ // null but server alive: window-level disappearance. Probe whether
1017
+ // the window truly no longer exists vs a transient query glitch.
1018
+ nullReason = "no_window";
1019
+ try {
1020
+ const windows = await TmuxManager.listWindows(this.tmuxSessionName);
1021
+ if (windows.some(w => w.name === this.name))
1022
+ nullReason = "window_present_query_glitch";
1023
+ }
1024
+ catch {
1025
+ nullReason = "query_error";
1026
+ }
1027
+ this.logger.warn({ exitCode, nullReason }, `${cliLabel} window not found (tmux server alive)`);
790
1028
  }
791
- this.logger.warn({ exitCode, nullReason }, `${cliLabel} window not found (tmux server alive)`);
792
1029
  }
793
- }
794
- else {
795
- this.logger.warn({ exitCode }, `${cliLabel} process exited`);
796
- }
797
- // Capture last output before killing. Best-effort even when the pane is
798
- // gone (paneStatus null) gives the crash record something to diagnose
799
- // from instead of an empty lastOutput.
800
- let lastOutput;
801
- try {
802
- const raw = await this.tmux.capturePaneWithHistory(50);
803
- // Strip ANSI escape codes for readability
804
- const cleaned = raw.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
805
- lastOutput = cleaned.trimEnd() || undefined;
806
- }
807
- catch { /* best effort pane may already be gone */ }
808
- // Kill the dead window (remain-on-exit keeps it around) before respawn
809
- if (paneStatus) {
810
- await this.tmux.killWindow();
811
- }
812
- // Detect claude-code background session conflict recover without counting as crash
813
- if (lastOutput && (lastOutput.includes("background agent") || lastOutput.includes("Session is currently running"))) {
814
- if (!this.backgroundSessionRecoveryAttempted) {
815
- this.backgroundSessionRecoveryAttempted = true;
816
- this.logger.warn("Detected lingering background agent session — starting fresh (no resume)");
817
- const sidFile = join(this.instanceDir, "session-id");
1030
+ else {
1031
+ this.logger.warn({ exitCode }, `${cliLabel} process exited`);
1032
+ }
1033
+ // Capture last output before killing. Best-effort even when the pane is
1034
+ // gone (paneStatus null) gives the crash record something to diagnose
1035
+ // from instead of an empty lastOutput.
1036
+ let lastOutput;
1037
+ try {
1038
+ const raw = await this.tmux.capturePaneWithHistory(50);
1039
+ // Strip ANSI escape codes for readability
1040
+ const cleaned = raw.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
1041
+ lastOutput = cleaned.trimEnd() || undefined;
1042
+ }
1043
+ catch { /* best effort — pane may already be gone */ }
1044
+ // Kill the dead window (remain-on-exit keeps it around) before respawn
1045
+ if (paneStatus) {
1046
+ await this.tmux.killWindow();
1047
+ }
1048
+ // Detect claude-code background session conflict — recover without counting as crash
1049
+ if (lastOutput && (lastOutput.includes("background agent") || lastOutput.includes("Session is currently running"))) {
1050
+ if (!this.backgroundSessionRecoveryAttempted) {
1051
+ this.backgroundSessionRecoveryAttempted = true;
1052
+ this.logger.warn("Detected lingering background agent session — starting fresh (no resume)");
1053
+ const sidFile = join(this.instanceDir, "session-id");
1054
+ try {
1055
+ unlinkSync(sidFile);
1056
+ }
1057
+ catch { }
1058
+ this.skipResume = true;
1059
+ await new Promise(r => setTimeout(r, 2_000));
1060
+ try {
1061
+ await this.spawnClaudeWindow();
1062
+ this.setProcessStatus("running");
1063
+ this.logger.info("Recovered from background session conflict");
1064
+ this.emit("crash_respawn", this.name);
1065
+ }
1066
+ catch (err) {
1067
+ this.logger.error({ err: err.message }, "Recovery from background session conflict failed");
1068
+ }
1069
+ return; // Don't count as crash
1070
+ }
1071
+ // Already attempted recovery — fall through to normal crash handling
1072
+ }
1073
+ // Detect a --continue/--resume failure (no conversation to resume). The
1074
+ // session-id file persists across the crash, so a blind respawn would add
1075
+ // --continue again and crash in the same way → loop. Clear the session id
1076
+ // and skip resume so the next spawn starts fresh. (skipResume also stops
1077
+ // saveSessionId below from resurrecting the id from statusline.json.)
1078
+ if (lastOutput && /no conversation found|no conversation to (continue|resume)|no previous (session|conversation)|--continue/i.test(lastOutput)) {
1079
+ this.logger.warn("Detected --continue/resume failure — clearing session-id; next spawn starts fresh");
818
1080
  try {
819
- unlinkSync(sidFile);
1081
+ unlinkSync(join(this.instanceDir, "session-id"));
820
1082
  }
821
- catch { }
1083
+ catch { /* may not exist */ }
822
1084
  this.skipResume = true;
823
- await new Promise(r => setTimeout(r, 2_000));
1085
+ }
1086
+ // Append to crash history
1087
+ this.appendCrashHistory({ exitCode, lastOutput, crashType, reason: nullReason });
1088
+ if (max_retries <= 0) {
1089
+ this.healthCheckPaused = true;
1090
+ this.logger.warn(`${cliLabel} window died — automatic restart is disabled`);
1091
+ this.emitSupervisionEnded("the CLI died and automatic restart is disabled (restart_policy.max_retries is 0)", "Start it manually, or raise max_retries in fleet.yaml.");
1092
+ return;
1093
+ }
1094
+ // Detect rapid crash: sliding window — 3+ crashes in 5 minutes
1095
+ this.crashTimestamps.push(Date.now());
1096
+ const crashWindowMs = 5 * 60_000;
1097
+ this.crashTimestamps = this.crashTimestamps.filter(t => t > Date.now() - crashWindowMs);
1098
+ if (this.crashTimestamps.length >= 3) {
1099
+ this.healthCheckPaused = true;
1100
+ this.logger.error({ crashesInWindow: this.crashTimestamps.length }, "3+ crashes in 5 minutes — pausing respawn");
1101
+ // P1: Persist crash state so next process restart skips resume
824
1102
  try {
825
- await this.spawnClaudeWindow();
826
- this.setProcessStatus("running");
827
- this.logger.info("Recovered from background session conflict");
828
- this.emit("crash_respawn", this.name);
829
- }
830
- catch (err) {
831
- this.logger.error({ err: err.message }, "Recovery from background session conflict failed");
1103
+ writeFileSync(join(this.instanceDir, "crash-state.json"), JSON.stringify({
1104
+ crashesInWindow: this.crashTimestamps.length,
1105
+ lastCrashAt: Date.now(),
1106
+ resumeDisabled: true,
1107
+ }));
832
1108
  }
833
- return; // Don't count as crash
1109
+ catch { /* best effort */ }
1110
+ this.emit("crash_loop", this.name);
1111
+ return; // don't schedule next — paused
834
1112
  }
835
- // Already attempted recovery fall through to normal crash handling
836
- }
837
- // Detect a --continue/--resume failure (no conversation to resume). The
838
- // session-id file persists across the crash, so a blind respawn would add
839
- // --continue again and crash in the same way → loop. Clear the session id
840
- // and skip resume so the next spawn starts fresh. (skipResume also stops
841
- // saveSessionId below from resurrecting the id from statusline.json.)
842
- if (lastOutput && /no conversation found|no conversation to (continue|resume)|no previous (session|conversation)|--continue/i.test(lastOutput)) {
843
- this.logger.warn("Detected --continue/resume failure — clearing session-id; next spawn starts fresh");
844
- try {
845
- unlinkSync(join(this.instanceDir, "session-id"));
846
- }
847
- catch { /* may not exist */ }
848
- this.skipResume = true;
849
- }
850
- // Append to crash history
851
- this.appendCrashHistory({ exitCode, lastOutput, crashType, reason: nullReason });
852
- if (max_retries <= 0) {
853
- this.healthCheckPaused = true;
854
- this.logger.warn(`${cliLabel} window died — automatic restart is disabled`);
855
- return;
856
- }
857
- // Detect rapid crash: sliding window — 3+ crashes in 5 minutes
858
- this.crashTimestamps.push(Date.now());
859
- const crashWindowMs = 5 * 60_000;
860
- this.crashTimestamps = this.crashTimestamps.filter(t => t > Date.now() - crashWindowMs);
861
- if (this.crashTimestamps.length >= 3) {
862
- this.healthCheckPaused = true;
863
- this.logger.error({ crashesInWindow: this.crashTimestamps.length }, "3+ crashes in 5 minutes — pausing respawn");
864
- // P1: Persist crash state so next process restart skips resume
865
- try {
866
- writeFileSync(join(this.instanceDir, "crash-state.json"), JSON.stringify({
867
- crashesInWindow: this.crashTimestamps.length,
868
- lastCrashAt: Date.now(),
869
- resumeDisabled: true,
870
- }));
1113
+ // Reset crash count if enough time has passed
1114
+ if (reset_after > 0 && Date.now() - this.lastCrashAt > reset_after) {
1115
+ this.crashCount = 0;
871
1116
  }
872
- catch { /* best effort */ }
873
- this.emit("crash_loop", this.name);
874
- return; // don't schedule next — paused
875
- }
876
- // Reset crash count if enough time has passed
877
- if (reset_after > 0 && Date.now() - this.lastCrashAt > reset_after) {
878
- this.crashCount = 0;
879
- }
880
- this.crashCount++;
881
- this.lastCrashAt = Date.now();
882
- if (this.crashCount > max_retries) {
883
- this.logger.error({ crashCount: this.crashCount, maxRetries: max_retries }, "Max crash retries exceeded — not respawning");
884
- return; // don't schedule next — given up
885
- }
886
- // Calculate backoff delay
887
- const delay = backoff === "exponential"
888
- ? Math.min(1000 * Math.pow(2, this.crashCount - 1), 60_000)
889
- : 1000 * this.crashCount;
890
- this.logger.warn({ crashCount: this.crashCount, delay }, `${cliLabel} window died — respawning after backoff`);
891
- await new Promise(r => setTimeout(r, delay));
892
- try {
893
- this.saveSessionId();
894
- this.transcriptMonitor?.resetOffset();
895
- // Kill orphan MCP server from the crashed CLI session.
896
- // MCP server writes its PID to channel.mcp.pid on startup.
897
- try {
898
- const pidFile = join(this.instanceDir, "channel.mcp.pid");
899
- const pid = parseInt(readFileSync(pidFile, "utf-8").trim(), 10);
900
- process.kill(pid, "SIGTERM");
901
- this.logger.info({ pid }, "Killed orphan MCP server");
1117
+ this.crashCount++;
1118
+ this.lastCrashAt = Date.now();
1119
+ if (this.crashCount > max_retries) {
1120
+ this.logger.error({ crashCount: this.crashCount, maxRetries: max_retries }, "Max crash retries exceeded — not respawning");
1121
+ this.healthCheckPaused = true;
1122
+ this.emitSupervisionEnded(`it crashed ${this.crashCount} times, exceeding restart_policy.max_retries (${max_retries})`, "Check the logs for the cause, then restart it.");
1123
+ return; // don't schedule next — given up
902
1124
  }
903
- catch { /* no pid file or process already dead */ }
904
- // Kill any same-name windows before respawn to prevent orphans.
905
- // Wrapped in try-catch: if tmux server is dead, listWindows throws —
906
- // must not block spawnClaudeWindow (which calls ensureSession).
1125
+ // Calculate backoff delay
1126
+ const delay = backoff === "exponential"
1127
+ ? Math.min(1000 * Math.pow(2, this.crashCount - 1), 60_000)
1128
+ : 1000 * this.crashCount;
1129
+ this.logger.warn({ crashCount: this.crashCount, delay }, `${cliLabel} window died — respawning after backoff`);
1130
+ await new Promise(r => setTimeout(r, delay));
907
1131
  try {
908
- const windows = await TmuxManager.listWindows(this.tmuxSessionName);
909
- for (const w of windows) {
910
- if (w.name === this.name) {
911
- const tm = new TmuxManager(this.tmuxSessionName, w.id);
912
- await tm.killWindow();
913
- }
1132
+ this.saveSessionId();
1133
+ this.transcriptMonitor?.resetOffset();
1134
+ // Kill orphan MCP server from the crashed CLI session.
1135
+ // MCP server writes its PID to channel.mcp.pid on startup.
1136
+ try {
1137
+ const pidFile = join(this.instanceDir, "channel.mcp.pid");
1138
+ const pid = parseInt(readFileSync(pidFile, "utf-8").trim(), 10);
1139
+ process.kill(pid, "SIGTERM");
1140
+ this.logger.info({ pid }, "Killed orphan MCP server");
914
1141
  }
915
- }
916
- catch { /* tmux server may be dead ensureSession in trySpawn will recover */ }
917
- // Write snapshot before spawn consumed only if resume fails
918
- this.writeRotationSnapshot("crash");
919
- // Try --resume first; spawnClaudeWindow falls back to fresh session if resume fails
920
- const resumed = await this.spawnClaudeWindow();
921
- if (!resumed) {
922
- // Resume failed → fresh session → inject snapshot for context
923
- await this.injectSnapshotMessage();
924
- }
925
- else {
926
- // Clean up stale snapshot — resume restored full context
1142
+ catch { /* no pid file or process already dead */ }
1143
+ // Kill any same-name windows before respawn to prevent orphans.
1144
+ // Wrapped in try-catch: if tmux server is dead, listWindows throws —
1145
+ // must not block spawnClaudeWindow (which calls ensureSession).
927
1146
  try {
928
- unlinkSync(join(this.instanceDir, "rotation-state.json"));
1147
+ const windows = await TmuxManager.listWindows(this.tmuxSessionName);
1148
+ for (const w of windows) {
1149
+ if (w.name === this.name) {
1150
+ const tm = new TmuxManager(this.tmuxSessionName, w.id);
1151
+ await tm.killWindow();
1152
+ }
1153
+ }
929
1154
  }
930
- catch { /* may not exist */ }
1155
+ catch { /* tmux server may be dead — ensureSession in trySpawn will recover */ }
1156
+ // Write snapshot before spawn — consumed only if resume fails
1157
+ this.writeRotationSnapshot("crash");
1158
+ // Try --resume first; spawnClaudeWindow falls back to fresh session if resume fails
1159
+ const resumed = await this.spawnClaudeWindow();
1160
+ if (!resumed) {
1161
+ // Resume failed → fresh session → inject snapshot for context
1162
+ await this.injectSnapshotMessage();
1163
+ }
1164
+ else {
1165
+ // Clean up stale snapshot — resume restored full context
1166
+ try {
1167
+ unlinkSync(join(this.instanceDir, "rotation-state.json"));
1168
+ }
1169
+ catch { /* may not exist */ }
1170
+ }
1171
+ this.setProcessStatus("running");
1172
+ this.logger.info({ resumed }, `Respawned ${cliLabel} window after crash`);
1173
+ this.emit("crash_respawn", this.name);
1174
+ }
1175
+ catch (err) {
1176
+ this.logger.error({ err }, `Failed to respawn ${cliLabel} window`);
931
1177
  }
932
- this.setProcessStatus("running");
933
- this.logger.info({ resumed }, `Respawned ${cliLabel} window after crash`);
934
- this.emit("crash_respawn", this.name);
935
1178
  }
936
1179
  catch (err) {
937
- this.logger.error({ err }, `Failed to respawn ${cliLabel} window`);
1180
+ this.logger.error({ err }, "Health check tick failed — continuing");
1181
+ // Surface it to the operator, not just the log — a health check that
1182
+ // keeps throwing means this instance is no longer being supervised.
1183
+ // Throttled: the tick repeats every ~30s, so an unnotified persistent
1184
+ // fault would otherwise post twice a minute forever.
1185
+ const now = Date.now();
1186
+ if (now - this.lastHealthErrorNotifyAt > HEALTH_ERROR_NOTIFY_INTERVAL_MS) {
1187
+ this.lastHealthErrorNotifyAt = now;
1188
+ this.emit("health_check_error", {
1189
+ name: this.name,
1190
+ message: err instanceof Error ? err.message : String(err),
1191
+ });
1192
+ }
1193
+ scheduleNext();
1194
+ return;
938
1195
  }
939
1196
  scheduleNext();
940
1197
  }, healthCheckIntervalMs);
@@ -946,6 +1203,21 @@ export class Daemon extends EventEmitter {
946
1203
  * A dead remain-on-exit pane still contains the old ready prompt, so allowing
947
1204
  * the pane monitor to capture it would re-create a false idle state.
948
1205
  */
1206
+ /**
1207
+ * Announce that this instance is no longer being supervised.
1208
+ *
1209
+ * Four health-check exits set `healthCheckPaused = true` and returned without
1210
+ * telling anyone: a clean CLI exit, `max_retries <= 0`, crash retries exhausted,
1211
+ * and the instance directory being deleted. Only the crash-LOOP case emitted an
1212
+ * event, so a fleet could quietly contain a dead instance that still looked fine
1213
+ * on the dashboard — while messages routed to it queued or failed with a bare ❌.
1214
+ *
1215
+ * `crash_loop` already had this treatment; this is the same bridge for the other
1216
+ * four, carrying a human-readable cause and the operator's next step.
1217
+ */
1218
+ emitSupervisionEnded(reason, remedy) {
1219
+ this.emit("supervision_ended", { name: this.name, reason, remedy });
1220
+ }
949
1221
  setProcessStatus(status) {
950
1222
  if (this.processStatus === status)
951
1223
  return;
@@ -991,6 +1263,7 @@ export class Daemon extends EventEmitter {
991
1263
  if (!this.backend)
992
1264
  return; // lightweight mode has no backend
993
1265
  const readyPattern = this.backend.getReadyPattern();
1266
+ const busyPattern = this.backend.getBusyPattern?.() ?? null;
994
1267
  this.errorMonitorTimer = setInterval(async () => {
995
1268
  if (!this.tmux || this.spawning)
996
1269
  return;
@@ -1003,20 +1276,30 @@ export class Daemon extends EventEmitter {
1003
1276
  for (const dialog of dialogs) {
1004
1277
  if (!dialog.pattern.test(pane))
1005
1278
  continue;
1006
- this.logger.info(`Auto-dismissing runtime dialog: ${dialog.description}`);
1007
- const SPECIAL_KEYS = new Set(["Up", "Down", "Enter", "Escape", "Right", "Left"]);
1008
- for (const key of dialog.keys) {
1009
- if (SPECIAL_KEYS.has(key)) {
1010
- await this.tmux.sendSpecialKey(key);
1011
- }
1012
- else {
1013
- await this.tmux.pasteText(key);
1279
+ // These keys go straight into the pane. Sent while a message delivery is
1280
+ // mid-transaction, an `Escape` wipes the pasted text and an `Enter`
1281
+ // submits it half-composed — the user sees a message that vanished. Skip
1282
+ // (not queue) when the pane is busy: this poller runs every 5s, and the
1283
+ // dialog will still be on screen next tick.
1284
+ const dismissed = await this.paneWriteLock.tryRun(async () => {
1285
+ this.logger.info(`Auto-dismissing runtime dialog: ${dialog.description}`);
1286
+ const SPECIAL_KEYS = new Set(["Up", "Down", "Enter", "Escape", "Right", "Left"]);
1287
+ for (const key of dialog.keys) {
1288
+ if (SPECIAL_KEYS.has(key)) {
1289
+ await this.tmux.sendSpecialKey(key);
1290
+ }
1291
+ else {
1292
+ await this.tmux.pasteText(key, this.systemPasteOptions());
1293
+ }
1294
+ await new Promise(r => setTimeout(r, 200));
1014
1295
  }
1015
- await new Promise(r => setTimeout(r, 200));
1296
+ });
1297
+ if (!dismissed) {
1298
+ this.logger.debug({ dialog: dialog.description }, "Dialog dismissal deferred — pane write in flight");
1016
1299
  }
1017
- return; // Dialog dismissed, skip error checks this cycle
1300
+ return; // Dialog handled (or deliberately deferred): skip error checks this cycle
1018
1301
  }
1019
- this.evaluateErrorPatterns(pane, patterns, readyPattern);
1302
+ this.evaluateErrorPatterns(pane, patterns, readyPattern, Date.now(), busyPattern);
1020
1303
  }
1021
1304
  catch {
1022
1305
  // capturePane can fail if window is transitioning — ignore
@@ -1024,7 +1307,7 @@ export class Daemon extends EventEmitter {
1024
1307
  }, 5_000); // Check every 5 seconds (runtime dialogs need fast response)
1025
1308
  }
1026
1309
  /** Evaluate one pane snapshot. Kept synchronous so state-machine edges are unit-testable. */
1027
- evaluateErrorPatterns(pane, patterns, readyPattern, now = Date.now()) {
1310
+ evaluateErrorPatterns(pane, patterns, readyPattern, now = Date.now(), busyPattern = null) {
1028
1311
  // Count occurrences across the WHOLE pane (not just text after the last
1029
1312
  // ready prompt). Clone with `g` so stateful backend regexes cannot leak
1030
1313
  // lastIndex between monitor cycles.
@@ -1032,10 +1315,15 @@ export class Daemon extends EventEmitter {
1032
1315
  const flags = pattern.flags.includes("g") ? pattern.flags : pattern.flags + "g";
1033
1316
  return (pane.match(new RegExp(pattern.source, flags)) || []).length;
1034
1317
  };
1318
+ // Same veto as the state machine: a backend whose ready marker is permanently
1319
+ // on screen would otherwise "recover" on the very first tick after the error,
1320
+ // rebaselining the occurrence count while the error is still displayed — one
1321
+ // notification, a false recovery log, then silence.
1322
+ const looksReady = () => !busyPattern?.test(pane) && readyPattern.test(pane);
1035
1323
  // State: waiting for recovery. A missing/outdated ready pattern must not
1036
1324
  // suppress every future error forever, so the gate has a hard deadline.
1037
1325
  if (this.errorWaitingForRecovery) {
1038
- if (readyPattern.test(pane)) {
1326
+ if (looksReady()) {
1039
1327
  const downtime = Math.round((now - this.errorDetectedAt) / 1000);
1040
1328
  const active = patterns.find(ep => Daemon.errorPatternKey(ep) === this.activeErrorPatternKey);
1041
1329
  if (active) {
@@ -1134,6 +1422,12 @@ export class Daemon extends EventEmitter {
1134
1422
  * Interrupt the CLI's current generation (cancel button / `/cancel`).
1135
1423
  * Direct tmux key event (not a paste) so it registers as the interrupt key.
1136
1424
  * kiro-cli interrupts on Ctrl+C; the others (claude-code, codex, …) on Escape.
1425
+ *
1426
+ * Deliberately NOT taken through `paneWriteLock`. Cancel is the user's way out
1427
+ * of a pane that is busy or wedged — queueing it behind the very delivery chain
1428
+ * it exists to unblock would make the button useless exactly when it is needed.
1429
+ * The cost is accepted: an Escape landing between a paste and its Enter discards
1430
+ * that message, which is what the user asked for anyway.
1137
1431
  */
1138
1432
  async sendEscape() {
1139
1433
  const cancelKey = this.backend?.getCancelKey() ?? "Escape";
@@ -1171,7 +1465,6 @@ export class Daemon extends EventEmitter {
1171
1465
  this.toolStatusDebounce = null;
1172
1466
  }
1173
1467
  this.pendingIpcRequests.clear();
1174
- this.hangDetector?.stop();
1175
1468
  if (this.adapter)
1176
1469
  await this.adapter.stop();
1177
1470
  // Notify MCP servers of graceful shutdown (prevents reconnect attempts)
@@ -1198,7 +1491,12 @@ export class Daemon extends EventEmitter {
1198
1491
  if (!killed)
1199
1492
  this.logger.warn("CLI did not exit gracefully within 3s, force killing window");
1200
1493
  // Always kill window — remain-on-exit keeps dead panes around after CLI exits
1494
+ const stoppedWindowId = this.tmux.getWindowId();
1201
1495
  await this.tmux.killWindow();
1496
+ // The control client is shared across the fleet and outlives this daemon, so
1497
+ // a registration left behind here is re-resolved on every reconnect forever.
1498
+ if (stoppedWindowId)
1499
+ this.controlClient?.unregisterWindow(stoppedWindowId);
1202
1500
  const windowIdFile = join(this.instanceDir, "window-id");
1203
1501
  try {
1204
1502
  unlinkSync(windowIdFile);
@@ -1241,6 +1539,16 @@ export class Daemon extends EventEmitter {
1241
1539
  };
1242
1540
  }
1243
1541
  /** Gracefully stop the CLI while keeping its remain-on-exit tmux window. */
1542
+ /**
1543
+ * Mark the instance to pause as soon as its pane is idle. Used for auth
1544
+ * failures: pause() alone no-ops while the CLI is busy/stuck, which is the
1545
+ * usual state when the error surfaces. Cleared by a successful pause or wake.
1546
+ */
1547
+ requestPauseWhenIdle() {
1548
+ if (this.pauseWakeState === "paused")
1549
+ return;
1550
+ this.pausePending = true;
1551
+ }
1244
1552
  async pause() {
1245
1553
  if (this.pauseWakeState === "paused")
1246
1554
  return;
@@ -1310,6 +1618,9 @@ export class Daemon extends EventEmitter {
1310
1618
  }
1311
1619
  /** Respawn the CLI in the preserved window and block until its prompt is ready. */
1312
1620
  async wake(timeoutMs = 30_000) {
1621
+ // An explicit wake (e.g. after the user re-logs in) cancels a deferred
1622
+ // auth pause — otherwise the instance would pause again the moment it idles.
1623
+ this.pausePending = false;
1313
1624
  if (this.pauseWakeState === "active")
1314
1625
  return;
1315
1626
  if (this.pauseWakeState === "pausing")
@@ -1321,7 +1632,7 @@ export class Daemon extends EventEmitter {
1321
1632
  return;
1322
1633
  }
1323
1634
  this.pauseWakeState = "waking";
1324
- this.spawning = true;
1635
+ this.beginSpawn();
1325
1636
  const transition = this.autoPauseController.wakeOnDeliver(async () => {
1326
1637
  let timeout;
1327
1638
  try {
@@ -1362,7 +1673,7 @@ export class Daemon extends EventEmitter {
1362
1673
  throw err;
1363
1674
  }
1364
1675
  finally {
1365
- this.spawning = false;
1676
+ this.endSpawn();
1366
1677
  if (this.pauseWakeTransition === transition)
1367
1678
  this.pauseWakeTransition = null;
1368
1679
  }
@@ -1374,6 +1685,10 @@ export class Daemon extends EventEmitter {
1374
1685
  // observations between enqueue and paste must not clear a newer inbound.
1375
1686
  if (snapshot.state === "idle" && previous !== "idle") {
1376
1687
  this.pendingWork.recordIdle(snapshot.observedAt);
1688
+ // The turn is over. A transcript can end on a tool_use with no matching
1689
+ // tool_result (interrupted, crashed, cancelled), which would otherwise leave
1690
+ // the last tool pinned to the progress line for the rest of the session.
1691
+ this.publishActivity(null);
1377
1692
  }
1378
1693
  if (snapshot.state !== previous) {
1379
1694
  this.logger.info({
@@ -1389,6 +1704,12 @@ export class Daemon extends EventEmitter {
1389
1704
  }
1390
1705
  if (snapshot.state !== "idle")
1391
1706
  this.pauseRequested = false;
1707
+ // A pause deferred by an auth failure: retry the moment the pane settles.
1708
+ if (this.pausePending && snapshot.state === "idle" && this.pasteQueueDepth === 0) {
1709
+ this.pausePending = false;
1710
+ this.emit("auto_pause_requested", { name: this.name, idleSince: snapshot.stateChangedAt });
1711
+ return;
1712
+ }
1392
1713
  if (!this.pauseRequested && this.pasteQueueDepth === 0 && this.autoPauseController.observe(snapshot.state)) {
1393
1714
  this.pauseRequested = true;
1394
1715
  this.emit("auto_pause_requested", { name: this.name, idleSince: snapshot.stateChangedAt });
@@ -1455,9 +1776,29 @@ export class Daemon extends EventEmitter {
1455
1776
  || (this.instanceStateLastOutputAt > 0 && this.instanceStateLastOutputAt >= captureStartedAt))
1456
1777
  return;
1457
1778
  const observedChangeAt = expectedOutputAt || captureStartedAt;
1458
- this.instanceStateMachine.observe(pane, observedChangeAt);
1459
- const snapshot = this.instanceStateMachine.observe(pane, Date.now());
1779
+ // One observation per capture. This used to call observe() twice with the
1780
+ // same pane: the second call always saw an unchanged pane, so the
1781
+ // "content moved → still working" branch could never fire and the state
1782
+ // came down to the busy/ready patterns alone. That is why a single
1783
+ // mismatched spinner frame was enough to report idle mid-turn.
1784
+ const settled = this.instanceStateLastOutputAt === 0
1785
+ || captureStartedAt - this.instanceStateLastOutputAt >= this.instanceStateIdleDebounceMs;
1786
+ // The two times are genuinely different and both matter: the content
1787
+ // changed when tmux reported output (observedChangeAt), but idle/stuck are
1788
+ // decisions about *now*. The old double-observe expressed that by calling
1789
+ // observe twice, which silently disabled the "content moved" branch.
1790
+ const snapshot = this.instanceStateMachine.observe(pane, Date.now(), {
1791
+ settled,
1792
+ changeAt: observedChangeAt,
1793
+ });
1460
1794
  this.applyInstanceStateSnapshot(snapshot, pane);
1795
+ // Backends without a transcript feed can still say what they are doing, if
1796
+ // their TUI names it. Free-riding on a capture that already happened: no
1797
+ // extra tmux call. Cadence is the capture cadence (idle debounce + the 60s
1798
+ // safety sweep), which matches the progress ticker's own 60s interval.
1799
+ if (this.backend?.getPaneActivity) {
1800
+ this.publishActivity(snapshot.state === "idle" ? null : this.backend.getPaneActivity(pane));
1801
+ }
1461
1802
  if (snapshot.state === "idle") {
1462
1803
  this.clearInstanceStateStuckTimer();
1463
1804
  }
@@ -1512,7 +1853,8 @@ export class Daemon extends EventEmitter {
1512
1853
  ? rawConfig.idle_debounce_ms
1513
1854
  : DEFAULT_STATE_IDLE_DEBOUNCE_MS;
1514
1855
  this.instanceStateReadyPattern = this.backend.getReadyPattern();
1515
- this.instanceStateMachine = new PaneStateMachine(this.instanceStateReadyPattern, this.instanceStateStuckTimeoutMs);
1856
+ this.instanceStateBusyPattern = this.backend.getBusyPattern?.() ?? null;
1857
+ this.instanceStateMachine = new PaneStateMachine(this.instanceStateReadyPattern, this.instanceStateStuckTimeoutMs, Date.now(), this.instanceStateBusyPattern);
1516
1858
  this.instanceStateLastOutputAt = 0;
1517
1859
  this.instanceStateMonitorActive = true;
1518
1860
  if (this.controlClient) {
@@ -1553,6 +1895,12 @@ export class Daemon extends EventEmitter {
1553
1895
  paneTail: sanitizePaneTail(pane),
1554
1896
  readyPattern: readyPattern.toString(),
1555
1897
  readyMatched: deterministicReadyPattern.test(pane),
1898
+ // A pane that is stuck *while* the ready marker matches means the busy
1899
+ // pattern is what held it back — worth seeing when tuning either regex.
1900
+ busyPattern: this.instanceStateBusyPattern?.toString(),
1901
+ busyMatched: this.instanceStateBusyPattern
1902
+ ? new RegExp(this.instanceStateBusyPattern.source, this.instanceStateBusyPattern.flags.replace(/[gy]/g, "")).test(pane)
1903
+ : undefined,
1556
1904
  unchangedForMs: snapshot.unchangedForMs,
1557
1905
  pendingWork: this.pendingWork.hasPendingWork(),
1558
1906
  };
@@ -1595,6 +1943,40 @@ export class Daemon extends EventEmitter {
1595
1943
  return this.messageBus;
1596
1944
  }
1597
1945
  // ── Tool status tracking ──────────────────────────────────────
1946
+ /**
1947
+ * Tell the fleet manager what this instance is doing right now, for the live
1948
+ * progress line on the cancel button.
1949
+ *
1950
+ * Purely cosmetic — nothing decides anything from it, so it is fine that only
1951
+ * backends with a transcript feed report at all (claude-code today). For the
1952
+ * rest the progress line keeps showing just elapsed time.
1953
+ *
1954
+ * Repeats are dropped: the ticker only edits the channel message when the text
1955
+ * changes, and a stream of identical broadcasts would defeat that.
1956
+ */
1957
+ publishActivity(activity) {
1958
+ const next = activity && activity.trim() ? activity.trim() : null;
1959
+ if (next === this.currentActivity)
1960
+ return;
1961
+ this.currentActivity = next;
1962
+ this.ipcServer?.broadcast({
1963
+ type: "instance_activity",
1964
+ instanceName: this.name,
1965
+ activity: next,
1966
+ });
1967
+ }
1968
+ /**
1969
+ * Options for every system-initiated paste (startup notice, session snapshot,
1970
+ * runtime-dialog keys).
1971
+ *
1972
+ * One place, so the three call sites cannot drift apart on it. Backends with a
1973
+ * native input queue do not get the retry Enter: on those, a second bare Enter
1974
+ * is not the no-op the retry assumed but a queue mutation — the same reason
1975
+ * `deliverMessage` refuses to probe for busy with one.
1976
+ */
1977
+ systemPasteOptions() {
1978
+ return { retryEnter: this.backend?.supportsQueuedInput?.() !== true };
1979
+ }
1598
1980
  summarizeTool(name, input) {
1599
1981
  const inp = input;
1600
1982
  if (!inp)
@@ -1684,7 +2066,6 @@ export class Daemon extends EventEmitter {
1684
2066
  writeFileSync(join(this.instanceDir, "prev-instructions"), this.pendingInstructionsUpdate);
1685
2067
  this.pendingInstructionsUpdate = undefined;
1686
2068
  }
1687
- this.hangDetector?.recordInbound();
1688
2069
  this.pendingWork.recordInbound();
1689
2070
  // v3: record user messages for rotation snapshot
1690
2071
  this.recordRecentUserMessage(content, meta);
@@ -1707,12 +2088,26 @@ export class Daemon extends EventEmitter {
1707
2088
  // #77: show the sender's display name for readability, keeping the machine
1708
2089
  // instance name in parens so the recipient's send_to_instance target is valid.
1709
2090
  const fromLabel = meta.from_display ? `${meta.from_display} (${fromInstance})` : fromInstance;
1710
- formatted = `[from:${fromLabel}] ${content}\n(If you need to reply, use send_to_instance tool, NOT direct text. If there is nothing to add, you may stay silent.)`;
2091
+ formatted = `[from:${fromLabel}] ${content}`;
2092
+ formatted += renderHandoffMetadata(meta);
2093
+ // A delegated task that requires a reply must not read like a chatty FYI —
2094
+ // the "you may stay silent" line is for the latter only.
2095
+ formatted += meta.requires_reply === "true"
2096
+ ? "\n(A reply IS required: use report_result with the correlation_id above — or send_to_instance. Not direct text.)"
2097
+ : "\n(If you need to reply, use send_to_instance tool, NOT direct text. If there is nothing to add, you may stay silent.)";
1711
2098
  }
1712
2099
  else {
1713
2100
  const via = meta.source ? ` via ${meta.source}` : "";
1714
2101
  const idTag = meta.user_id ? `, id:${meta.user_id}` : "";
1715
- formatted = `[user:${user}${via}${idTag}] ${content}\n(Reply using the reply tool — do NOT respond with direct text)`;
2102
+ formatted = `[user:${user}${via}${idTag}] ${content}`;
2103
+ // Reactions queued since the last real message (#432). One leading line of
2104
+ // context, present only when something is pending — a reaction no longer
2105
+ // costs a turn of its own.
2106
+ if (meta.pending_reactions) {
2107
+ formatted = `[Recent reactions: ${meta.pending_reactions}]\n${formatted}`;
2108
+ }
2109
+ formatted += renderHandoffMetadata(meta);
2110
+ formatted += "\n(Reply using the reply tool — do NOT respond with direct text)";
1716
2111
  }
1717
2112
  if (meta.reply_to_text) {
1718
2113
  formatted += `\n(reply_to: "${meta.reply_to_text}")`;
@@ -1755,18 +2150,19 @@ export class Daemon extends EventEmitter {
1755
2150
  }
1756
2151
  /**
1757
2152
  * Deliver a single message and drive its status reactions:
1758
- * ⏳ message_queued — CLI busy; queued, waiting for idle
2153
+ * ⏳ message_queued — CLI busy; queued locally or by the backend
1759
2154
  * 👀 message_delivered — pasted + Enter sent; agent now has it
1760
- * ✅ message_confirmed — idle→busy transition observed; agent is processing
2155
+ * ✅ message_confirmed — processing observed, or native queue handoff succeeded
1761
2156
  * ❌ message_failed — tmux window gone, paste retries exhausted
1762
2157
  * Returns true once the message is in the CLI, false only on real delivery failure.
1763
2158
  *
1764
2159
  * Bug A (silent message loss): paste failures retry with backoff (window recovery)
1765
2160
  * and emit `message_failed` if all attempts fail.
1766
- * Busy handling (UX): we never force-paste into a busy CLI and never give up on a
1767
- * busy one — we show ⏳ and wait for idle indefinitely (a genuinely hung CLI is the
1768
- * hang detector's job; a user who can't wait presses Cancel → Escape → idle). The
1769
- * pasteLock is serial, so later messages naturally queue behind this wait.
2161
+ * Busy handling (UX): backends without a native input queue show and wait for
2162
+ * idle indefinitely (a genuinely hung CLI is the hang detector's job). Backends
2163
+ * that explicitly support queued input receive one complete paste+Enter transaction
2164
+ * immediately and own the application-level queue themselves. The pasteLock remains
2165
+ * serial in both cases so separate PTY writes can never overlap.
1770
2166
  */
1771
2167
  async deliverMessage(formatted, status) {
1772
2168
  // Sanitize unclosed code fences — they cause CLI to wait for closure on Enter
@@ -1775,14 +2171,50 @@ export class Daemon extends EventEmitter {
1775
2171
  // Odd number of fences = unclosed. Remove all code fences from the message.
1776
2172
  formatted = formatted.replace(/```/g, "");
1777
2173
  }
2174
+ // Before anything reads the window id: a spawn in progress is about to change it.
2175
+ await this.waitForSpawnToSettle();
1778
2176
  let windowId = this.getWindowId();
1779
- // If the CLI is busy, show ⏳ and wait for it to go idle — no timeout, no force.
2177
+ const supportsQueuedInput = this.backend?.supportsQueuedInput?.() === true;
2178
+ let handingOffToNativeQueue = false;
2179
+ // If the CLI is busy, either hand the complete submission to its native input
2180
+ // queue or wait for idle. Native queue support is an explicit backend capability:
2181
+ // normal Enter input has steering/interrupt semantics in several other CLIs.
1780
2182
  if (windowId && this.controlClient && !this.controlClient.isIdle(windowId)) {
1781
2183
  if (status)
1782
2184
  this.emit("message_queued", status);
1783
- this.logger.debug("CLI busy — queuing message until idle");
1784
- await this.controlClient.waitUntilIdle(windowId);
2185
+ if (supportsQueuedInput) {
2186
+ handingOffToNativeQueue = true;
2187
+ this.logger.debug("CLI busy — handing message to backend-native input queue");
2188
+ }
2189
+ else {
2190
+ this.logger.debug("CLI busy — queuing message until idle");
2191
+ const becameIdle = await this.controlClient.waitUntilIdle(windowId);
2192
+ if (!becameIdle) {
2193
+ // The pane never freed up. Report the failure instead of pasting into a
2194
+ // wedged CLI (where the text would sit unsubmitted and the next message
2195
+ // would land on top of it) — and instead of holding the queue silently.
2196
+ this.logger.error("Pane still busy after the idle wait — reporting delivery failure");
2197
+ if (status)
2198
+ this.emit("message_failed", status); // ❌
2199
+ return false;
2200
+ }
2201
+ }
1785
2202
  }
2203
+ // Everything above is *waiting*; everything below *writes*. Only the write is
2204
+ // held under the pane lock — holding it across the idle wait (up to 30 min)
2205
+ // would starve the runtime-dialog dismisser, which is often the very thing
2206
+ // that would let the pane go idle again.
2207
+ return this.paneWriteLock.run(() => this.writeMessageToPane(formatted, windowId, handingOffToNativeQueue, status));
2208
+ }
2209
+ /**
2210
+ * The write half of a delivery: paste → Enter → confirm, with retries.
2211
+ *
2212
+ * Always called under {@link paneWriteLock}. Split out from `deliverMessage`
2213
+ * precisely so the lock's scope is visible at the call site rather than being
2214
+ * an invariant maintained by comments.
2215
+ */
2216
+ async writeMessageToPane(formatted, initialWindowId, handingOffToNativeQueue, status) {
2217
+ let windowId = initialWindowId;
1786
2218
  // Bug A: paste with backoff. Transient failures are usually a stale window id
1787
2219
  // after a crash/respawn — recover by name and retry (max 3 attempts, 2s apart).
1788
2220
  const maxAttempts = 3;
@@ -1807,9 +2239,56 @@ export class Daemon extends EventEmitter {
1807
2239
  await this.tmux.sendSpecialKey("Enter");
1808
2240
  if (status)
1809
2241
  this.emit("message_delivered", status); // 👀
1810
- // Confirm the CLI accepted the message by transitioning idle→busy (new output
1811
- // after Enter). If still idle after ~2s the Enter was likely swallowed while
1812
- // the TUI was redrawing re-send Enter once and re-check.
2242
+ // Busy queue-capable CLIs (codex) may accept paste without an idle→busy
2243
+ // transition. Do NOT probe for busy a second bare Enter can mutate the
2244
+ // queue. Instead verify the paste actually landed in the pane (TUI redraw
2245
+ // can silently swallow it). Idle submissions keep the swallowed-Enter path.
2246
+ if (handingOffToNativeQueue) {
2247
+ await new Promise(r => setTimeout(r, NATIVE_QUEUE_PASTE_VERIFY_MS));
2248
+ if (await this.nativeQueuePasteVisible(formatted)) {
2249
+ if (status)
2250
+ this.emit("message_confirmed", status); // ✅ native queue accepted
2251
+ return true;
2252
+ }
2253
+ // Silent loss: fall back once to the normal idle-gated path.
2254
+ this.logger.warn("Native-queue paste not visible in pane — retrying via idle-gated delivery");
2255
+ if (windowId && this.controlClient) {
2256
+ await this.controlClient.waitUntilIdle(windowId);
2257
+ }
2258
+ const repasted = await this.tmux.pasteBuffer(formatted);
2259
+ if (!repasted) {
2260
+ this.logger.error("Idle-gated redelivery paste failed after native-queue silent loss");
2261
+ if (status)
2262
+ this.emit("message_failed", status); // ❌
2263
+ return false;
2264
+ }
2265
+ await new Promise(r => setTimeout(r, NORMAL_ENTER_SETTLE_MS));
2266
+ const retryAt = Date.now();
2267
+ await this.tmux.sendSpecialKey("Enter");
2268
+ if (windowId && this.controlClient) {
2269
+ let becameBusy = await this.confirmBusyAfterEnter(windowId, retryAt);
2270
+ if (!becameBusy) {
2271
+ this.logger.warn("No idle→busy after idle-gated redelivery — re-sending Enter once");
2272
+ const retry2At = Date.now();
2273
+ await this.tmux.sendSpecialKey("Enter");
2274
+ becameBusy = await this.confirmBusyAfterEnter(windowId, retry2At);
2275
+ }
2276
+ if (becameBusy) {
2277
+ if (status)
2278
+ this.emit("message_confirmed", status); // ✅
2279
+ return true;
2280
+ }
2281
+ }
2282
+ else if (await this.nativeQueuePasteVisible(formatted)) {
2283
+ if (status)
2284
+ this.emit("message_confirmed", status); // ✅
2285
+ return true;
2286
+ }
2287
+ this.logger.error("Idle-gated redelivery also failed after native-queue silent loss");
2288
+ if (status)
2289
+ this.emit("message_failed", status); // ❌
2290
+ return false;
2291
+ }
1813
2292
  if (windowId && this.controlClient) {
1814
2293
  let becameBusy = await this.confirmBusyAfterEnter(windowId, enterAt);
1815
2294
  if (!becameBusy) {
@@ -1818,8 +2297,20 @@ export class Daemon extends EventEmitter {
1818
2297
  await this.tmux.sendSpecialKey("Enter");
1819
2298
  becameBusy = await this.confirmBusyAfterEnter(windowId, retryAt);
1820
2299
  }
1821
- if (becameBusy && status)
1822
- this.emit("message_confirmed", status); // ✅
2300
+ if (becameBusy) {
2301
+ if (status)
2302
+ this.emit("message_confirmed", status); // ✅
2303
+ }
2304
+ else {
2305
+ // Both Enters were swallowed: the text is sitting UNSUBMITTED in the
2306
+ // CLI's input box. This used to return true, so the reaction stayed at 👀
2307
+ // forever and the next delivery pasted on top — submitting two messages
2308
+ // as one. Say so instead.
2309
+ this.logger.error("Message pasted but never submitted (no idle→busy after two Enters)");
2310
+ if (status)
2311
+ this.emit("message_failed", status); // ❌
2312
+ return false;
2313
+ }
1823
2314
  }
1824
2315
  else {
1825
2316
  // No control client to observe output: fall back to the legacy double-Enter.
@@ -1835,8 +2326,36 @@ export class Daemon extends EventEmitter {
1835
2326
  this.emit("message_failed", status); // ❌
1836
2327
  return false;
1837
2328
  }
2329
+ /**
2330
+ * True when a busy native-queue paste appears to have landed: Codex shows a
2331
+ * `↳` queue marker, or a distinctive slice of the pasted text is on screen.
2332
+ * Used only to detect silent paste loss — not as a general ready check.
2333
+ */
2334
+ async nativeQueuePasteVisible(formatted) {
2335
+ if (!this.tmux)
2336
+ return false;
2337
+ try {
2338
+ const pane = await this.tmux.capturePane();
2339
+ if (pane.includes("↳"))
2340
+ return true;
2341
+ for (const line of formatted.split(/\r?\n/)) {
2342
+ const t = line.trim();
2343
+ if (t.length >= 8 && pane.includes(t))
2344
+ return true;
2345
+ }
2346
+ const compact = formatted.replace(/\s+/g, " ").trim();
2347
+ if (compact.length >= 8 && pane.includes(compact.slice(0, Math.min(80, compact.length)))) {
2348
+ return true;
2349
+ }
2350
+ return false;
2351
+ }
2352
+ catch {
2353
+ return false;
2354
+ }
2355
+ }
1838
2356
  /** Re-resolve this instance's tmux window by name (stale id after crash/respawn). */
1839
2357
  async recoverWindow() {
2358
+ const previousWindowId = this.tmux?.getWindowId();
1840
2359
  try {
1841
2360
  const windows = await TmuxManager.listWindows(this.tmuxSessionName);
1842
2361
  const match = windows.find(w => w.name === this.name);
@@ -1844,6 +2363,12 @@ export class Daemon extends EventEmitter {
1844
2363
  return undefined;
1845
2364
  this.tmux = new TmuxManager(this.tmuxSessionName, match.id, resolveTmuxLogicalSize(this.config.terminal));
1846
2365
  writeFileSync(join(this.instanceDir, "window-id"), match.id);
2366
+ // The window we were talking to is gone; leaving it registered means the
2367
+ // control client re-resolves a dead id — one tmux subprocess — on every
2368
+ // reconnect, for the life of the fleet process.
2369
+ if (previousWindowId && previousWindowId !== match.id) {
2370
+ this.controlClient?.unregisterWindow(previousWindowId);
2371
+ }
1847
2372
  await this.controlClient?.registerWindow(match.id);
1848
2373
  this.bindInstanceStateOutputListener(match.id);
1849
2374
  this.logger.info({ windowId: match.id }, "Recovered window ID for message delivery");
@@ -1854,12 +2379,38 @@ export class Daemon extends EventEmitter {
1854
2379
  return undefined;
1855
2380
  }
1856
2381
  }
1857
- /** Poll up to ~2s (200ms × 10) for the pane to emit output after `since`. */
2382
+ /**
2383
+ * Poll up to ~2s (200ms × 10) for the pane to emit output after `since`.
2384
+ *
2385
+ * A `false` here is not "no output" — it is read as **"the message was pasted but
2386
+ * never submitted"**, which re-sends Enter and then reports ❌ to the user. So a
2387
+ * false negative costs a possible double submit and a failure notice for a
2388
+ * message that actually arrived.
2389
+ *
2390
+ * That is exactly what a control-mode reconnect used to produce. `connect()`
2391
+ * drops every output timestamp, so `hasOutputSince` answers `false` for a pane
2392
+ * that did react — the evidence was thrown away, not absent. Blind time is
2393
+ * therefore not counted against the budget: after a reset the poll restarts from
2394
+ * the moment observation resumed. A hard wall-clock cap keeps a reconnect loop
2395
+ * from extending this forever.
2396
+ */
1858
2397
  async confirmBusyAfterEnter(windowId, since) {
1859
- for (let i = 0; i < 10; i++) {
1860
- await new Promise(r => setTimeout(r, 200));
1861
- if (this.controlClient.hasOutputSince(windowId, since))
2398
+ const client = this.controlClient;
2399
+ const hardDeadline = Date.now() + CONFIRM_BUSY_MAX_WAIT_MS;
2400
+ let observedFrom = since;
2401
+ let polls = 0;
2402
+ while (polls < CONFIRM_BUSY_POLLS) {
2403
+ await new Promise(r => setTimeout(r, CONFIRM_BUSY_POLL_MS));
2404
+ if (client.hasOutputSince(windowId, observedFrom))
1862
2405
  return true;
2406
+ const resetAt = client.getObservationResetAt();
2407
+ if (resetAt > observedFrom && Date.now() < hardDeadline) {
2408
+ this.logger.debug({ windowId, resetAt }, "Control mode reconnected mid-confirmation — restarting the Enter check from when we could see again");
2409
+ observedFrom = resetAt;
2410
+ polls = 0;
2411
+ continue;
2412
+ }
2413
+ polls++;
1863
2414
  }
1864
2415
  return false;
1865
2416
  }
@@ -1930,8 +2481,8 @@ export class Daemon extends EventEmitter {
1930
2481
  });
1931
2482
  const timeout = setTimeout(() => {
1932
2483
  this.pendingIpcRequests.delete(fleetReqId);
1933
- respond(null, "Task operation timed out after 30s");
1934
- }, 30_000);
2484
+ respond(null, `Task operation timed out after ${daemonBudgetMs(tool) / 1000}s`);
2485
+ }, daemonBudgetMs(tool));
1935
2486
  this.pendingIpcRequests.set(fleetReqId, (respMsg) => {
1936
2487
  clearTimeout(timeout);
1937
2488
  respond(respMsg.result, respMsg.error);
@@ -1953,8 +2504,8 @@ export class Daemon extends EventEmitter {
1953
2504
  });
1954
2505
  const timeout = setTimeout(() => {
1955
2506
  this.pendingIpcRequests.delete(fleetReqId);
1956
- respond(null, "Decision operation timed out after 30s");
1957
- }, 30_000);
2507
+ respond(null, `Decision operation timed out after ${daemonBudgetMs(tool) / 1000}s`);
2508
+ }, daemonBudgetMs(tool));
1958
2509
  this.pendingIpcRequests.set(fleetReqId, (respMsg) => {
1959
2510
  clearTimeout(timeout);
1960
2511
  respond(respMsg.result, respMsg.error);
@@ -1980,8 +2531,8 @@ export class Daemon extends EventEmitter {
1980
2531
  // Wait for fleet_schedule_response via pending request map
1981
2532
  const timeout = setTimeout(() => {
1982
2533
  this.pendingIpcRequests.delete(fleetReqId);
1983
- respond(null, "Schedule operation timed out after 30s");
1984
- }, 30_000);
2534
+ respond(null, `Schedule operation timed out after ${daemonBudgetMs(tool) / 1000}s`);
2535
+ }, daemonBudgetMs(tool));
1985
2536
  this.pendingIpcRequests.set(fleetReqId, (respMsg) => {
1986
2537
  clearTimeout(timeout);
1987
2538
  respond(respMsg.result, respMsg.error);
@@ -2002,7 +2553,9 @@ export class Daemon extends EventEmitter {
2002
2553
  fleetRequestId: fleetReqId,
2003
2554
  senderSessionName,
2004
2555
  });
2005
- const crossTimeoutMs = (tool === "start_instance" || tool === "create_instance" || tool === "replace_instance") ? 60_000 : 30_000;
2556
+ // Shared table, so delete_instance and the deployment tools get the same
2557
+ // budget here as the MCP client expects (the ad-hoc list omitted them).
2558
+ const crossTimeoutMs = daemonBudgetMs(tool);
2006
2559
  const timeout = setTimeout(() => {
2007
2560
  this.pendingIpcRequests.delete(fleetReqId);
2008
2561
  respond(null, `Cross-instance operation timed out after ${crossTimeoutMs / 1000}s`);
@@ -2045,8 +2598,8 @@ export class Daemon extends EventEmitter {
2045
2598
  this.ipcServer?.broadcast({ type: "fleet_outbound", tool, args, fleetRequestId: fleetReqId });
2046
2599
  const timeout = setTimeout(() => {
2047
2600
  this.pendingIpcRequests.delete(outboundKey);
2048
- respond(null, "Fleet outbound timed out after 30s");
2049
- }, 30_000);
2601
+ respond(null, `Fleet outbound timed out after ${daemonBudgetMs(tool) / 1000}s`);
2602
+ }, daemonBudgetMs(tool));
2050
2603
  this.pendingIpcRequests.set(outboundKey, (respMsg) => {
2051
2604
  clearTimeout(timeout);
2052
2605
  respond(respMsg.result, respMsg.error);
@@ -2110,7 +2663,14 @@ export class Daemon extends EventEmitter {
2110
2663
  try {
2111
2664
  const all = JSON.parse(process.env.AGEND_DECISIONS);
2112
2665
  const workDir = this.config.working_directory;
2113
- decisions = all.filter(d => d.scope === "fleet" || d.project_root === workDir);
2666
+ // `scope: "fleet"` used to bypass the project check entirely, so a
2667
+ // single-project worker inherited every other project's playbook (8 of
2668
+ // 14 injected here were foreign) — resent on every API call. Now a
2669
+ // fleet decision must also be RELEVANT: global (no project_root), the
2670
+ // same project, or the same project family (worktrees/checkouts of one
2671
+ // repo). A general is exempt: it dispatches across all projects, so
2672
+ // cross-project routing rules are exactly what it needs.
2673
+ decisions = selectRelevantDecisions(all, workDir, this.config.general_topic === true);
2114
2674
  // Stable ordering so identical decision sets always build byte-identical
2115
2675
  // instructions — otherwise source ordering jitter flips the warmup hash.
2116
2676
  decisions.sort((a, b) => a.title.localeCompare(b.title));
@@ -2124,9 +2684,18 @@ export class Daemon extends EventEmitter {
2124
2684
  AGEND_SOCKET_PATH: sockPath,
2125
2685
  AGEND_INSTANCE_NAME: this.name,
2126
2686
  AGEND_WORKING_DIR: this.config.working_directory,
2687
+ AGEND_INSTANCE_KIND: this.runtimeIdentity?.kind ?? "fleet-topic",
2688
+ AGEND_BACKEND: this.runtimeIdentity?.backend ?? this.config.backend ?? this.backend?.binaryName ?? "unknown",
2689
+ AGEND_MODEL: this.runtimeIdentity?.model ?? this.config.model ?? "default",
2127
2690
  };
2128
- if (this.config.tool_set)
2129
- mcpEnv.AGEND_TOOL_SET = this.config.tool_set;
2691
+ // A general is a dispatcher, not a worker: default it to the `general` tool
2692
+ // profile instead of `full`. Every tool's schema is resent on every API call,
2693
+ // so full (44 tools) costs a general ~22k chars per turn — the single largest
2694
+ // slice of its prompt. Explicit config still wins.
2695
+ const defaultToolSet = this.config.general_topic ? "general" : undefined;
2696
+ const toolSet = this.config.tool_set ?? defaultToolSet;
2697
+ if (toolSet)
2698
+ mcpEnv.AGEND_TOOL_SET = toolSet;
2130
2699
  if (this.config.display_name)
2131
2700
  mcpEnv.AGEND_DISPLAY_NAME = this.config.display_name;
2132
2701
  if (this.config.description)
@@ -2159,6 +2728,7 @@ export class Daemon extends EventEmitter {
2159
2728
  instructions = buildFleetInstructions({
2160
2729
  instanceName: this.name,
2161
2730
  workingDirectory: this.config.working_directory,
2731
+ runtimeIdentity: this.runtimeIdentity,
2162
2732
  displayName: this.config.display_name,
2163
2733
  description: this.config.description,
2164
2734
  customPrompt: resolvedCustomPrompt,
@@ -2171,6 +2741,7 @@ export class Daemon extends EventEmitter {
2171
2741
  instructions = buildFleetInstructions({
2172
2742
  instanceName: this.name,
2173
2743
  workingDirectory: this.config.working_directory,
2744
+ runtimeIdentity: this.runtimeIdentity,
2174
2745
  displayName: this.config.display_name,
2175
2746
  description: this.config.description,
2176
2747
  customPrompt: resolvedCustomPrompt,
@@ -2192,6 +2763,7 @@ export class Daemon extends EventEmitter {
2192
2763
  },
2193
2764
  skipPermissions: this.config.skipPermissions,
2194
2765
  model: this.modelOverride ?? this.config.model,
2766
+ effort: this.config.effort,
2195
2767
  kiroUi: this.config.kiro_ui,
2196
2768
  skipResume: this.skipResume,
2197
2769
  instructions,
@@ -2217,7 +2789,9 @@ export class Daemon extends EventEmitter {
2217
2789
  // Small delay to let the CLI fully render its ready prompt
2218
2790
  await new Promise(r => setTimeout(r, 1_000));
2219
2791
  try {
2220
- await this.tmux.pasteText(`[system:session-snapshot]\n${snapshot}\n\nThis is a background context restore do NOT reply to or acknowledge this message. Simply resume normal operation when the next user or instance message arrives.`);
2792
+ // Messages can arrive during a restart and be queued on pasteLock before the
2793
+ // snapshot lands; both write to the pane, so both go through the same lock.
2794
+ await this.paneWriteLock.run(() => this.tmux.pasteText(`[system:session-snapshot]\n${snapshot}\n\nThis is a background context restore — do NOT reply to or acknowledge this message. Simply resume normal operation when the next user or instance message arrives.`, this.systemPasteOptions()));
2221
2795
  this.logger.info("Injected session snapshot as first message");
2222
2796
  this.emit("snapshot_injected", this.name);
2223
2797
  }
@@ -2226,9 +2800,69 @@ export class Daemon extends EventEmitter {
2226
2800
  this.emit("snapshot_failed", this.name);
2227
2801
  }
2228
2802
  }
2803
+ /**
2804
+ * Depth, not a flag: the wake path marks a spawn and then calls trySpawn, which
2805
+ * marks another. With a plain boolean the inner one's completion would clear the
2806
+ * latch while the outer spawn was still dismissing dialogs, releasing delivery
2807
+ * into exactly the window this exists to close.
2808
+ */
2809
+ beginSpawn() {
2810
+ this.spawnDepth++;
2811
+ this.spawning = true;
2812
+ if (!this.spawnSettled) {
2813
+ this.spawnSettled = new Promise(resolve => { this.resolveSpawnSettled = resolve; });
2814
+ }
2815
+ }
2816
+ endSpawn() {
2817
+ this.spawnDepth = Math.max(0, this.spawnDepth - 1);
2818
+ if (this.spawnDepth > 0)
2819
+ return; // an enclosing spawn is still running
2820
+ this.spawning = false;
2821
+ const resolve = this.resolveSpawnSettled;
2822
+ this.spawnSettled = null;
2823
+ this.resolveSpawnSettled = null;
2824
+ resolve?.();
2825
+ }
2826
+ /**
2827
+ * Hold until the CLI has finished starting up.
2828
+ *
2829
+ * Startup is not a quiet period: `dismissDialogsUntilReady` is clicking through
2830
+ * trust prompts and session pickers. A pane showing a modal dialog produces no
2831
+ * output, so `waitUntilIdle` reports it idle and a queued message gets pasted
2832
+ * *into the dialog* — where the text is discarded and the Enter picks a menu
2833
+ * item. The paste and the Enter both "succeed", so this loses the message
2834
+ * without even a ❌.
2835
+ *
2836
+ * The pane write lock (#414) does not cover this: it serialises each key
2837
+ * sequence, but it is released between one dialog and the next.
2838
+ *
2839
+ * Bounded, and called BEFORE the pane lock is taken — waiting on the spawn while
2840
+ * holding the lock the spawn itself needs would deadlock.
2841
+ */
2842
+ async waitForSpawnToSettle() {
2843
+ const settled = this.spawnSettled;
2844
+ if (!settled)
2845
+ return;
2846
+ this.logger.debug("Holding delivery until the CLI has finished starting up");
2847
+ let timer;
2848
+ const cap = new Promise(resolve => {
2849
+ timer = setTimeout(resolve, SPAWN_SETTLE_MAX_WAIT_MS);
2850
+ timer.unref?.();
2851
+ });
2852
+ try {
2853
+ await Promise.race([settled, cap]);
2854
+ }
2855
+ finally {
2856
+ if (timer)
2857
+ clearTimeout(timer);
2858
+ }
2859
+ if (this.spawning) {
2860
+ this.logger.warn("CLI still starting after the delivery hold — delivering anyway");
2861
+ }
2862
+ }
2229
2863
  /** Spawn a CLI window. Returns true if --resume was used successfully. */
2230
2864
  async spawnClaudeWindow() {
2231
- this.spawning = true;
2865
+ this.beginSpawn();
2232
2866
  let resumedSuccessfully = false;
2233
2867
  try {
2234
2868
  this.toolStatusLines = [];
@@ -2265,7 +2899,7 @@ export class Daemon extends EventEmitter {
2265
2899
  this.backgroundSessionRecoveryAttempted = false;
2266
2900
  }
2267
2901
  finally {
2268
- this.spawning = false;
2902
+ this.endSpawn();
2269
2903
  }
2270
2904
  return resumedSuccessfully;
2271
2905
  }
@@ -2336,11 +2970,13 @@ export class Daemon extends EventEmitter {
2336
2970
  // the global web token) from impersonating instances.
2337
2971
  const agentTokenPath = join(this.instanceDir, "agent.token");
2338
2972
  const agentToken = randomBytes(32).toString("hex");
2339
- writeFileSync(agentTokenPath, agentToken, { mode: 0o600 });
2340
- try {
2341
- chmodSync(agentTokenPath, 0o600);
2973
+ const tokenWrite = writeSecretFile(agentTokenPath, agentToken);
2974
+ if (!tokenWrite.ok) {
2975
+ // Do not fail the spawn over it — an instance that cannot start is worse
2976
+ // than one whose token is readable. But say so, loudly: the whole point of
2977
+ // this token is that other local processes cannot use it.
2978
+ this.logger.error({ path: agentTokenPath, mode: tokenWrite.mode?.toString(8), reason: tokenWrite.reason }, "Agent token file is not owner-only — other local users can impersonate this instance");
2342
2979
  }
2343
- catch { }
2344
2980
  // AGEND_HOME points the child's agent-cli at the same data dir the daemon
2345
2981
  // is using, so it can locate <instanceDir>/agent.token.
2346
2982
  const agendHome = join(this.instanceDir, "..", "..");
@@ -2358,7 +2994,12 @@ export class Daemon extends EventEmitter {
2358
2994
  windowId = this.tmux.getWindowId();
2359
2995
  }
2360
2996
  else {
2997
+ // A crash respawn makes a brand new window. The dead one stays registered
2998
+ // unless we say so — the respawn branch above already does.
2999
+ const retired = this.tmux.getWindowId();
2361
3000
  windowId = await this.tmux.createWindow(cmd, resolvedCwd, this.name);
3001
+ if (retired && retired !== windowId)
3002
+ this.controlClient?.unregisterWindow(retired);
2362
3003
  }
2363
3004
  writeFileSync(join(this.instanceDir, "window-id"), windowId);
2364
3005
  // Enable remain-on-exit to capture exit codes on crash
@@ -2366,7 +3007,9 @@ export class Daemon extends EventEmitter {
2366
3007
  this.logger.warn({ err }, "Failed to set remain-on-exit — exit codes will not be captured");
2367
3008
  });
2368
3009
  if (reuseWindow && !this.config.lightweight) {
2369
- await this.tmux.pipeOutput(join(this.instanceDir, "output.log")).catch(err => {
3010
+ const outputLog = join(this.instanceDir, "output.log");
3011
+ rotateLogIfNeeded(outputLog);
3012
+ await this.tmux.pipeOutput(outputLog).catch(err => {
2370
3013
  this.logger.warn({ err }, "Failed to restore pipe-pane after wake");
2371
3014
  });
2372
3015
  }
@@ -2422,15 +3065,20 @@ export class Daemon extends EventEmitter {
2422
3065
  for (const dialog of startupDialogs) {
2423
3066
  if (dialog.pattern.test(pane)) {
2424
3067
  this.logger.debug(`Dismissing startup dialog: ${dialog.description}`);
2425
- for (const key of dialog.keys) {
2426
- if (key === "Up" || key === "Down" || key === "Enter" || key === "Escape") {
2427
- await this.tmux.sendSpecialKey(key);
3068
+ // Restart is exactly when inbound messages pile up, and nothing gates
3069
+ // delivery on `spawning`. Take the pane lock for the key sequence so a
3070
+ // queued message cannot be pasted into a half-dismissed trust dialog.
3071
+ await this.paneWriteLock.run(async () => {
3072
+ for (const key of dialog.keys) {
3073
+ if (key === "Up" || key === "Down" || key === "Enter" || key === "Escape") {
3074
+ await this.tmux.sendSpecialKey(key);
3075
+ }
3076
+ else {
3077
+ await this.tmux.sendKeys(key);
3078
+ }
3079
+ await new Promise(r => setTimeout(r, 200));
2428
3080
  }
2429
- else {
2430
- await this.tmux.sendKeys(key);
2431
- }
2432
- await new Promise(r => setTimeout(r, 200));
2433
- }
3081
+ });
2434
3082
  // Wait for next screen to render
2435
3083
  if (this.controlClient) {
2436
3084
  const wid = readFileSync(join(this.instanceDir, "window-id"), "utf-8").trim();