@songsid/agend 2.1.4-beta.6 → 2.1.4-beta.61

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (185) hide show
  1. package/README.md +1 -1
  2. package/README.zh-TW.md +1 -1
  3. package/dist/agent-cli-instructions.md +2 -2
  4. package/dist/agent-endpoint.js +1 -1
  5. package/dist/agent-endpoint.js.map +1 -1
  6. package/dist/backend/antigravity.d.ts +12 -8
  7. package/dist/backend/antigravity.js +189 -42
  8. package/dist/backend/antigravity.js.map +1 -1
  9. package/dist/backend/claude-code.d.ts +87 -23
  10. package/dist/backend/claude-code.js +361 -46
  11. package/dist/backend/claude-code.js.map +1 -1
  12. package/dist/backend/codex.d.ts +18 -0
  13. package/dist/backend/codex.js +143 -7
  14. package/dist/backend/codex.js.map +1 -1
  15. package/dist/backend/gemini-cli.js +2 -2
  16. package/dist/backend/gemini-cli.js.map +1 -1
  17. package/dist/backend/grok.js +2 -2
  18. package/dist/backend/grok.js.map +1 -1
  19. package/dist/backend/kiro.d.ts +52 -2
  20. package/dist/backend/kiro.js +267 -12
  21. package/dist/backend/kiro.js.map +1 -1
  22. package/dist/backend/opencode.js +17 -4
  23. package/dist/backend/opencode.js.map +1 -1
  24. package/dist/backend/types.d.ts +122 -3
  25. package/dist/backend/types.js +5 -1
  26. package/dist/backend/types.js.map +1 -1
  27. package/dist/backend-outage.d.ts +61 -0
  28. package/dist/backend-outage.js +71 -0
  29. package/dist/backend-outage.js.map +1 -0
  30. package/dist/channel/adapters/discord.d.ts +44 -2
  31. package/dist/channel/adapters/discord.js +529 -89
  32. package/dist/channel/adapters/discord.js.map +1 -1
  33. package/dist/channel/adapters/telegram.d.ts +30 -0
  34. package/dist/channel/adapters/telegram.js +221 -19
  35. package/dist/channel/adapters/telegram.js.map +1 -1
  36. package/dist/channel/agy-mcp-launcher.d.ts +2 -0
  37. package/dist/channel/agy-mcp-launcher.js +28 -0
  38. package/dist/channel/agy-mcp-launcher.js.map +1 -0
  39. package/dist/channel/ipc-bridge.d.ts +2 -1
  40. package/dist/channel/ipc-bridge.js +33 -4
  41. package/dist/channel/ipc-bridge.js.map +1 -1
  42. package/dist/channel/markdown-chunk.d.ts +42 -0
  43. package/dist/channel/markdown-chunk.js +205 -0
  44. package/dist/channel/markdown-chunk.js.map +1 -0
  45. package/dist/channel/mcp-server.js +28 -25
  46. package/dist/channel/mcp-server.js.map +1 -1
  47. package/dist/channel/mcp-tools.js +6 -4
  48. package/dist/channel/mcp-tools.js.map +1 -1
  49. package/dist/channel/message-queue.js +64 -8
  50. package/dist/channel/message-queue.js.map +1 -1
  51. package/dist/channel/types.d.ts +31 -1
  52. package/dist/classic-channel-manager.d.ts +65 -0
  53. package/dist/classic-channel-manager.js +172 -3
  54. package/dist/classic-channel-manager.js.map +1 -1
  55. package/dist/cli.js +124 -83
  56. package/dist/cli.js.map +1 -1
  57. package/dist/config-validator.js +16 -0
  58. package/dist/config-validator.js.map +1 -1
  59. package/dist/config.d.ts +4 -0
  60. package/dist/config.js +12 -1
  61. package/dist/config.js.map +1 -1
  62. package/dist/cross-instance-envelope.d.ts +6 -0
  63. package/dist/cross-instance-envelope.js +30 -0
  64. package/dist/cross-instance-envelope.js.map +1 -0
  65. package/dist/daemon.d.ts +506 -11
  66. package/dist/daemon.js +2162 -313
  67. package/dist/daemon.js.map +1 -1
  68. package/dist/doctor.d.ts +41 -0
  69. package/dist/doctor.js +267 -0
  70. package/dist/doctor.js.map +1 -0
  71. package/dist/fleet-context.d.ts +63 -0
  72. package/dist/fleet-context.js.map +1 -1
  73. package/dist/fleet-manager.d.ts +462 -12
  74. package/dist/fleet-manager.js +3027 -634
  75. package/dist/fleet-manager.js.map +1 -1
  76. package/dist/fleet-yaml-slim.d.ts +10 -0
  77. package/dist/fleet-yaml-slim.js +59 -0
  78. package/dist/fleet-yaml-slim.js.map +1 -0
  79. package/dist/general-knowledge/skills/backend-providers/SKILL.md +114 -0
  80. package/dist/general-knowledge/skills/cross-instance-messaging/SKILL.md +4 -1
  81. package/dist/general-knowledge/skills/delegation-playbook/SKILL.md +52 -0
  82. package/dist/general-knowledge/skills/development-workflow/SKILL.md +28 -0
  83. package/dist/general-knowledge/skills/fleet-config/SKILL.md +1 -0
  84. package/dist/general-knowledge/skills/fleet-health/SKILL.md +1 -0
  85. package/dist/general-knowledge/skills/fleet-restart/SKILL.md +1 -0
  86. package/dist/general-knowledge/skills/instance-lifecycle/SKILL.md +1 -0
  87. package/dist/general-knowledge/skills/model-discovery/SKILL.md +64 -16
  88. package/dist/general-knowledge/skills/multi-channel/SKILL.md +1 -0
  89. package/dist/general-knowledge/skills/scheduling/SKILL.md +1 -0
  90. package/dist/general-knowledge/skills/session-management/SKILL.md +172 -8
  91. package/dist/general-knowledge/skills/tui-effort/SKILL.md +1 -0
  92. package/dist/general-knowledge/skills/worker-collaboration/SKILL.md +30 -0
  93. package/dist/instance-lifecycle.d.ts +99 -0
  94. package/dist/instance-lifecycle.js +357 -11
  95. package/dist/instance-lifecycle.js.map +1 -1
  96. package/dist/instructions.d.ts +12 -0
  97. package/dist/instructions.js +74 -3
  98. package/dist/instructions.js.map +1 -1
  99. package/dist/locale.js +1022 -310
  100. package/dist/locale.js.map +1 -1
  101. package/dist/login-flows.d.ts +91 -0
  102. package/dist/login-flows.js +170 -0
  103. package/dist/login-flows.js.map +1 -0
  104. package/dist/login-manager.d.ts +63 -0
  105. package/dist/login-manager.js +134 -0
  106. package/dist/login-manager.js.map +1 -0
  107. package/dist/network-family.d.ts +18 -0
  108. package/dist/network-family.js +20 -0
  109. package/dist/network-family.js.map +1 -0
  110. package/dist/outbound-handlers.d.ts +16 -1
  111. package/dist/outbound-handlers.js +174 -40
  112. package/dist/outbound-handlers.js.map +1 -1
  113. package/dist/outbound-schemas.d.ts +11 -3
  114. package/dist/outbound-schemas.js +11 -0
  115. package/dist/outbound-schemas.js.map +1 -1
  116. package/dist/pane-input-residue.d.ts +52 -0
  117. package/dist/pane-input-residue.js +107 -0
  118. package/dist/pane-input-residue.js.map +1 -0
  119. package/dist/process-memory.d.ts +6 -0
  120. package/dist/process-memory.js +45 -0
  121. package/dist/process-memory.js.map +1 -1
  122. package/dist/reply-dedup.d.ts +7 -8
  123. package/dist/reply-dedup.js +0 -0
  124. package/dist/reply-dedup.js.map +1 -1
  125. package/dist/restart-progress.d.ts +2 -0
  126. package/dist/restart-progress.js +3 -0
  127. package/dist/restart-progress.js.map +1 -1
  128. package/dist/scheduler/db.d.ts +12 -0
  129. package/dist/scheduler/db.js +59 -0
  130. package/dist/scheduler/db.js.map +1 -1
  131. package/dist/service-installer.d.ts +11 -0
  132. package/dist/service-installer.js +84 -18
  133. package/dist/service-installer.js.map +1 -1
  134. package/dist/settings-api.js +1 -1
  135. package/dist/settings-api.js.map +1 -1
  136. package/dist/setup-wizard.js +2 -2
  137. package/dist/setup-wizard.js.map +1 -1
  138. package/dist/spawn-gate.d.ts +30 -0
  139. package/dist/spawn-gate.js +79 -0
  140. package/dist/spawn-gate.js.map +1 -0
  141. package/dist/steer-capability.d.ts +1 -0
  142. package/dist/steer-capability.js +6 -0
  143. package/dist/steer-capability.js.map +1 -0
  144. package/dist/storm-window.d.ts +83 -0
  145. package/dist/storm-window.js +251 -0
  146. package/dist/storm-window.js.map +1 -0
  147. package/dist/tips.d.ts +44 -0
  148. package/dist/tips.js +355 -0
  149. package/dist/tips.js.map +1 -0
  150. package/dist/tmux-manager.d.ts +16 -0
  151. package/dist/tmux-manager.js +110 -25
  152. package/dist/tmux-manager.js.map +1 -1
  153. package/dist/tool-progress.d.ts +40 -0
  154. package/dist/tool-progress.js +289 -0
  155. package/dist/tool-progress.js.map +1 -0
  156. package/dist/topic-commands.d.ts +45 -6
  157. package/dist/topic-commands.js +394 -66
  158. package/dist/topic-commands.js.map +1 -1
  159. package/dist/transcript-monitor.d.ts +15 -2
  160. package/dist/transcript-monitor.js +63 -17
  161. package/dist/transcript-monitor.js.map +1 -1
  162. package/dist/transcript-sources.d.ts +131 -0
  163. package/dist/transcript-sources.js +580 -0
  164. package/dist/transcript-sources.js.map +1 -0
  165. package/dist/types.d.ts +11 -0
  166. package/dist/ui/dashboard.html +55 -32
  167. package/dist/ui/settings.html +200 -60
  168. package/dist/ui/view.html +147 -30
  169. package/dist/usage/format-rich.d.ts +1 -1
  170. package/dist/usage/format-rich.js +28 -24
  171. package/dist/usage/format-rich.js.map +1 -1
  172. package/dist/usage/i18n-keys.d.ts +7 -0
  173. package/dist/usage/i18n-keys.js +34 -0
  174. package/dist/usage/i18n-keys.js.map +1 -0
  175. package/dist/usage/i18n.d.ts +5 -0
  176. package/dist/usage/i18n.js +27 -0
  177. package/dist/usage/i18n.js.map +1 -0
  178. package/dist/usage/providers.d.ts +21 -0
  179. package/dist/usage/providers.js +153 -75
  180. package/dist/usage/providers.js.map +1 -1
  181. package/dist/usage/usage-api.d.ts +11 -3
  182. package/dist/usage/usage-api.js +61 -24
  183. package/dist/usage/usage-api.js.map +1 -1
  184. package/dist/workflow-templates/default.md +2 -1
  185. package/package.json +2 -2
package/dist/daemon.js CHANGED
@@ -8,6 +8,9 @@ import { mcpServerState } from "./mcp-liveness.js";
8
8
  import { clearPausedMarker, writePausedMarker } from "./pause-marker.js";
9
9
  import { TmuxManager, resolveTmuxLogicalSize } from "./tmux-manager.js";
10
10
  import { TranscriptMonitor } from "./transcript-monitor.js";
11
+ import { createTranscriptSource } from "./transcript-sources.js";
12
+ import { LOGIN_FLOWS } from "./login-flows.js";
13
+ import { ProgressAccumulator, summarizeProgress } from "./tool-progress.js";
11
14
  import { ContextGuardian } from "./context-guardian.js";
12
15
  import { IpcServer } from "./channel/ipc-bridge.js";
13
16
  import { daemonBudgetMs } from "./channel/ipc-timeouts.js";
@@ -19,6 +22,8 @@ import { HangDetector } from "./hang-detector.js";
19
22
  import { writeSecretFile } from "./secret-file.js";
20
23
  import { PaneWriteLock } from "./pane-write-lock.js";
21
24
  import { buildFleetInstructions } from "./instructions.js";
25
+ import { formatCrossInstanceInboundMessage, renderCrossInstanceHandoffMetadata, } from "./cross-instance-envelope.js";
26
+ import { bottomRowIsReady, pasteLeftInInput, strandedAgendMessageInInput } from "./pane-input-residue.js";
22
27
  const __filename = fileURLToPath(import.meta.url);
23
28
  const __dirname = dirname(__filename);
24
29
  // Tool routing sets — module-level to avoid re-creation on every handleToolCall
@@ -45,6 +50,10 @@ export function buildInstructionReloadNotice(binaryName, instanceName, instanceD
45
50
  export const DEFAULT_STUCK_TIMEOUT_MS = 10 * 60_000;
46
51
  export const DEFAULT_STATE_IDLE_DEBOUNCE_MS = 2_000;
47
52
  export const DEFAULT_STATE_SAFETY_SWEEP_MS = 60_000;
53
+ /** Coalesce one cosmetic-redraw burst before comparing the visible pane. */
54
+ export const PERIODIC_REDRAW_PROBE_MS = 25;
55
+ /** A foreground server/port-forward should hand control back or be acknowledged. */
56
+ export const DEFAULT_BLOCKING_PROCESS_GRACE_MS = 2 * 60_000;
48
57
  const LAST_INBOUND_FILE = "last-inbound-at";
49
58
  /** Minimum gap between "health check is failing" notifications for one instance. */
50
59
  const HEALTH_ERROR_NOTIFY_INTERVAL_MS = 10 * 60_000;
@@ -131,19 +140,7 @@ export function writeLastInboundAt(instanceDir, timestamp) {
131
140
  * message_id line.
132
141
  */
133
142
  export function renderHandoffMetadata(meta) {
134
- const rows = [];
135
- const add = (label, value) => {
136
- if (value && value.trim())
137
- rows.push(`${label}: ${value.trim()}`);
138
- };
139
- add("message_id", meta.message_id);
140
- add("correlation_id", meta.correlation_id);
141
- add("request_kind", meta.request_kind);
142
- add("task_summary", meta.task_summary);
143
- add("working_directory", meta.working_directory);
144
- add("branch", meta.branch);
145
- add("attachment_file_id", meta.attachment_file_id);
146
- return rows.length ? `\n(${rows.join(" | ")})` : "";
143
+ return renderCrossInstanceHandoffMetadata(meta);
147
144
  }
148
145
  /** Headless inactivity timer used by the daemon and unit tests. */
149
146
  export class AutoPauseController {
@@ -157,6 +154,11 @@ export class AutoPauseController {
157
154
  recordActivity(now = Date.now()) {
158
155
  this.lastActivityAt = now;
159
156
  }
157
+ /** Apply a new idle threshold and start a fresh countdown from this update. */
158
+ reconfigure(thresholdMs, now = Date.now()) {
159
+ this.thresholdMs = Math.max(0, thresholdMs);
160
+ this.lastActivityAt = now;
161
+ }
160
162
  observe(state, now = Date.now()) {
161
163
  if (this.pausedAt !== null || this.thresholdMs <= 0)
162
164
  return false;
@@ -279,9 +281,11 @@ export class PaneStateMachine {
279
281
  * working" and a finished turn would not be seen as idle until the next 60s
280
282
  * safety sweep. Output timestamps are the honest liveness signal here, and
281
283
  * the caller has them.
284
+ * @param opts.forceBusy the backend has positively identified a still-running
285
+ * foreground tool even though its TUI keeps an old ready footer visible.
282
286
  */
283
287
  observe(pane, now = Date.now(), opts = {}) {
284
- const { settled = false, changeAt = now } = opts;
288
+ const { settled = false, changeAt = now, forceBusy = false } = opts;
285
289
  const paneHash = createHash("sha256").update(pane).digest("hex");
286
290
  const firstObservation = this.lastPaneHash === null;
287
291
  const paneChanged = this.lastPaneHash !== paneHash;
@@ -290,7 +294,10 @@ export class PaneStateMachine {
290
294
  this.lastPaneChangeAt = changeAt;
291
295
  }
292
296
  this.lastObservedAt = now;
293
- const ready = this.isReady(pane);
297
+ // Some TUIs keep their ready footer visible while a foreground tool owns
298
+ // stdin. Backends can positively identify that tool from the pane; in that
299
+ // case the old prompt must not clear pending work or retire Cancel.
300
+ const ready = !forceBusy && this.isReady(pane);
294
301
  const nextState = firstObservation
295
302
  ? ready ? "idle" : "working"
296
303
  : paneChanged && !settled
@@ -352,6 +359,33 @@ export class PendingWorkTracker {
352
359
  }
353
360
  }
354
361
  const NORMAL_ENTER_SETTLE_MS = 500;
362
+ /** Bottom-ready re-poll cadence for Enter-dropping TUIs once the pane is quiet. */
363
+ const BOTTOM_READY_POLL_MS = 250;
364
+ /** Consecutive unreadable pane probes tolerated by the delivery gate before the delivery is failed (≈10s at the poll cadence). */
365
+ const DIALOG_PROBE_UNKNOWN_MAX = 40;
366
+ /** Startup dialog scan: poll cadence and wall-clock budget (keys and waits included). */
367
+ const STARTUP_DIALOG_POLL_MS = 500;
368
+ const STARTUP_DIALOG_BUDGET_MS = 30_000;
369
+ /** A blocking dialog still on screen this long after first being seen is reported for a human. */
370
+ const DIALOG_PARKED_NOTIFY_MS = 60_000;
371
+ /** How many "dialog painted just before the write → wait → retry" rounds a delivery tolerates. */
372
+ const LATE_DIALOG_WRITE_ROUNDS = 3;
373
+ /**
374
+ * Startup failed because the CLI's backend is unreachable (see backend-outage.ts).
375
+ * The session-id is deliberately KEPT; the fleet schedules a delayed retry.
376
+ */
377
+ export class BackendUnreachableStartupError extends Error {
378
+ backend;
379
+ constructor(backend) {
380
+ super(`CLI failed to start: the ${backend} backend is unreachable — session kept for a later retry`);
381
+ this.backend = backend;
382
+ this.name = "BackendUnreachableStartupError";
383
+ }
384
+ }
385
+ /** Bounded wait (under the pane lock) for the prompt to return before retrying a dropped Enter. */
386
+ const STRANDED_RETRY_READY_WAIT_MS = 30_000;
387
+ /** Max "stranded text → submit → wait for prompt → re-check" rounds per delivery; each may send one recovery Enter. */
388
+ const STRANDED_INPUT_MAX_ROUNDS = 3;
355
389
  const FIRST_ENTER_SETTLE_MS = 1_750;
356
390
  const FIRST_DELIVERY_WINDOW_MS = 5_000;
357
391
  /** After busy native-queue paste+Enter, wait before checking the pane for silent loss. */
@@ -529,6 +563,66 @@ export function extractProxyReplyText(pane, opts = {}) {
529
563
  // The end of the turn is the answer — when over budget, keep the tail.
530
564
  return text.length > maxChars ? text.slice(text.length - maxChars) : text;
531
565
  }
566
+ /**
567
+ * Find the narrow Claude XML failure signature reported in #648.
568
+ *
569
+ * This intentionally runs only at the daemon's busy→idle edge (the caller
570
+ * enforces that) and only scans output after the current inbound marker. A
571
+ * partial tool call while Claude is still streaming is therefore never acted
572
+ * on. The extracted argument goes through the same credential redaction as the
573
+ * dead-MCP pane proxy before it can leave the host.
574
+ */
575
+ export function detectMalformedClaudeToolCall(pane, opts = {}) {
576
+ let clean = pane
577
+ .replace(/\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])/g, "")
578
+ .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "");
579
+ const marker = opts.inboundMarker?.trim();
580
+ if (marker) {
581
+ const markerAt = clean.lastIndexOf(marker);
582
+ if (markerAt >= 0)
583
+ clean = clean.slice(markerAt + marker.length);
584
+ }
585
+ const lines = clean.split(/\r?\n/).slice(-(opts.maxLines ?? 120));
586
+ const tail = lines.join("\n");
587
+ const closingTags = [...tail.matchAll(/<\/(?:parameter|invoke|function_calls)>/gi)];
588
+ if (closingTags.length === 0)
589
+ return null;
590
+ const parameterOpen = /<parameter\b[^>]*\bname\s*=\s*["']text["'][^>]*>/gi;
591
+ const parameterOpens = [...tail.matchAll(parameterOpen)];
592
+ const open = parameterOpens.at(-1);
593
+ if (!open || open.index === undefined) {
594
+ // Base stale suppression on the XML artifact itself, not unrelated pane
595
+ // redraws appended after it. The occurrence counts still distinguish a
596
+ // genuinely new identical malformed call while both remain visible.
597
+ const signature = createHash("sha256")
598
+ .update(`0:${closingTags.length}:${closingTags.map(tag => tag[0].toLowerCase()).join(":")}`)
599
+ .digest("hex");
600
+ return { signature, text: null };
601
+ }
602
+ const contentAt = open.index + open[0].length;
603
+ const closeAt = tail.slice(contentAt).search(/<\/parameter>/i);
604
+ if (closeAt < 0) {
605
+ const signature = createHash("sha256")
606
+ .update(`${parameterOpens.length}:${closingTags.length}:${open[0]}`)
607
+ .digest("hex");
608
+ return { signature, text: null };
609
+ }
610
+ const fragment = tail.slice(contentAt, contentAt + closeAt);
611
+ // Fingerprint only the malformed XML artifact; never put its contents in logs.
612
+ const signature = createHash("sha256")
613
+ .update(`${parameterOpens.length}:${closingTags.length}:${open[0]}${fragment}</parameter>`)
614
+ .digest("hex");
615
+ let text = sanitizePaneTail(fragment, opts.maxLines ?? 120)
616
+ .join("\n")
617
+ .replace(/\n{3,}/g, "\n\n")
618
+ .trim();
619
+ if (text.replace(/[^\p{L}\p{N}]/gu, "").length < 2)
620
+ return { signature, text: null };
621
+ const maxChars = opts.maxChars ?? 12_000;
622
+ if (text.length > maxChars)
623
+ text = text.slice(0, maxChars);
624
+ return { signature, text };
625
+ }
532
626
  const INTERACTIVE_PROMPT_PATTERNS = [
533
627
  { kind: "sudo_password", pattern: /^\s*\[sudo\]\s+password\s+for\s+[^:\n]+:\s*$/im },
534
628
  // macOS sudo and several package installers use this exact no-echo prompt.
@@ -591,6 +685,69 @@ export class InteractivePromptDetector {
591
685
  this.notifiedSignature = null;
592
686
  }
593
687
  }
688
+ const BLOCKING_PROCESS_PATTERNS = [
689
+ // kubectl port-forward (including a backgrounded child whose inherited
690
+ // stdout keeps the parent shell tool open).
691
+ /^\s*(?:Forwarding from\s+(?:127\.0\.0\.1|localhost|\[::1\]):\d+\s+->\s+\d+|Handling connection for\s+\d+)\s*$/im,
692
+ // Common development servers. Keep these line-shaped and require an address
693
+ // or port so prose such as "the server is running" does not arm the detector.
694
+ /^\s*(?:(?:INFO:\s*)?Uvicorn running on|Serving HTTP on|Listening on|Server (?:is )?(?:listening|running) (?:at|on)|Application startup complete[^\n]*(?:port|https?:\/\/))[^\n]*(?:https?:\/\/|\b(?:localhost|127\.0\.0\.1|0\.0\.0\.0|\[::\]|port)\b)[^\n]*$/im,
695
+ ];
696
+ /**
697
+ * Detect a foreground shell tool which has become a long-lived server.
698
+ *
699
+ * Unlike the generic pane-stuck clock, this clock deliberately survives new
700
+ * output: access logs and `Handling connection` lines are exactly why a
701
+ * foreground server can remain blocked forever without ever looking silent.
702
+ * Once a server marker is seen, the still-running tool identity is the stable
703
+ * signal; the marker may scroll out of the 32-row pane.
704
+ */
705
+ export class BlockingProcessDetector {
706
+ graceMs;
707
+ activity = null;
708
+ evidence = null;
709
+ detectedAt = 0;
710
+ notified = false;
711
+ constructor(graceMs = DEFAULT_BLOCKING_PROCESS_GRACE_MS) {
712
+ this.graceMs = graceMs;
713
+ }
714
+ observe(pane, activity, now = Date.now()) {
715
+ const shellActivity = activity && /^(?:shell|bash|exec_command|terminal|command)(?::|$)/i.test(activity);
716
+ if (!shellActivity) {
717
+ this.reset();
718
+ return null;
719
+ }
720
+ if (activity !== this.activity) {
721
+ this.reset();
722
+ this.activity = activity;
723
+ }
724
+ if (!this.evidence) {
725
+ const tail = sanitizePaneTail(pane, 40).join("\n");
726
+ for (const pattern of BLOCKING_PROCESS_PATTERNS) {
727
+ const match = tail.match(pattern);
728
+ if (!match)
729
+ continue;
730
+ this.evidence = match[0].trim().slice(0, 200);
731
+ this.detectedAt = now;
732
+ break;
733
+ }
734
+ }
735
+ if (!this.evidence || this.notified || now - this.detectedAt < this.graceMs)
736
+ return null;
737
+ this.notified = true;
738
+ return {
739
+ activity,
740
+ evidence: this.evidence,
741
+ blockedForMs: now - this.detectedAt,
742
+ };
743
+ }
744
+ reset() {
745
+ this.activity = null;
746
+ this.evidence = null;
747
+ this.detectedAt = 0;
748
+ this.notified = false;
749
+ }
750
+ }
594
751
  export class Daemon extends EventEmitter {
595
752
  name;
596
753
  config;
@@ -599,6 +756,9 @@ export class Daemon extends EventEmitter {
599
756
  backend;
600
757
  controlClient;
601
758
  runtimeIdentity;
759
+ spawnGate;
760
+ stormWindow;
761
+ backendOutage;
602
762
  logger;
603
763
  tmuxSessionName;
604
764
  tmux = null;
@@ -608,6 +768,13 @@ export class Daemon extends EventEmitter {
608
768
  guardian = null;
609
769
  adapter = null;
610
770
  pendingIpcRequests = new Map();
771
+ /**
772
+ * Daemon-global request sequence. MCP request ids are only process-local:
773
+ * Claude can have more than one MCP child connected to the same daemon and
774
+ * each starts at 1, so using requestId alone lets one caller steal another's
775
+ * response.
776
+ */
777
+ fleetRequestSeq = 0;
611
778
  // Track chatId/threadId from inbound messages for automatic outbound routing
612
779
  lastChatId;
613
780
  lastThreadId;
@@ -618,10 +785,8 @@ export class Daemon extends EventEmitter {
618
785
  currentActivity = null;
619
786
  // Session identity: map IPC socket → sessionName (from mcp_ready)
620
787
  socketSessionNames = new Map();
621
- // Crash recovery
622
- static tmuxServerCrashTimestamps = [];
623
- static tmuxServerPaused = false;
624
- static tmuxServerRecoveryTimer = null;
788
+ // Crash recovery is fleet-owned; all daemon objects share the coordinators
789
+ // injected by FleetManager below.
625
790
  healthCheckTimer = null;
626
791
  crashCount = 0;
627
792
  lastCrashAt = 0;
@@ -642,6 +807,12 @@ export class Daemon extends EventEmitter {
642
807
  resolveSpawnSettled = null;
643
808
  spawnDepth = 0;
644
809
  skipResume = false;
810
+ startupAborted = false;
811
+ /** First time the current on-screen blocking dialog was seen (0 = none); drives the parked report. */
812
+ dialogParkedSince = 0;
813
+ /** Identity of that dialog: its pattern, not its description (two tables may describe one screen differently). */
814
+ dialogParkedKey = null;
815
+ dialogParkedReported = false;
645
816
  backgroundSessionRecoveryAttempted = false;
646
817
  /** Whether the last spawn started a fresh session (not resumed). */
647
818
  isNewSession = false;
@@ -655,6 +826,7 @@ export class Daemon extends EventEmitter {
655
826
  /** Fallback safety sweep used only when no shared tmux control client exists. */
656
827
  instanceStateMonitorTimer = null;
657
828
  instanceStateIdleTimer = null;
829
+ instanceStateOutputProbeTimer = null;
658
830
  instanceStateStuckTimer = null;
659
831
  instanceStateOutputListener = null;
660
832
  instanceStateOutputEventName = null;
@@ -673,8 +845,12 @@ export class Daemon extends EventEmitter {
673
845
  // between, the agent's answer exists only on screen — the daemon relays it.
674
846
  turnHadInbound = false;
675
847
  turnOutboundDelivered = false;
848
+ /** Successful `reply` specifically; cross-instance tools do not satisfy #648. */
849
+ turnReplyDelivered = false;
676
850
  turnCorrelationId;
677
851
  turnInboundMarker;
852
+ /** Prevent a visible stale XML fragment from being recovered on later turns. */
853
+ lastMalformedToolCallSignature;
678
854
  proxyReplySeq = 0;
679
855
  autoPauseController;
680
856
  pauseRequested = false;
@@ -685,6 +861,35 @@ export class Daemon extends EventEmitter {
685
861
  * would silently no-op exactly when we most need to stop feeding the CLI.
686
862
  */
687
863
  pausePending = false;
864
+ /**
865
+ * An auth failure was detected and nothing has resolved it yet.
866
+ *
867
+ * Separate from `pausePending` because it must outlive it: the deferred pause
868
+ * clears `pausePending` as soon as it fires, but if that pause is refused the
869
+ * instance is still auth-broken and must not resume raising alarms.
870
+ */
871
+ authFailureUnresolved = false;
872
+ /** One login-screen auth report per spawn — the screen persists across polls. */
873
+ loginScreenReported = false;
874
+ /** A fatal startup screen (see StartupDialog.fatal) was already reported this run. */
875
+ fatalStartupReported = false;
876
+ /**
877
+ * The CLI is parked on a fatal startup screen and every pane write must be
878
+ * refused. This is deliberately separate from the pause that the reported
879
+ * pty_error triggers: the pause is applied by an async handler, and a queued
880
+ * delivery would race it into the modal (where paste+Enter can confirm
881
+ * "Exit and fix manually"). Set synchronously at dialog-match time; cleared
882
+ * when a startup scan reaches a real ready prompt, or on wake().
883
+ */
884
+ fatalStartupBlocked = false;
885
+ /** model_error seen mid-turn: notify only if it survives to the idle screen. */
886
+ pendingModelErrorKey = null;
887
+ /**
888
+ * Let the next pause() proceed from a stuck pane. Set only for the auth-deferred
889
+ * pause — see the pausePending consumption site for why waiting for idle there
890
+ * is waiting forever.
891
+ */
892
+ pauseAllowStuck = false;
688
893
  pauseWakeState = "active";
689
894
  pauseWakeTransition = null;
690
895
  // Model failover: override model on next spawn when rate-limited
@@ -698,6 +903,12 @@ export class Daemon extends EventEmitter {
698
903
  * and ⏳/👀/✅ reactions live here). It does NOT cover the other writers that
699
904
  * reach the pane — {@link paneWriteLock} does. */
700
905
  pasteLock = Promise.resolve();
906
+ /**
907
+ * Fleet cancellation epoch. Queue entries retain the epoch they arrived with;
908
+ * advancing it invalidates only work received before the cancel, never a new
909
+ * user message which arrives immediately afterwards.
910
+ */
911
+ deliveryEpoch = 0;
701
912
  /** Mutual exclusion for *every* write into the pane, whichever subsystem it
702
913
  * comes from. See PaneWriteLock for why interleaving is destructive. */
703
914
  paneWriteLock = new PaneWriteLock();
@@ -711,9 +922,21 @@ export class Daemon extends EventEmitter {
711
922
  lastBuiltInstructions = "";
712
923
  pasteQueueDepth = 0;
713
924
  firstDeliveryDelay = new FirstDeliveryDelay();
925
+ /** Orders /steer pastes against each other. Deliberately NOT pasteLock:
926
+ * a steer must not queue behind the normal deliveries it exists to
927
+ * overtake. Pane-level exclusion still comes from paneWriteLock inside
928
+ * deliverMessage, so a steer and a normal delivery can never interleave
929
+ * their PTY writes — the steer just doesn't wait for idle. */
930
+ steerLock = Promise.resolve();
931
+ /** Tool lines shown in the channel processing bubble for the current turn. */
932
+ turnProgress = new ProgressAccumulator();
933
+ lastProgressBroadcast = "";
934
+ progressBroadcastTimer = null;
935
+ lastProgressBroadcastAt = 0;
714
936
  // PTY error pattern monitoring
715
937
  errorMonitorTimer = null;
716
938
  interactivePromptDetector = new InteractivePromptDetector();
939
+ blockingProcessDetector = new BlockingProcessDetector();
717
940
  /** Same 5-min gate the error monitor uses, so a dead MCP server alerts once. */
718
941
  static MCP_DEATH_COOLDOWN_MS = 5 * 60_000;
719
942
  lastMcpDeathNotifiedAt = 0;
@@ -725,6 +948,16 @@ export class Daemon extends EventEmitter {
725
948
  * past this point interrupting the turn costs less than staying mute.
726
949
  */
727
950
  static MCP_RESTART_MAX_WAIT_MS = 30 * 60_000;
951
+ /**
952
+ * How long the revival restart waits, at the point of firing, for a
953
+ * CLI-respawned MCP server to announce itself.
954
+ *
955
+ * Codex <0.146 tears down and recreates ALL MCP connections on an auth/config
956
+ * change — healthy ones included — so the daemon sees the old pid die while
957
+ * the CLI is fine and a replacement is seconds away. Module load measured
958
+ * ~0.36s, so 2s is roughly 5x headroom without making a genuine loss wait.
959
+ */
960
+ static MCP_REPLACEMENT_GRACE_MS = 2_000;
728
961
  /**
729
962
  * Sticky "restart to revive the MCP server as soon as the pane idles" — same
730
963
  * shape as pausePending, because the death is usually detected mid-turn when
@@ -733,6 +966,25 @@ export class Daemon extends EventEmitter {
733
966
  */
734
967
  mcpRestartPending = false;
735
968
  mcpRestartStaleTimer = null;
969
+ /** In-flight replacement grace; also the guard against three triggers stacking. */
970
+ mcpRestartGraceTimer = null;
971
+ /**
972
+ * Which trigger the in-flight grace will commit under.
973
+ *
974
+ * The single-timer guard would otherwise SWALLOW a stale_timeout that lands
975
+ * during an idle-gated grace — and because the stale timer is one-shot and
976
+ * already cleared itself, the 30-minute backstop would be gone for good on an
977
+ * instance that never idles, which is the exact case it exists for.
978
+ */
979
+ mcpRestartGraceTrigger = null;
980
+ /**
981
+ * A dead-MCP sighting held back once, pending a possible replacement.
982
+ *
983
+ * Deliberately NOT the same field as `mcpDeathNotifiedForPid`: deferring must
984
+ * not consume the once-per-pid budget, or a suspicion that turns out to be a
985
+ * real loss could never be reported at all.
986
+ */
987
+ mcpDeathDeferredForPid = null;
736
988
  /** Prevent in-flight monitor callbacks from re-arming after a pause. */
737
989
  runtimeMonitorsFrozen = false;
738
990
  errorWaitingForRecovery = false; // true = error detected, waiting for ready pattern
@@ -741,7 +993,7 @@ export class Daemon extends EventEmitter {
741
993
  activeErrorPatternKey = null;
742
994
  /** Whether this instance is in an abnormal error state (auto-pause is normal). */
743
995
  get isErrorState() {
744
- return this.errorWaitingForRecovery || (this.healthCheckPaused && !this.isPaused) || Daemon.tmuxServerPaused;
996
+ return this.errorWaitingForRecovery || (this.healthCheckPaused && !this.isPaused) || this.stormWindow?.isActive() === true;
745
997
  }
746
998
  get isPaused() { return this.pauseWakeState !== "active"; }
747
999
  /** Current CLI process state for status surfaces and recovery commands. */
@@ -781,7 +1033,9 @@ export class Daemon extends EventEmitter {
781
1033
  this.errorRecoveryDeadlineAt = 0;
782
1034
  this.activeErrorPatternKey = null;
783
1035
  }
784
- constructor(name, config, instanceDir, topicMode = false, backend, controlClient, rootLogger, runtimeIdentity) {
1036
+ constructor(name, config, instanceDir, topicMode = false, backend, controlClient, rootLogger, runtimeIdentity, spawnGate, stormWindow,
1037
+ /** Fleet-level "is this CLI's backend down?" memory — see backend-outage.ts. */
1038
+ backendOutage) {
785
1039
  super();
786
1040
  this.name = name;
787
1041
  this.config = config;
@@ -790,6 +1044,9 @@ export class Daemon extends EventEmitter {
790
1044
  this.backend = backend;
791
1045
  this.controlClient = controlClient;
792
1046
  this.runtimeIdentity = runtimeIdentity;
1047
+ this.spawnGate = spawnGate;
1048
+ this.stormWindow = stormWindow;
1049
+ this.backendOutage = backendOutage;
793
1050
  if (!rootLogger)
794
1051
  throw new Error("Daemon requires a shared root logger");
795
1052
  this.runtimeIdentity ??= {
@@ -878,20 +1135,7 @@ export class Daemon extends EventEmitter {
878
1135
  this.handleToolCall(msg, socket);
879
1136
  }
880
1137
  else if (msg.type === "mcp_ready") {
881
- const sessionName = msg.sessionName;
882
- if (sessionName) {
883
- this.socketSessionNames.set(socket, sessionName);
884
- socket.on("close", () => {
885
- this.socketSessionNames.delete(socket);
886
- // Notify fleet manager so it can clean up sessionRegistry
887
- if (sessionName !== this.name) {
888
- this.ipcServer?.broadcast({ type: "session_disconnected", sessionName });
889
- }
890
- });
891
- }
892
- this.logger.debug({ sessionName }, "MCP channel server connected and ready");
893
- // Notify FleetManager's IPC client that MCP is ready
894
- this.ipcServer?.broadcast({ type: "mcp_ready", sessionName });
1138
+ this.handleMcpReady(msg, socket);
895
1139
  }
896
1140
  else if (msg.type === "query_sessions") {
897
1141
  // Fleet manager asks for all registered session names (catches sessions
@@ -911,43 +1155,38 @@ export class Daemon extends EventEmitter {
911
1155
  // Fleet manager routed a message to us (topic mode)
912
1156
  const meta = msg.meta;
913
1157
  const targetSession = msg.targetSession;
1158
+ const deliveryEpoch = this.captureDeliveryEpoch(msg.delivery_epoch);
914
1159
  void this.wake().then(() => {
915
- this.pushChannelMessage(msg.content, meta, targetSession);
1160
+ this.pushChannelMessage(msg.content, meta, targetSession, deliveryEpoch);
916
1161
  }).catch(err => {
917
1162
  this.logger.error({ err: err.message }, "Wake failed for inbound delivery");
918
1163
  });
919
1164
  }
920
1165
  else if (msg.type === "raw_paste") {
921
1166
  // Paste raw text directly to CLI without [user:] wrapping.
922
- if (this.tmux) {
923
- const rawText = msg.content;
924
- this.pasteLock = this.pasteLock.then(async () => {
925
- await this.wake();
926
- await this.deliverMessage(rawText);
927
- this.logger.debug({ text: rawText.slice(0, 100) }, "Raw paste delivered");
928
- }).catch(err => {
929
- this.logger.warn({ err: err.message }, "raw_paste delivery error");
930
- });
931
- }
1167
+ this.queueRawPaste(msg.content, this.captureDeliveryEpoch(msg.delivery_epoch), msg.confirm_clear === true);
1168
+ }
1169
+ else if (msg.type === "config_update") {
1170
+ this.applyConfigUpdate(msg.config);
1171
+ }
1172
+ else if (msg.type === "steer") {
1173
+ const meta = (msg.meta ?? {});
1174
+ this.steerMessage(msg.content, meta, this.captureDeliveryEpoch(msg.delivery_epoch));
1175
+ }
1176
+ else if (msg.type === "btw") {
1177
+ const meta = (msg.meta ?? {});
1178
+ this.btwMessage(msg.content, meta, this.captureDeliveryEpoch(msg.delivery_epoch));
932
1179
  }
933
1180
  else if (msg.type === "fleet_schedule_trigger") {
934
1181
  const payload = msg.payload;
935
1182
  const meta = msg.meta;
936
- void this.wake().then(() => this.pushChannelMessage(payload.message, meta)).catch(err => {
1183
+ const deliveryEpoch = this.captureDeliveryEpoch(msg.delivery_epoch);
1184
+ void this.wake().then(() => this.pushChannelMessage(payload.message, meta, undefined, deliveryEpoch)).catch(err => {
937
1185
  this.logger.error({ err: err.message }, "Wake failed for scheduled delivery");
938
1186
  });
939
1187
  }
940
1188
  else if (msg.type === "query_instance_state") {
941
- const snapshot = this.getInstanceStateSnapshot();
942
- this.ipcServer?.send(socket, {
943
- type: "instance_state_response",
944
- requestId: msg.requestId,
945
- instanceName: this.name,
946
- ...snapshot,
947
- state: this.isPaused ? "paused" : snapshot.state,
948
- processStatus: this.processStatus,
949
- pausedAt: this.lastPausedAt,
950
- });
1189
+ void this.respondToInstanceStateQuery(msg, socket);
951
1190
  }
952
1191
  });
953
1192
  // 2. Tmux — ensure session, create window if not alive
@@ -1033,8 +1272,11 @@ export class Daemon extends EventEmitter {
1033
1272
  const outputLog = join(this.instanceDir, "output.log");
1034
1273
  rotateLogIfNeeded(outputLog);
1035
1274
  await this.tmux.pipeOutput(outputLog).catch(() => { });
1036
- // 4. Transcript monitor
1037
- this.transcriptMonitor = new TranscriptMonitor(this.instanceDir, this.logger);
1275
+ // 4. Transcript monitor. claude-code is handled inside the monitor
1276
+ // (statusline transcript); codex/kiro/opencode read their CLI's own
1277
+ // conversation store via a pluggable source. Backends with no known
1278
+ // source stay inert exactly as before.
1279
+ this.transcriptMonitor = new TranscriptMonitor(this.instanceDir, this.logger, createTranscriptSource(this.config.backend ?? "claude-code", this.config.working_directory));
1038
1280
  // 5. Wire transcript events
1039
1281
  const ackIfPending = () => {
1040
1282
  if (!this.pendingAckMessage || !this.adapter)
@@ -1050,6 +1292,7 @@ export class Daemon extends EventEmitter {
1050
1292
  this.recordRecentEvent({ type: "tool_use", name, preview: this.summarizeTool(name, input) });
1051
1293
  this.recordRecentToolActivity(this.summarizeTool(name, input));
1052
1294
  this.publishActivity(this.summarizeTool(name, input));
1295
+ this.recordToolProgress(name, input);
1053
1296
  });
1054
1297
  this.transcriptMonitor.on("tool_result", (name, _output) => {
1055
1298
  this.recordRecentEvent({ type: "tool_result", name });
@@ -1112,6 +1355,42 @@ export class Daemon extends EventEmitter {
1112
1355
  * process would have no client reading it. Only the CLI can restore its own
1113
1356
  * tools — hence notify, and let the operator decide about restarting.
1114
1357
  */
1358
+ /**
1359
+ * An MCP server has connected and announced itself.
1360
+ *
1361
+ * Beyond the existing session bookkeeping this is the earliest evidence that
1362
+ * tools are back — roughly 30s ahead of the next health tick, which is
1363
+ * exactly the window #663's false restarts fired in.
1364
+ */
1365
+ handleMcpReady(msg, socket) {
1366
+ const sessionName = msg.sessionName;
1367
+ if (sessionName) {
1368
+ this.socketSessionNames.set(socket, sessionName);
1369
+ socket.on("close", () => {
1370
+ this.socketSessionNames.delete(socket);
1371
+ // Notify fleet manager so it can clean up sessionRegistry
1372
+ if (sessionName !== this.name) {
1373
+ this.ipcServer?.broadcast({ type: "session_disconnected", sessionName });
1374
+ }
1375
+ });
1376
+ }
1377
+ this.logger.debug({ sessionName, pid: msg.pid }, "MCP channel server connected and ready");
1378
+ // Gated on ownership: this ipcServer also accepts EXTERNAL sessions (a
1379
+ // `claude` run in another directory registers as `external-<cwd>-<pid>`).
1380
+ // Without the gate an unrelated session could stand down a restart armed
1381
+ // for THIS instance's genuinely dead server. The liveness re-read below is
1382
+ // a second guard, but ownership is the one that is semantically right.
1383
+ if (sessionName === this.name && this.mcpRestartPending) {
1384
+ const live = mcpServerState(this.instanceDir);
1385
+ if (live.state === "alive") {
1386
+ this.logger.info({ pid: live.pid }, "MCP server replaced — standing down the pending revival restart");
1387
+ this.mcpDeathNotifiedForPid = null;
1388
+ this.mcpDeathDeferredForPid = null;
1389
+ this.clearMcpRestartRequest();
1390
+ }
1391
+ }
1392
+ this.ipcServer?.broadcast({ type: "mcp_ready", sessionName });
1393
+ }
1115
1394
  checkMcpServerAlive() {
1116
1395
  if (this.isPaused)
1117
1396
  return;
@@ -1120,22 +1399,53 @@ export class Daemon extends EventEmitter {
1120
1399
  return;
1121
1400
  if (status.state === "alive") {
1122
1401
  this.mcpDeathNotifiedForPid = null; // the CLI respawned it — re-arm
1402
+ this.mcpDeathDeferredForPid = null;
1123
1403
  this.clearMcpRestartRequest(); // tools are back — stand down a pending auto-restart
1124
1404
  return;
1125
1405
  }
1126
1406
  // Dead: report once per pid, and at most once per cooldown window.
1127
1407
  if (this.mcpDeathNotifiedForPid === status.pid)
1128
1408
  return;
1409
+ // An auth-broken CLI cannot keep an MCP server alive, and restarting it
1410
+ // only loops back to the sign-in screen — hold the auto-restart until a
1411
+ // re-login (which restarts the instance and replaces this daemon anyway).
1412
+ const authSuspected = this.authFailureUnresolved;
1413
+ const autoRestart = this.config.mcp_auto_restart !== false && !authSuspected;
1414
+ // Codex <0.146 kills and recreates its MCP connections wholesale, so the
1415
+ // FIRST sighting of a dead pid under a live Codex is more likely a swap
1416
+ // than a loss. Hold the alarm for one tick — only the alarm: the restart is
1417
+ // still armed, and the chokepoint's re-read cancels it if the replacement
1418
+ // arrives. Not generalised to other backends, where a dead MCP under a live
1419
+ // CLI is a genuine incident that silence would mask.
1420
+ //
1421
+ // Crucially this does NOT touch mcpDeathNotifiedForPid or the cooldown
1422
+ // clock: if the suspicion is wrong the incident must still be reportable.
1423
+ if (this.suspectsTransientMcpReplacement() && this.mcpDeathDeferredForPid !== status.pid) {
1424
+ this.mcpDeathDeferredForPid = status.pid;
1425
+ this.logger.warn({ pid: status.pid }, "MCP server pid is gone under a live Codex — holding the alarm one tick for a possible replacement");
1426
+ if (autoRestart)
1427
+ this.armMcpRestartWhenIdle();
1428
+ return;
1429
+ }
1129
1430
  if (Date.now() - this.lastMcpDeathNotifiedAt < Daemon.MCP_DEATH_COOLDOWN_MS)
1130
1431
  return;
1131
1432
  this.mcpDeathNotifiedForPid = status.pid;
1433
+ this.mcpDeathDeferredForPid = null;
1132
1434
  this.lastMcpDeathNotifiedAt = Date.now();
1133
- const autoRestart = this.config.mcp_auto_restart !== false;
1134
- this.logger.error({ pid: status.pid, autoRestart }, "MCP server process is gone — instance has no agend tools");
1135
- this.emit("mcp_died", { name: this.name, pid: status.pid, autoRestart });
1435
+ this.logger.error({ pid: status.pid, autoRestart, authSuspected }, "MCP server process is gone — instance has no agend tools");
1436
+ this.emit("mcp_died", { name: this.name, pid: status.pid, autoRestart, authSuspected });
1136
1437
  if (autoRestart)
1137
1438
  this.armMcpRestartWhenIdle();
1138
1439
  }
1440
+ /**
1441
+ * Whether a dead MCP pid on this backend is more likely a CLI-driven
1442
+ * replacement than a real loss. Codex only, and deliberately narrow: it is
1443
+ * the backend whose released fix (0.146) confirms it tore down healthy MCP
1444
+ * connections alongside the one it meant to restart.
1445
+ */
1446
+ suspectsTransientMcpReplacement() {
1447
+ return (this.config.backend ?? this.backend?.binaryName) === "codex";
1448
+ }
1139
1449
  /**
1140
1450
  * Arm an automatic instance restart to revive a dead MCP server. Fires on the
1141
1451
  * next busy→idle edge — or immediately when the pane is already idle, which is
@@ -1158,21 +1468,110 @@ export class Daemon extends EventEmitter {
1158
1468
  this.fireMcpRestartRequest("already_idle");
1159
1469
  }
1160
1470
  }
1471
+ /**
1472
+ * Single chokepoint for all three revival triggers (already_idle, idle_edge,
1473
+ * stale_timeout) — and therefore the one place worth re-checking reality.
1474
+ *
1475
+ * The pending flag is armed by a health tick that saw the OLD pid die, and
1476
+ * was previously fired without ever looking again. When a CLI legitimately
1477
+ * respawns its own MCP child, the replacement is already serving by the time
1478
+ * the turn ends — but the idle edge fires inside the ~30s before the next
1479
+ * health tick can stand the request down, restarting a healthy instance. One
1480
+ * external user was restarted 18 times in two days this way (#663).
1481
+ *
1482
+ * Re-reading here is safe for every backend: worst case it clears a pending
1483
+ * restart that should not have existed.
1484
+ */
1161
1485
  fireMcpRestartRequest(trigger) {
1162
- // A recovery is already reshaping the CLI: crash-loop handling
1163
- // (healthCheckPaused), a respawn in flight (spawning), or a pause
1164
- // (isPaused / frozen monitors). Each of those paths ends in a fresh CLI —
1165
- // and with it a fresh MCP server — or in supervision ending, where a
1166
- // restart on top would fight the recovery. Either way the revival restart
1167
- // is moot: cancel it rather than defer it.
1168
- if (this.healthCheckPaused || this.spawning || this.isPaused || this.runtimeMonitorsFrozen) {
1169
- this.logger.warn({ trigger }, "MCP revival restart cancelled — instance is already pausing/spawning/recovering");
1486
+ if (this.mcpRestartCancelledByRecovery(trigger))
1487
+ return;
1488
+ // Authoritative re-read — the CLI may have replaced the server since the
1489
+ // health tick that armed this.
1490
+ if (mcpServerState(this.instanceDir).state === "alive") {
1491
+ this.logger.info({ trigger }, "MCP revival restart stood down — a live MCP server is already serving");
1170
1492
  this.clearMcpRestartRequest();
1171
1493
  return;
1172
1494
  }
1495
+ // Not back yet: give a CLI-driven replacement a bounded moment rather than
1496
+ // restarting the instance out from under it. One timer covers all three
1497
+ // triggers; a second trigger during the wait is a no-op.
1498
+ if (this.mcpRestartGraceTimer) {
1499
+ // stale_timeout outranks the idle-gated triggers: it is the backstop for
1500
+ // a turn that never ends, so it must not be lost to an in-flight grace.
1501
+ if (trigger === "stale_timeout")
1502
+ this.mcpRestartGraceTrigger = "stale_timeout";
1503
+ return;
1504
+ }
1505
+ this.mcpRestartGraceTrigger = trigger;
1506
+ this.mcpRestartGraceTimer = setTimeout(() => {
1507
+ this.mcpRestartGraceTimer = null;
1508
+ const committed = this.mcpRestartGraceTrigger ?? trigger;
1509
+ this.mcpRestartGraceTrigger = null;
1510
+ if (!this.mcpRestartPending || this.runtimeMonitorsFrozen)
1511
+ return;
1512
+ if (this.mcpRestartCancelledByRecovery(committed))
1513
+ return;
1514
+ if (mcpServerState(this.instanceDir).state === "alive") {
1515
+ this.logger.info({ trigger: committed }, "MCP revival restart stood down — replacement came up during the grace window");
1516
+ this.clearMcpRestartRequest();
1517
+ return;
1518
+ }
1519
+ // Re-check the idle gate. The grace turned what used to be a synchronous
1520
+ // decision into a 2s window, and "idle edge, then a queued message lands"
1521
+ // is the NORMAL ordering for queued delivery — so without this the
1522
+ // restart tears down a turn that started during the wait. Worse than the
1523
+ // race it replaced: the #485 proxy reply only runs at turn end, so an
1524
+ // in-flight answer would be swallowed with no fallback. Only the
1525
+ // 30-minute stale timer may interrupt a live turn, by design.
1526
+ //
1527
+ // Deliberately does NOT clear pending: the MCP really is gone, so the
1528
+ // request must survive to the next idle edge (which re-enters here), and
1529
+ // the stale timer keeps running underneath as the backstop.
1530
+ if (committed !== "stale_timeout" && !(this.instanceState === "idle" && this.pasteQueueDepth === 0)) {
1531
+ this.logger.info({ trigger: committed, instanceState: this.instanceState, pasteQueueDepth: this.pasteQueueDepth }, "MCP revival restart deferred — the pane went busy during the grace window");
1532
+ return;
1533
+ }
1534
+ // The suspicion was wrong: a real loss. Report it BEFORE asking for the
1535
+ // restart — instance-lifecycle's suppression path documents that it
1536
+ // assumes the mcp_died notification has already gone out, and this daemon
1537
+ // does not survive the restart to send it afterwards.
1538
+ this.reportDeferredMcpDeath();
1539
+ this.clearMcpRestartRequest();
1540
+ this.logger.warn({ trigger: committed }, "Requesting instance restart to revive its MCP server");
1541
+ this.emit("mcp_restart_requested", { name: this.name, trigger: committed });
1542
+ }, Daemon.MCP_REPLACEMENT_GRACE_MS);
1543
+ this.mcpRestartGraceTimer.unref?.();
1544
+ }
1545
+ /**
1546
+ * A recovery is already reshaping the CLI: crash-loop handling
1547
+ * (healthCheckPaused), a respawn in flight (spawning), or a pause
1548
+ * (isPaused / frozen monitors). Each ends in a fresh CLI — and with it a
1549
+ * fresh MCP server — or in supervision ending, where a restart on top would
1550
+ * fight the recovery. Either way the revival restart is moot.
1551
+ */
1552
+ mcpRestartCancelledByRecovery(trigger) {
1553
+ if (!(this.healthCheckPaused || this.spawning || this.isPaused || this.runtimeMonitorsFrozen || this.stormWindow?.isActive()))
1554
+ return false;
1555
+ this.logger.warn({ trigger }, "MCP revival restart cancelled — instance is already pausing/spawning/recovering");
1173
1556
  this.clearMcpRestartRequest();
1174
- this.logger.warn({ trigger }, "Requesting instance restart to revive its MCP server");
1175
- this.emit("mcp_restart_requested", { name: this.name, trigger });
1557
+ return true;
1558
+ }
1559
+ /**
1560
+ * Emit a dead-MCP report that was held back as a suspected replacement, once
1561
+ * it is clear no replacement is coming. Consumes the once-per-pid budget
1562
+ * here — deferring was to avoid spending it on a false alarm, not to drop
1563
+ * the incident.
1564
+ */
1565
+ reportDeferredMcpDeath() {
1566
+ const pid = this.mcpDeathDeferredForPid;
1567
+ if (pid == null || this.mcpDeathNotifiedForPid === pid)
1568
+ return;
1569
+ const authSuspected = this.authFailureUnresolved;
1570
+ const autoRestart = this.config.mcp_auto_restart !== false && !authSuspected;
1571
+ this.mcpDeathNotifiedForPid = pid;
1572
+ this.lastMcpDeathNotifiedAt = Date.now();
1573
+ this.logger.error({ pid, autoRestart, authSuspected }, "MCP server process is gone — no replacement arrived, instance has no agend tools");
1574
+ this.emit("mcp_died", { name: this.name, pid, autoRestart, authSuspected });
1176
1575
  }
1177
1576
  clearMcpRestartRequest() {
1178
1577
  this.mcpRestartPending = false;
@@ -1180,6 +1579,11 @@ export class Daemon extends EventEmitter {
1180
1579
  clearTimeout(this.mcpRestartStaleTimer);
1181
1580
  this.mcpRestartStaleTimer = null;
1182
1581
  }
1582
+ if (this.mcpRestartGraceTimer) {
1583
+ clearTimeout(this.mcpRestartGraceTimer);
1584
+ this.mcpRestartGraceTimer = null;
1585
+ }
1586
+ this.mcpRestartGraceTrigger = null;
1183
1587
  }
1184
1588
  startHealthCheck() {
1185
1589
  if (this.runtimeMonitorsFrozen || this.healthCheckTimer)
@@ -1213,7 +1617,7 @@ export class Daemon extends EventEmitter {
1213
1617
  this.emitSupervisionEnded("instance directory was removed from disk", "Recreate the instance, or delete it from fleet.yaml.");
1214
1618
  return;
1215
1619
  }
1216
- if (!this.tmux || this.spawning || this.healthCheckPaused || Daemon.tmuxServerPaused) {
1620
+ if (!this.tmux || this.spawning || this.healthCheckPaused) {
1217
1621
  scheduleNext();
1218
1622
  return;
1219
1623
  }
@@ -1271,28 +1675,29 @@ export class Daemon extends EventEmitter {
1271
1675
  let nullReason;
1272
1676
  if (!paneStatus) {
1273
1677
  const serverAlive = await TmuxManager.sessionExists(this.tmuxSessionName);
1274
- if (!serverAlive) {
1678
+ // A server may have already restarted before this daemon's health
1679
+ // tick. The new PID is the durable generation boundary: treat it as
1680
+ // a fleet storm even though `has-session` is true again.
1681
+ const generationChanged = serverAlive
1682
+ ? this.stormWindow?.observeServerAlive(await TmuxManager.getServerPid(this.tmuxSessionName)) === true
1683
+ : false;
1684
+ if (generationChanged)
1685
+ this.emit("tmux_server_crash", this.name);
1686
+ if (!serverAlive || generationChanged || this.stormWindow?.needsRecovery(this.name)) {
1275
1687
  crashType = "server";
1276
- nullReason = "server_gone";
1277
- this.logger.error(`tmux server died — all ${cliLabel} windows lost`);
1278
- // Fleet-level circuit breaker: pause all instances on repeated tmux server crashes
1279
- Daemon.tmuxServerCrashTimestamps.push(Date.now());
1280
- const cutoff = Date.now() - 5 * 60_000;
1281
- Daemon.tmuxServerCrashTimestamps = Daemon.tmuxServerCrashTimestamps.filter(t => t > cutoff);
1282
- if (Daemon.tmuxServerCrashTimestamps.length >= 2 && !Daemon.tmuxServerPaused) {
1283
- Daemon.tmuxServerPaused = true;
1284
- this.logger.error("Fleet-level tmux server circuit breaker triggered — pausing all respawns for 30s");
1285
- this.emit("tmux_server_crash", this.name);
1286
- if (!Daemon.tmuxServerRecoveryTimer) {
1287
- Daemon.tmuxServerRecoveryTimer = setTimeout(() => {
1288
- Daemon.tmuxServerRecoveryTimer = null;
1289
- Daemon.tmuxServerPaused = false;
1290
- }, 30_000);
1291
- }
1292
- scheduleNext();
1293
- return;
1688
+ nullReason = serverAlive ? "server_storm_window_loss" : "server_gone";
1689
+ if (!serverAlive) {
1690
+ this.logger.error(`tmux server died — all ${cliLabel} windows lost`);
1691
+ const distinct = this.stormWindow?.recordServerDead(this.name, [this.name]) ?? true;
1692
+ if (distinct)
1693
+ this.emit("tmux_server_crash", this.name);
1294
1694
  }
1295
- await new Promise(r => setTimeout(r, 2_000)); // let session stabilize
1695
+ else {
1696
+ this.stormWindow?.addAffected(this.name);
1697
+ }
1698
+ // The fleet breaker owns the delay and extends it on every new
1699
+ // server generation. Do not schedule a competing fixed timer.
1700
+ await this.stormWindow?.waitForSpawnAllowed();
1296
1701
  }
1297
1702
  else {
1298
1703
  // null but server alive: window-level disappearance. Probe whether
@@ -1388,6 +1793,14 @@ export class Daemon extends EventEmitter {
1388
1793
  }
1389
1794
  // Append to crash history
1390
1795
  this.appendCrashHistory({ exitCode, lastOutput, crashType, reason: nullReason });
1796
+ // A fleet-wide server loss is one infrastructure incident, not N
1797
+ // independent CLI crash loops. StormWindow owns its escalation; do
1798
+ // not let the same three server generations permanently trip every
1799
+ // daemon's per-instance 3-in-5m breaker.
1800
+ if (crashType === "server") {
1801
+ this.crashTimestamps = [];
1802
+ this.crashCount = 0;
1803
+ }
1391
1804
  if (max_retries <= 0) {
1392
1805
  this.healthCheckPaused = true;
1393
1806
  this.logger.warn(`${cliLabel} window died — automatic restart is disabled`);
@@ -1474,9 +1887,15 @@ export class Daemon extends EventEmitter {
1474
1887
  this.setProcessStatus("running");
1475
1888
  this.logger.info({ resumed }, `Respawned ${cliLabel} window after crash`);
1476
1889
  this.emit("crash_respawn", this.name);
1890
+ // Emit the per-instance audit event while the storm is still active
1891
+ // so its chat notification is suppressed into the fleet summary.
1892
+ if (crashType === "server")
1893
+ this.stormWindow?.markRecovered(this.name);
1477
1894
  }
1478
1895
  catch (err) {
1479
- this.logger.error({ err }, `Failed to respawn ${cliLabel} window`);
1896
+ if (!this.handOffBackendUnreachableRespawn(err)) {
1897
+ this.logger.error({ err }, `Failed to respawn ${cliLabel} window`);
1898
+ }
1480
1899
  }
1481
1900
  }
1482
1901
  catch (err) {
@@ -1585,10 +2004,37 @@ export class Daemon extends EventEmitter {
1585
2004
  // A prompt is not an error and must not enter the PTY recovery gate.
1586
2005
  // Continue scanning real errors in this same snapshot.
1587
2006
  }
1588
- // Auto-dismiss runtime dialogs (e.g. Codex rate limit model switch)
2007
+ // If a backend can derive activity from this exact pane, trust its
2008
+ // explicit null (tool completed) instead of falling back to a possibly
2009
+ // stale transcript activity from the previous scan.
2010
+ const paneActivity = this.backend?.getPaneActivity
2011
+ ? this.backend.getPaneActivity(pane)
2012
+ : this.currentActivity;
2013
+ const hasPendingWork = this.pendingWork.hasPendingWork();
2014
+ if (!hasPendingWork)
2015
+ this.blockingProcessDetector.reset();
2016
+ const blockingProcess = hasPendingWork
2017
+ ? this.blockingProcessDetector.observe(pane, paneActivity, Date.now())
2018
+ : null;
2019
+ if (blockingProcess) {
2020
+ // Reuse the hang-notification bridge: it offers explicit Restart/Wait
2021
+ // choices and, unlike pty_error, does not retire a still-useful Cancel
2022
+ // button merely because a foreground process owns stdin.
2023
+ this.logger.warn(blockingProcess, "Foreground process is blocking the agent input loop");
2024
+ this.hangDetector?.emit("hang", { unchangedForMs: blockingProcess.blockedForMs });
2025
+ }
2026
+ // Auto-dismiss runtime dialogs (e.g. Codex rate limit model switch, or
2027
+ // Claude's session-resume prompt painted after the startup scan moved
2028
+ // on). Runtime table only: startup patterns are loose on purpose and
2029
+ // must never send keys into an ordinary transcript.
2030
+ let blockingSeen = null;
1589
2031
  for (const dialog of dialogs) {
1590
- if (!dialog.pattern.test(pane))
2032
+ if (!Daemon.dialogMatches(dialog, pane))
1591
2033
  continue;
2034
+ if (dialog.blocksDelivery || dialog.holdOnly)
2035
+ blockingSeen = dialog;
2036
+ if (dialog.holdOnly)
2037
+ break; // recognised, deliberately not answered; trackDialogParked reports it
1592
2038
  // These keys go straight into the pane. Sent while a message delivery is
1593
2039
  // mid-transaction, an `Escape` wipes the pasted text and an `Enter`
1594
2040
  // submits it half-composed — the user sees a message that vanished. Skip
@@ -1608,10 +2054,14 @@ export class Daemon extends EventEmitter {
1608
2054
  }
1609
2055
  });
1610
2056
  if (!dismissed) {
1611
- this.logger.debug({ dialog: dialog.description }, "Dialog dismissal deferred — pane write in flight");
2057
+ this.logger.info({ dialog: dialog.description }, "Dialog dismissal deferred — pane write in flight");
1612
2058
  }
2059
+ this.trackDialogParked(blockingSeen);
1613
2060
  return; // Dialog handled (or deliberately deferred): skip error checks this cycle
1614
2061
  }
2062
+ this.trackDialogParked(blockingSeen);
2063
+ if (blockingSeen)
2064
+ return; // held dialog: skip error checks this cycle
1615
2065
  this.evaluateErrorPatterns(pane, patterns, readyPattern, Date.now(), busyPattern);
1616
2066
  }
1617
2067
  catch {
@@ -1657,6 +2107,7 @@ export class Daemon extends EventEmitter {
1657
2107
  if (seen > 0)
1658
2108
  this.lastErrorCount.set(Daemon.errorPatternKey(ep), seen);
1659
2109
  }
2110
+ this.authFailureUnresolved = false;
1660
2111
  this.clearErrorRecoveryGate();
1661
2112
  this.logger.info({ downtime_s: downtime }, "PTY error recovered — agent is ready again");
1662
2113
  this.emit("pty_recovered", { name: this.name, downtime_s: downtime });
@@ -1677,6 +2128,19 @@ export class Daemon extends EventEmitter {
1677
2128
  this.logger.warn({ errorType: this.lastDetectedErrorType }, "PTY error recovery deadline expired — resuming error scan");
1678
2129
  this.clearErrorRecoveryGate();
1679
2130
  }
2131
+ // A model_error deferred mid-turn: the turn is over (or the pane left
2132
+ // "working"), so decide now — still on screen means the CLI gave up and
2133
+ // the user must act; gone means a retry got past it and nobody needs to know.
2134
+ if (this.pendingModelErrorKey && this.instanceState !== "working") {
2135
+ const key = this.pendingModelErrorKey;
2136
+ this.pendingModelErrorKey = null;
2137
+ const ep = patterns.find(candidate => Daemon.errorPatternKey(candidate) === key);
2138
+ if (ep && ep.pattern.test(pane)) {
2139
+ this.emitErrorPattern(ep, key, pane, now);
2140
+ return;
2141
+ }
2142
+ this.logger.debug({ errorKey: key }, "Deferred model_error cleared before idle — dropped");
2143
+ }
1680
2144
  // State: monitoring — count-based new-error detection over the full pane.
1681
2145
  for (const ep of patterns) {
1682
2146
  const key = Daemon.errorPatternKey(ep);
@@ -1704,25 +2168,49 @@ export class Daemon extends EventEmitter {
1704
2168
  continue;
1705
2169
  }
1706
2170
  this.lastErrorCount.set(key, count);
1707
- // skipRecoveryWait: this error self-recovers (e.g. a timeout — Kiro is
1708
- // back at its prompt immediately). Keep monitoring so the next occurrence
1709
- // can fire without waiting for a possibly startup-only ready pattern.
1710
- if (!ep.skipRecoveryWait) {
1711
- this.errorWaitingForRecovery = true;
1712
- this.errorDetectedAt = now;
1713
- this.errorRecoveryDeadlineAt = now + Daemon.ERROR_RECOVERY_TIMEOUT_MS;
1714
- this.activeErrorPatternKey = key;
1715
- this.lastDetectedErrorType = ep.type;
1716
- }
1717
- this.lastErrorNotifiedAt.set(key, now);
1718
- if (ep.action === "failover")
1719
- this.lastFailoverAt = now;
1720
- const message = this.resolveErrorMessage(pane, ep);
1721
- this.logger.warn({ errorType: ep.type, action: ep.action }, `PTY error detected: ${message}`);
1722
- this.emit("pty_error", { name: this.name, ...ep, message });
2171
+ // Kiro's `auto` model picks a concrete model per turn and reroutes around
2172
+ // a temporarily unavailable one on its own — telling the user to run
2173
+ // /model would ask them to fix something the CLI is already handling.
2174
+ if (ep.type === "model_error" && ep.action === "notify" && this.config.model === "auto") {
2175
+ this.logger.debug({ errorType: ep.type }, "model_error suppressed — model 'auto' reroutes on its own");
2176
+ continue;
2177
+ }
2178
+ // A mid-turn model error is often a transient the CLI retries past. Hold
2179
+ // the notification until the turn ends and notify only if the error is
2180
+ // still on the idle screen (see the deferred check above).
2181
+ if (ep.type === "model_error" && ep.action === "notify" && this.instanceState === "working") {
2182
+ this.pendingModelErrorKey = key;
2183
+ this.logger.debug({ errorType: ep.type }, "model_error deferred — instance is mid-turn, re-checking at idle");
2184
+ continue;
2185
+ }
2186
+ this.emitErrorPattern(ep, key, pane, now);
1723
2187
  break; // Only handle first unsuppressed new error per scan
1724
2188
  }
1725
2189
  }
2190
+ /** Arm the recovery gates and emit one detected error pattern. */
2191
+ emitErrorPattern(ep, key, pane, now) {
2192
+ // skipRecoveryWait: this error self-recovers (e.g. a timeout — Kiro is
2193
+ // back at its prompt immediately). Keep monitoring so the next occurrence
2194
+ // can fire without waiting for a possibly startup-only ready pattern.
2195
+ if (!ep.skipRecoveryWait) {
2196
+ this.errorWaitingForRecovery = true;
2197
+ this.errorDetectedAt = now;
2198
+ this.errorRecoveryDeadlineAt = now + Daemon.ERROR_RECOVERY_TIMEOUT_MS;
2199
+ this.activeErrorPatternKey = key;
2200
+ this.lastDetectedErrorType = ep.type;
2201
+ }
2202
+ this.lastErrorNotifiedAt.set(key, now);
2203
+ if (ep.action === "failover")
2204
+ this.lastFailoverAt = now;
2205
+ const message = this.resolveErrorMessage(pane, ep);
2206
+ // Remember an auth failure for as long as it is unresolved. An auth-broken
2207
+ // CLI cannot finish the turn it is holding, so every other detector in this
2208
+ // scan would otherwise keep reporting the same dead instance forever.
2209
+ if (ep.type === "auth_error")
2210
+ this.authFailureUnresolved = true;
2211
+ this.logger.warn({ errorType: ep.type, action: ep.action }, `PTY error detected: ${message}`);
2212
+ this.emit("pty_error", { name: this.name, ...ep, message });
2213
+ }
1726
2214
  /**
1727
2215
  * Notification text for a detected pattern. `formatMessage` patterns build it
1728
2216
  * from the match so the user gets the specifics (e.g. which period and what
@@ -1761,6 +2249,97 @@ export class Daemon extends EventEmitter {
1761
2249
  const cancelKey = this.backend?.getCancelKey() ?? "Escape";
1762
2250
  await this.tmux?.sendSpecialKey(cancelKey);
1763
2251
  }
2252
+ /**
2253
+ * Drop every not-yet-started pane delivery which predates a user cancel.
2254
+ * Promise chains cannot be removed, so entries become generation-checked
2255
+ * no-ops. An entry already past its check is allowed to finish its current PTY
2256
+ * transaction; the interrupt key still cancels the CLI generation.
2257
+ */
2258
+ clearPendingDeliveries(fleetEpoch) {
2259
+ this.deliveryEpoch = fleetEpoch === undefined
2260
+ ? this.deliveryEpoch + 1
2261
+ : Math.max(this.deliveryEpoch, fleetEpoch);
2262
+ this.logger.info({ deliveryEpoch: this.deliveryEpoch }, "Pending delivery queue cleared by user cancel");
2263
+ }
2264
+ captureDeliveryEpoch(value) {
2265
+ const epoch = typeof value === "number" && Number.isSafeInteger(value) && value >= 0
2266
+ ? value
2267
+ : this.deliveryEpoch;
2268
+ if (epoch > this.deliveryEpoch)
2269
+ this.deliveryEpoch = epoch;
2270
+ return epoch;
2271
+ }
2272
+ isDeliveryEpochCurrent(epoch) {
2273
+ return epoch === this.deliveryEpoch;
2274
+ }
2275
+ queueRawPaste(rawText, deliveryEpoch = this.deliveryEpoch, confirmClear = false) {
2276
+ if (!this.tmux || !this.isDeliveryEpochCurrent(deliveryEpoch))
2277
+ return;
2278
+ this.pasteLock = this.pasteLock.then(async () => {
2279
+ if (!this.isDeliveryEpochCurrent(deliveryEpoch)) {
2280
+ this.logger.info("Pending raw delivery dropped by user cancel");
2281
+ return;
2282
+ }
2283
+ await this.wake();
2284
+ if (!this.isDeliveryEpochCurrent(deliveryEpoch))
2285
+ return;
2286
+ await this.deliverMessage(rawText, undefined, { deliveryEpoch });
2287
+ if (confirmClear)
2288
+ await this.confirmBackendClearDialog();
2289
+ this.logger.debug({ text: rawText.slice(0, 100) }, "Raw paste delivered");
2290
+ }).catch(err => {
2291
+ this.logger.warn({ err: err.message }, "raw_paste delivery error");
2292
+ });
2293
+ }
2294
+ /**
2295
+ * Confirm a backend's terminal-level /clear prompt after the channel-level
2296
+ * admin confirmation has already succeeded. Poll instead of pasting `/clear\ny`
2297
+ * as one buffer: Kiro versions that clear immediately interpret that queued
2298
+ * `y` as a brand-new user message.
2299
+ */
2300
+ async confirmBackendClearDialog() {
2301
+ const dialog = this.backend?.getClearConfirmationDialog?.();
2302
+ if (!dialog || !this.tmux)
2303
+ return;
2304
+ const matches = (pane) => {
2305
+ dialog.pattern.lastIndex = 0;
2306
+ return dialog.pattern.test(pane);
2307
+ };
2308
+ const deadline = Date.now() + 3_000;
2309
+ while (Date.now() <= deadline) {
2310
+ const pane = await this.tmux.capturePane().catch(() => "");
2311
+ if (matches(pane)) {
2312
+ const sent = await this.paneWriteLock.run(async () => {
2313
+ // Re-check under the write lock so a prompt that disappeared between
2314
+ // capture and lock acquisition never receives a stray literal `y`.
2315
+ const currentPane = await this.tmux.capturePane().catch(() => "");
2316
+ if (!matches(currentPane))
2317
+ return false;
2318
+ const specialKeys = new Set(["Up", "Down", "Enter", "Escape", "Right", "Left"]);
2319
+ for (const key of dialog.keys) {
2320
+ const ok = specialKeys.has(key)
2321
+ ? await this.tmux.sendSpecialKey(key)
2322
+ : await this.tmux.sendKeys(key);
2323
+ if (!ok)
2324
+ return false;
2325
+ await new Promise(resolve => setTimeout(resolve, 100));
2326
+ }
2327
+ return true;
2328
+ });
2329
+ if (sent) {
2330
+ this.logger.info({ dialog: dialog.description }, "Auto-confirmed backend clear dialog");
2331
+ }
2332
+ else {
2333
+ this.logger.warn({ dialog: dialog.description }, "Backend clear dialog disappeared or confirmation keys failed");
2334
+ }
2335
+ return;
2336
+ }
2337
+ await new Promise(resolve => setTimeout(resolve, 100));
2338
+ }
2339
+ // Recent Kiro releases may clear immediately without a second prompt. That
2340
+ // is a successful no-op here, not an error.
2341
+ this.logger.debug({ dialog: dialog.description }, "Backend clear dialog did not appear");
2342
+ }
1764
2343
  /** Send the backend-specific graceful quit command/key sequence. */
1765
2344
  async sendQuitSequence() {
1766
2345
  if (!this.tmux || !this.backend)
@@ -1768,18 +2347,20 @@ export class Daemon extends EventEmitter {
1768
2347
  const quitCmd = this.backend.getQuitCommand();
1769
2348
  const quitKey = this.backend.getQuitKey?.();
1770
2349
  if (quitCmd) {
1771
- await this.tmux.sendKeys(quitCmd);
2350
+ if (!await this.tmux.sendKeys(quitCmd))
2351
+ return false;
1772
2352
  // Delay before Enter to prevent tmux server races when instances stop in
1773
2353
  // parallel (same pattern as pasteText).
1774
2354
  await new Promise(r => setTimeout(r, 150));
1775
- await this.tmux.sendSpecialKey("Enter");
1776
- return true;
2355
+ return this.tmux.sendSpecialKey("Enter");
1777
2356
  }
1778
2357
  if (!quitKey)
1779
2358
  return false;
1780
2359
  const presses = Math.max(1, Math.floor(this.backend.getQuitKeyPresses?.() ?? 1));
1781
2360
  for (let i = 0; i < presses; i++) {
1782
- await this.tmux.sendSpecialKey(quitKey);
2361
+ const sent = await this.tmux.sendSpecialKey(quitKey);
2362
+ if (!sent)
2363
+ return false;
1783
2364
  if (i + 1 < presses)
1784
2365
  await new Promise(r => setTimeout(r, 250));
1785
2366
  }
@@ -1812,8 +2393,23 @@ export class Daemon extends EventEmitter {
1812
2393
  }
1813
2394
  }
1814
2395
  }
1815
- if (!killed)
1816
- this.logger.warn("CLI did not exit gracefully within 3s, force killing window");
2396
+ if (!killed) {
2397
+ this.logger.warn({ quitSent }, "CLI did not exit gracefully within 3s — falling back to SIGTERM");
2398
+ await this.killProcessTree("SIGTERM");
2399
+ for (let i = 0; i < 5; i++) {
2400
+ await new Promise(r => setTimeout(r, 200));
2401
+ const status = await this.tmux.getPaneStatus();
2402
+ if (!status || !status.alive) {
2403
+ killed = true;
2404
+ break;
2405
+ }
2406
+ }
2407
+ }
2408
+ if (!killed) {
2409
+ this.logger.warn("CLI process tree survived SIGTERM — falling back to SIGKILL");
2410
+ await this.killProcessTree("SIGKILL");
2411
+ await new Promise(r => setTimeout(r, 200));
2412
+ }
1817
2413
  // Always kill window — remain-on-exit keeps dead panes around after CLI exits
1818
2414
  const stoppedWindowId = this.tmux.getWindowId();
1819
2415
  await this.tmux.killWindow();
@@ -1862,6 +2458,33 @@ export class Daemon extends EventEmitter {
1862
2458
  stateChangedAt: Date.now(),
1863
2459
  };
1864
2460
  }
2461
+ /**
2462
+ * Answer a FleetManager state query. Most callers only need the cached state,
2463
+ * but lifecycle decisions such as post-reply Cancel retirement need an
2464
+ * authoritative pane observation: after startup the last transition can stay
2465
+ * "working" even though the CLI has since settled without emitting another
2466
+ * control-mode output record.
2467
+ */
2468
+ async respondToInstanceStateQuery(msg, socket) {
2469
+ try {
2470
+ if (msg.refresh === true) {
2471
+ await this.captureAndEvaluateInstanceState("state_query");
2472
+ }
2473
+ const snapshot = this.getInstanceStateSnapshot();
2474
+ this.ipcServer?.send(socket, {
2475
+ type: "instance_state_response",
2476
+ requestId: msg.requestId,
2477
+ instanceName: this.name,
2478
+ ...snapshot,
2479
+ state: this.isPaused ? "paused" : snapshot.state,
2480
+ processStatus: this.processStatus,
2481
+ pausedAt: this.lastPausedAt,
2482
+ });
2483
+ }
2484
+ catch (err) {
2485
+ this.logger.debug({ err: err.message }, "Instance state query failed");
2486
+ }
2487
+ }
1865
2488
  /** Gracefully stop the CLI while keeping its remain-on-exit tmux window. */
1866
2489
  /**
1867
2490
  * Mark the instance to pause as soon as its pane is idle. Used for auth
@@ -1881,7 +2504,14 @@ export class Daemon extends EventEmitter {
1881
2504
  if (this.getPauseWakeState() !== "active")
1882
2505
  return;
1883
2506
  }
1884
- if (this.instanceState !== "idle" || this.pasteQueueDepth > 0) {
2507
+ // `pauseAllowStuck` is consumed here, not merely read: it authorises exactly
2508
+ // one pause from a stuck pane (the auth-deferred one) and must never leak
2509
+ // into an ordinary idle-timeout pause of a busy instance.
2510
+ const allowStuck = this.pauseAllowStuck;
2511
+ this.pauseAllowStuck = false;
2512
+ const pausableState = this.instanceState === "idle"
2513
+ || (allowStuck && this.instanceState === "stuck");
2514
+ if (!pausableState || this.pasteQueueDepth > 0) {
1885
2515
  this.pauseRequested = false;
1886
2516
  return;
1887
2517
  }
@@ -1945,6 +2575,10 @@ export class Daemon extends EventEmitter {
1945
2575
  // An explicit wake (e.g. after the user re-logs in) cancels a deferred
1946
2576
  // auth pause — otherwise the instance would pause again the moment it idles.
1947
2577
  this.pausePending = false;
2578
+ this.authFailureUnresolved = false;
2579
+ this.loginScreenReported = false;
2580
+ this.fatalStartupReported = false;
2581
+ this.fatalStartupBlocked = false;
1948
2582
  if (this.pauseWakeState === "active")
1949
2583
  return;
1950
2584
  if (this.pauseWakeState === "pausing")
@@ -1957,19 +2591,30 @@ export class Daemon extends EventEmitter {
1957
2591
  }
1958
2592
  this.pauseWakeState = "waking";
1959
2593
  this.beginSpawn();
2594
+ // A wake is a real resume launch: it gets at least the backend's resume
2595
+ // budget (kiro: 60s — the conversation must come back from the backend
2596
+ // before anything paints), never less than the caller's or the configured
2597
+ // startup_timeout_ms.
2598
+ const budgetMs = this.wakeBudgetMs(timeoutMs);
2599
+ // No outer race. trySpawn cannot be cancelled: racing it against a timer
2600
+ // and giving up left the daemon "paused" (monitors frozen) while the spawn
2601
+ // went on to revive the window behind its back. Every phase inside is
2602
+ // bounded on its own terms — the SpawnGate wait is deliberate storm /
2603
+ // concurrency coordination, first output + idle is `budgetMs`, the dialog
2604
+ // scan is STARTUP_DIALOG_BUDGET_MS — so the outcome is always trySpawn's
2605
+ // own verdict. A soft timer only makes a slow wake visible in the log.
1960
2606
  const transition = this.autoPauseController.wakeOnDeliver(async () => {
1961
- let timeout;
2607
+ const slow = setTimeout(() => {
2608
+ this.logger.warn({ budgetMs }, "Wake is exceeding its budget — still waiting for the spawn (it cannot be cancelled)");
2609
+ }, budgetMs + STARTUP_DIALOG_BUDGET_MS);
2610
+ slow.unref?.();
1962
2611
  try {
1963
- const ready = await Promise.race([
1964
- this.trySpawn(true, timeoutMs),
1965
- new Promise(resolve => { timeout = setTimeout(() => resolve(false), timeoutMs); }),
1966
- ]);
2612
+ const ready = await this.trySpawn(true, budgetMs);
1967
2613
  if (!ready)
1968
- throw new Error(`Wake timed out before CLI became ready (${timeoutMs}ms)`);
2614
+ throw new Error(`Wake failed: the CLI did not become ready (budget ${budgetMs}ms)`);
1969
2615
  }
1970
2616
  finally {
1971
- if (timeout)
1972
- clearTimeout(timeout);
2617
+ clearTimeout(slow);
1973
2618
  }
1974
2619
  });
1975
2620
  this.pauseWakeTransition = transition;
@@ -2021,6 +2666,7 @@ export class Daemon extends EventEmitter {
2021
2666
  // tool_result (interrupted, crashed, cancelled), which would otherwise leave
2022
2667
  // the last tool pinned to the progress line for the rest of the session.
2023
2668
  this.publishActivity(null);
2669
+ this.resetToolProgress();
2024
2670
  // Must run before the mcpRestartPending branch below: the pane text is the
2025
2671
  // only copy of the answer, and the revival restart is about to clear it.
2026
2672
  this.maybeProxyReplyOnTurnEnd(pane);
@@ -2040,8 +2686,13 @@ export class Daemon extends EventEmitter {
2040
2686
  if (snapshot.state !== "idle")
2041
2687
  this.pauseRequested = false;
2042
2688
  // A pause deferred by an auth failure: retry the moment the pane settles.
2043
- if (this.pausePending && snapshot.state === "idle" && this.pasteQueueDepth === 0) {
2689
+ // "stuck" counts as settled here, and that is the whole point — an expired
2690
+ // session cannot complete the turn it is holding, so the pane never returns
2691
+ // to idle and waiting for idle meant waiting forever. The instance stayed
2692
+ // live, kept its warm slot, and re-raised a hang alert on every scan.
2693
+ if (this.pausePending && (snapshot.state === "idle" || snapshot.state === "stuck") && this.pasteQueueDepth === 0) {
2044
2694
  this.pausePending = false;
2695
+ this.pauseAllowStuck = snapshot.state === "stuck";
2045
2696
  this.emit("auto_pause_requested", { name: this.name, idleSince: snapshot.stateChangedAt });
2046
2697
  return;
2047
2698
  }
@@ -2071,6 +2722,7 @@ export class Daemon extends EventEmitter {
2071
2722
  return;
2072
2723
  this.turnHadInbound = true;
2073
2724
  this.turnOutboundDelivered = false;
2725
+ this.turnReplyDelivered = false;
2074
2726
  this.turnCorrelationId = meta.correlation_id || undefined;
2075
2727
  // The last non-empty line of what we pasted: everything on screen after it
2076
2728
  // is the agent's own output.
@@ -2086,13 +2738,38 @@ export class Daemon extends EventEmitter {
2086
2738
  maybeProxyReplyOnTurnEnd(pane) {
2087
2739
  const hadInbound = this.turnHadInbound;
2088
2740
  const delivered = this.turnOutboundDelivered;
2741
+ const replyDelivered = this.turnReplyDelivered;
2089
2742
  const correlationId = this.turnCorrelationId;
2090
2743
  const inboundMarker = this.turnInboundMarker;
2091
2744
  this.turnHadInbound = false;
2092
2745
  this.turnOutboundDelivered = false;
2746
+ this.turnReplyDelivered = false;
2093
2747
  this.turnCorrelationId = undefined;
2094
2748
  this.turnInboundMarker = undefined;
2095
- if (!hadInbound || delivered || this.isPaused)
2749
+ if (!hadInbound || this.isPaused)
2750
+ return;
2751
+ // Claude sometimes prints a broken XML tool call as plain text and returns
2752
+ // idle without ever invoking `reply`. This is independent of MCP liveness:
2753
+ // the model malformed the call before the server could receive it.
2754
+ if (!replyDelivered && this.isClaudeCodeBackend() && pane) {
2755
+ const malformed = detectMalformedClaudeToolCall(pane, { inboundMarker });
2756
+ if (malformed) {
2757
+ const stale = malformed.signature === this.lastMalformedToolCallSignature;
2758
+ this.lastMalformedToolCallSignature = malformed.signature;
2759
+ if (!stale) {
2760
+ const recovered = malformed.text != null;
2761
+ this.logger.warn({ correlationId, recovered }, recovered
2762
+ ? "Malformed Claude tool call detected — attempting to recover reply text"
2763
+ : "Malformed Claude tool call detected — reply text could not be extracted");
2764
+ this.emit("malformed_tool_call", { name: this.name, correlationId, recovered });
2765
+ if (malformed.text)
2766
+ this.sendRecoveredMalformedReply(malformed.text, correlationId);
2767
+ }
2768
+ // Never let the broader dead-MCP fallback relay the XML/chrome too.
2769
+ return;
2770
+ }
2771
+ }
2772
+ if (delivered)
2096
2773
  return;
2097
2774
  // Opt-in: raw pane text can carry secrets past the regex redaction.
2098
2775
  if (this.config.mcp_proxy_reply !== true)
@@ -2102,6 +2779,11 @@ export class Daemon extends EventEmitter {
2102
2779
  return;
2103
2780
  void this.sendProxyReply(pane, inboundMarker, correlationId);
2104
2781
  }
2782
+ isClaudeCodeBackend() {
2783
+ return this.runtimeIdentity?.backend === "claude-code"
2784
+ || this.config.backend === "claude-code"
2785
+ || this.backend?.binaryName === "claude";
2786
+ }
2105
2787
  /** Relay the pane's final text to the channel, marked as a daemon proxy reply. */
2106
2788
  async sendProxyReply(pane, inboundMarker, correlationId) {
2107
2789
  try {
@@ -2119,55 +2801,92 @@ export class Daemon extends EventEmitter {
2119
2801
  let body = `⚠️ [MCP unavailable — proxy reply]\n\n${text}`;
2120
2802
  if (correlationId)
2121
2803
  body += `\n\n(correlation_id: ${correlationId})`;
2122
- const args = { text: body };
2123
- // Same context-bound routing as the reply tool in handleToolCall.
2124
- if (this.lastChatId) {
2125
- args.chat_id = this.lastChatId;
2126
- if (this.lastThreadId)
2127
- args.thread_id = this.lastThreadId;
2128
- }
2129
2804
  this.logger.warn({ correlationId }, "MCP server dead and the turn sent no reply — relaying the pane text as a proxy reply");
2130
2805
  this.emit("mcp_proxy_reply", { name: this.name, correlationId });
2131
- const adapters = this.messageBus.getAllAdapters();
2132
- if (adapters.length > 0) {
2133
- routeToolCall(adapters[0], "reply", args, this.lastThreadId, (_result, error) => {
2134
- if (error)
2135
- this.logger.error({ error }, "Dead-MCP proxy reply failed");
2136
- });
2137
- return;
2138
- }
2139
- if (!this.ipcServer)
2140
- return;
2141
- const fleetReqId = `proxyreply_${++this.proxyReplySeq}`;
2142
- this.ipcServer.broadcast({
2143
- type: "fleet_outbound",
2144
- tool: "reply",
2145
- args,
2146
- fleetRequestId: fleetReqId,
2147
- adapterId: this.lastAdapterId,
2148
- });
2149
- const timeout = setTimeout(() => {
2150
- this.pendingIpcRequests.delete(fleetReqId);
2151
- this.logger.error("Dead-MCP proxy reply timed out waiting for the fleet manager");
2152
- }, daemonBudgetMs("reply"));
2153
- timeout.unref?.();
2154
- this.pendingIpcRequests.set(fleetReqId, (respMsg) => {
2155
- clearTimeout(timeout);
2156
- if (respMsg.error)
2157
- this.logger.error({ error: respMsg.error }, "Dead-MCP proxy reply failed");
2158
- else
2159
- this.logger.info("Dead-MCP proxy reply delivered");
2160
- });
2806
+ this.deliverDaemonReply(body, "proxyreply", "Dead-MCP proxy reply");
2161
2807
  }
2162
2808
  catch (err) {
2163
2809
  this.logger.error({ err: err.message }, "Dead-MCP proxy reply attempt failed");
2164
2810
  }
2165
2811
  }
2812
+ /** Send a narrowly extracted #648 reply; operator notification is a separate event. */
2813
+ sendRecoveredMalformedReply(text, correlationId) {
2814
+ try {
2815
+ this.deliverDaemonReply(text, "malformedreply", "Malformed tool-call recovery");
2816
+ }
2817
+ catch (err) {
2818
+ this.logger.error({ err: err.message, correlationId }, "Malformed tool-call recovery attempt failed");
2819
+ }
2820
+ }
2821
+ /** Context-bound reply path shared by dead-MCP and malformed-call recovery. */
2822
+ deliverDaemonReply(body, requestPrefix, logLabel) {
2823
+ const args = { text: body };
2824
+ if (this.lastChatId) {
2825
+ args.chat_id = this.lastChatId;
2826
+ if (this.lastThreadId)
2827
+ args.thread_id = this.lastThreadId;
2828
+ }
2829
+ const adapters = this.messageBus.getAllAdapters();
2830
+ if (adapters.length > 0) {
2831
+ routeToolCall(adapters[0], "reply", args, this.lastThreadId, (_result, error) => {
2832
+ if (error)
2833
+ this.logger.error({ error }, `${logLabel} failed`);
2834
+ else
2835
+ this.logger.info(`${logLabel} delivered`);
2836
+ });
2837
+ return;
2838
+ }
2839
+ if (!this.ipcServer) {
2840
+ this.logger.error(`${logLabel} failed — no adapter or fleet IPC route`);
2841
+ return;
2842
+ }
2843
+ const fleetReqId = `${requestPrefix}_${++this.proxyReplySeq}`;
2844
+ const timeout = setTimeout(() => {
2845
+ this.pendingIpcRequests.delete(fleetReqId);
2846
+ this.logger.error(`${logLabel} timed out waiting for the fleet manager`);
2847
+ }, daemonBudgetMs("reply"));
2848
+ timeout.unref?.();
2849
+ // Register before broadcast: an in-process/fast adapter can answer in the
2850
+ // same tick, and a waiter installed afterwards would miss that response.
2851
+ this.pendingIpcRequests.set(fleetReqId, (respMsg) => {
2852
+ clearTimeout(timeout);
2853
+ if (respMsg.error)
2854
+ this.logger.error({ error: respMsg.error }, `${logLabel} failed`);
2855
+ else
2856
+ this.logger.info(`${logLabel} delivered`);
2857
+ });
2858
+ this.ipcServer.broadcast({
2859
+ type: "fleet_outbound",
2860
+ tool: "reply",
2861
+ args,
2862
+ fleetRequestId: fleetReqId,
2863
+ adapterId: this.lastAdapterId,
2864
+ });
2865
+ }
2166
2866
  clearInstanceStateIdleTimer() {
2167
2867
  if (this.instanceStateIdleTimer)
2168
2868
  clearTimeout(this.instanceStateIdleTimer);
2169
2869
  this.instanceStateIdleTimer = null;
2170
2870
  }
2871
+ clearInstanceStateOutputProbeTimer() {
2872
+ if (this.instanceStateOutputProbeTimer)
2873
+ clearTimeout(this.instanceStateOutputProbeTimer);
2874
+ this.instanceStateOutputProbeTimer = null;
2875
+ }
2876
+ /**
2877
+ * Periodic status-line hooks can generate `%output` without changing a single
2878
+ * visible cell. Probe the pane once per burst: a changed capture is real work;
2879
+ * an identical ready capture remains idle and emits no state edge.
2880
+ */
2881
+ scheduleInstanceStateOutputProbe() {
2882
+ if (this.instanceStateOutputProbeTimer)
2883
+ return;
2884
+ this.instanceStateOutputProbeTimer = setTimeout(() => {
2885
+ this.instanceStateOutputProbeTimer = null;
2886
+ void this.captureAndEvaluateInstanceState("output_probe", this.instanceStateLastOutputAt);
2887
+ }, PERIODIC_REDRAW_PROBE_MS);
2888
+ this.instanceStateOutputProbeTimer.unref?.();
2889
+ }
2171
2890
  clearInstanceStateStuckTimer() {
2172
2891
  if (this.instanceStateStuckTimer)
2173
2892
  clearTimeout(this.instanceStateStuckTimer);
@@ -2212,6 +2931,8 @@ export class Daemon extends EventEmitter {
2212
2931
  if (this.statePollInFlight) {
2213
2932
  if (reason === "idle_debounce")
2214
2933
  this.scheduleInstanceStateIdleCapture();
2934
+ if (reason === "output_probe")
2935
+ this.scheduleInstanceStateOutputProbe();
2215
2936
  return;
2216
2937
  }
2217
2938
  this.statePollInFlight = true;
@@ -2220,9 +2941,19 @@ export class Daemon extends EventEmitter {
2220
2941
  const pane = await this.tmux.capturePane();
2221
2942
  // Output received while capture-pane was in flight makes this snapshot
2222
2943
  // stale. Its output handler has already armed a new debounce.
2223
- if ((expectedOutputAt > 0 && this.instanceStateLastOutputAt > expectedOutputAt)
2224
- || (this.instanceStateLastOutputAt > 0 && this.instanceStateLastOutputAt >= captureStartedAt))
2944
+ const outputMovedDuringCapture = (expectedOutputAt > 0 && this.instanceStateLastOutputAt > expectedOutputAt)
2945
+ || (this.instanceStateLastOutputAt > 0 && this.instanceStateLastOutputAt >= captureStartedAt);
2946
+ if (outputMovedDuringCapture) {
2947
+ if (reason === "output_probe") {
2948
+ // A second burst overtook the probe. Do not bless its stale capture as
2949
+ // idle: continuous output is the safer working signal, while the next
2950
+ // quiet probe/debounce can still prove that it was only repainting.
2951
+ this.applyInstanceStateSnapshot(this.instanceStateMachine.recordOutput(this.instanceStateLastOutputAt));
2952
+ this.scheduleInstanceStateIdleCapture();
2953
+ this.scheduleInstanceStateStuckDeadline(this.instanceStateLastOutputAt);
2954
+ }
2225
2955
  return;
2956
+ }
2226
2957
  const observedChangeAt = expectedOutputAt || captureStartedAt;
2227
2958
  // One observation per capture. This used to call observe() twice with the
2228
2959
  // same pane: the second call always saw an unchanged pane, so the
@@ -2231,6 +2962,7 @@ export class Daemon extends EventEmitter {
2231
2962
  // mismatched spinner frame was enough to report idle mid-turn.
2232
2963
  const settled = this.instanceStateLastOutputAt === 0
2233
2964
  || captureStartedAt - this.instanceStateLastOutputAt >= this.instanceStateIdleDebounceMs;
2965
+ const paneActivity = this.backend?.getPaneActivity?.(pane) ?? null;
2234
2966
  // The two times are genuinely different and both matter: the content
2235
2967
  // changed when tmux reported output (observedChangeAt), but idle/stuck are
2236
2968
  // decisions about *now*. The old double-observe expressed that by calling
@@ -2238,6 +2970,7 @@ export class Daemon extends EventEmitter {
2238
2970
  const snapshot = this.instanceStateMachine.observe(pane, Date.now(), {
2239
2971
  settled,
2240
2972
  changeAt: observedChangeAt,
2973
+ forceBusy: paneActivity !== null,
2241
2974
  });
2242
2975
  this.applyInstanceStateSnapshot(snapshot, pane);
2243
2976
  // Backends without a transcript feed can still say what they are doing, if
@@ -2245,7 +2978,7 @@ export class Daemon extends EventEmitter {
2245
2978
  // extra tmux call. Cadence is the capture cadence (idle debounce + the 60s
2246
2979
  // safety sweep), which matches the progress ticker's own 60s interval.
2247
2980
  if (this.backend?.getPaneActivity) {
2248
- this.publishActivity(snapshot.state === "idle" ? null : this.backend.getPaneActivity(pane));
2981
+ this.publishActivity(snapshot.state === "idle" ? null : paneActivity);
2249
2982
  }
2250
2983
  if (snapshot.state === "idle") {
2251
2984
  this.clearInstanceStateStuckTimer();
@@ -2272,6 +3005,14 @@ export class Daemon extends EventEmitter {
2272
3005
  if (!windowId || event.windowId !== windowId || !this.instanceStateMachine)
2273
3006
  return;
2274
3007
  this.instanceStateLastOutputAt = event.at;
3008
+ if (this.backend?.hasPeriodicPaneRedraw?.() === true && this.instanceState === "idle") {
3009
+ // Do not emit idle→working solely because agy repainted an identical
3010
+ // footer. The short capture probe below still observes genuine pane
3011
+ // changes promptly and moves the state to working.
3012
+ this.scheduleInstanceStateOutputProbe();
3013
+ this.scheduleInstanceStateIdleCapture();
3014
+ return;
3015
+ }
2275
3016
  const snapshot = this.instanceStateMachine.recordOutput(event.at);
2276
3017
  this.applyInstanceStateSnapshot(snapshot);
2277
3018
  this.scheduleInstanceStateIdleCapture();
@@ -2322,6 +3063,7 @@ export class Daemon extends EventEmitter {
2322
3063
  stopInstanceStateMonitor() {
2323
3064
  this.instanceStateMonitorActive = false;
2324
3065
  this.clearInstanceStateIdleTimer();
3066
+ this.clearInstanceStateOutputProbeTimer();
2325
3067
  this.clearInstanceStateStuckTimer();
2326
3068
  if (this.instanceStateMonitorTimer)
2327
3069
  clearInterval(this.instanceStateMonitorTimer);
@@ -2356,6 +3098,15 @@ export class Daemon extends EventEmitter {
2356
3098
  this.logger.debug(diagnostic, "Suppressing stuck notification without pending work");
2357
3099
  return;
2358
3100
  }
3101
+ // An auth-broken instance IS stuck, and stays stuck: it cannot finish the
3102
+ // turn it is holding, so pendingWork never clears and this fired on every
3103
+ // scan forever. The notification is also useless — it offers Restart/Wait
3104
+ // and asks General for help, but only the user re-logging in fixes this, and
3105
+ // they have already been told exactly that by the auth alert.
3106
+ if (this.authFailureUnresolved) {
3107
+ this.logger.debug(diagnostic, "Suppressing stuck notification — unresolved auth failure (re-login required)");
3108
+ return;
3109
+ }
2359
3110
  this.logger.warn(diagnostic, "Instance pane stuck with pending work");
2360
3111
  this.hangDetector?.emit("hang", { unchangedForMs: snapshot.unchangedForMs });
2361
3112
  }
@@ -2375,6 +3126,7 @@ export class Daemon extends EventEmitter {
2375
3126
  // fire into a stopped daemon).
2376
3127
  this.clearMcpRestartRequest();
2377
3128
  this.interactivePromptDetector.reset();
3129
+ this.blockingProcessDetector.reset();
2378
3130
  this.stopInstanceStateMonitor();
2379
3131
  this.transcriptMonitor?.stop();
2380
3132
  this.guardian?.stop();
@@ -2400,9 +3152,9 @@ export class Daemon extends EventEmitter {
2400
3152
  * Tell the fleet manager what this instance is doing right now, for the live
2401
3153
  * progress line on the cancel button.
2402
3154
  *
2403
- * Purely cosmetic — nothing decides anything from it, so it is fine that only
2404
- * backends with a transcript feed report at all (claude-code today). For the
2405
- * rest the progress line keeps showing just elapsed time.
3155
+ * Primarily cosmetic. The foreground-process detector also uses a current
3156
+ * shell activity as one of several positive signals, but never makes a state
3157
+ * decision from an arbitrary activity label alone.
2406
3158
  *
2407
3159
  * Repeats are dropped: the ticker only edits the channel message when the text
2408
3160
  * changes, and a stream of identical broadcasts would defeat that.
@@ -2418,47 +3170,318 @@ export class Daemon extends EventEmitter {
2418
3170
  activity: next,
2419
3171
  });
2420
3172
  }
3173
+ /** Effective tool_progress level, hardened against junk config values.
3174
+ * Off unless explicitly enabled: an update must never make bubbles start
3175
+ * listing tool activity the operator did not ask to broadcast. */
3176
+ toolProgressLevel() {
3177
+ const raw = this.config.tool_progress;
3178
+ return raw === "standard" || raw === "verbose" ? raw : "off";
3179
+ }
2421
3180
  /**
2422
- * Options for every system-initiated paste (startup notice, session snapshot,
2423
- * runtime-dialog keys).
3181
+ * Apply a tool-progress setting without restarting the CLI session.
2424
3182
  *
2425
- * One place, so the three call sites cannot drift apart on it. Backends with a
2426
- * native input queue do not get the retry Enter: on those, a second bare Enter
2427
- * is not the no-op the retry assumed but a queue mutation — the same reason
2428
- * `deliverMessage` refuses to probe for busy with one.
3183
+ * Drop the current accumulator at the boundary: verbose entries may contain
3184
+ * command previews which must not survive a downgrade to standard/off, and a
3185
+ * newly enabled level should start with a clean, internally consistent list.
2429
3186
  */
2430
- systemPasteOptions() {
2431
- return { retryEnter: this.backend?.supportsQueuedInput?.() !== true };
3187
+ updateToolProgress(level) {
3188
+ const previous = this.toolProgressLevel();
3189
+ this.config.tool_progress = level;
3190
+ if (this.toolProgressLevel() !== previous)
3191
+ this.resetToolProgress();
2432
3192
  }
2433
- summarizeTool(name, input) {
2434
- const inp = input;
2435
- if (!inp)
2436
- return name;
2437
- if (name === "Read")
2438
- return `Read ${inp.file_path ?? ""}`;
2439
- if (name === "Edit")
2440
- return `Edit ${inp.file_path ?? ""}`;
2441
- if (name === "Write")
2442
- return `Write ${inp.file_path ?? ""}`;
2443
- // Command name only — never the arguments. See shellCommandLabel.
2444
- if (name === "Bash")
2445
- return `$ ${shellCommandLabel(String(inp.command ?? ""))}`;
2446
- if (name === "Glob")
2447
- return `Glob ${inp.pattern ?? ""}`;
2448
- if (name === "Grep")
2449
- return `Grep ${inp.pattern ?? ""}`;
2450
- if (name === "Agent")
2451
- return "Agent (subagent)";
3193
+ /** Snapshot the configuration actually owned by this live daemon. */
3194
+ getConfigSnapshot() {
3195
+ return structuredClone(this.config);
3196
+ }
3197
+ /**
3198
+ * Apply the whitelisted hot configuration carried by FleetManager IPC.
3199
+ * null removes an optional value; invalid/unlisted fields are ignored rather
3200
+ * than allowing another client on the per-instance socket to mutate cold
3201
+ * process/session settings.
3202
+ */
3203
+ applyConfigUpdate(value) {
3204
+ if (!value || typeof value !== "object" || Array.isArray(value))
3205
+ return;
3206
+ const update = value;
3207
+ if (update.tool_progress === null || ["off", "standard", "verbose"].includes(String(update.tool_progress))) {
3208
+ this.updateToolProgress(update.tool_progress === null ? undefined : update.tool_progress);
3209
+ }
3210
+ if (typeof update.mcp_proxy_reply === "boolean")
3211
+ this.config.mcp_proxy_reply = update.mcp_proxy_reply;
3212
+ else if (update.mcp_proxy_reply === null)
3213
+ delete this.config.mcp_proxy_reply;
3214
+ if (typeof update.warm_cap === "number" && Number.isInteger(update.warm_cap) && update.warm_cap >= 0)
3215
+ this.config.warm_cap = update.warm_cap;
3216
+ else if (update.warm_cap === null)
3217
+ delete this.config.warm_cap;
3218
+ const nextAutoPause = typeof update.auto_pause_after === "number"
3219
+ && Number.isFinite(update.auto_pause_after)
3220
+ && update.auto_pause_after >= 0
3221
+ ? update.auto_pause_after
3222
+ : update.auto_pause_after === null ? 0 : undefined;
3223
+ if (nextAutoPause !== undefined) {
3224
+ const previousAutoPause = typeof this.config.auto_pause_after === "number" ? this.config.auto_pause_after : 0;
3225
+ if (update.auto_pause_after === null)
3226
+ delete this.config.auto_pause_after;
3227
+ else
3228
+ this.config.auto_pause_after = nextAutoPause;
3229
+ if (nextAutoPause !== previousAutoPause) {
3230
+ const isGeneral = this.config.general_topic === true || this.name === "general";
3231
+ this.autoPauseController.reconfigure((isGeneral ? 0 : nextAutoPause) * 60_000);
3232
+ }
3233
+ }
3234
+ for (const key of ["display_name", "description"]) {
3235
+ if (typeof update[key] === "string")
3236
+ this.config[key] = update[key];
3237
+ else if (update[key] === null)
3238
+ delete this.config[key];
3239
+ }
3240
+ if (Array.isArray(update.tags) && update.tags.every(tag => typeof tag === "string"))
3241
+ this.config.tags = [...update.tags];
3242
+ else if (update.tags === null)
3243
+ delete this.config.tags;
3244
+ if (["trace", "debug", "info", "warn", "error"].includes(String(update.log_level))) {
3245
+ const level = update.log_level;
3246
+ if (this.config.log_level !== level) {
3247
+ this.config.log_level = level;
3248
+ this.logger.level = level;
3249
+ }
3250
+ }
3251
+ }
3252
+ /**
3253
+ * Accumulate one semantic progress line for the channel bubble and schedule
3254
+ * a coalesced broadcast. Separate from publishActivity on purpose: that one
3255
+ * is the single-line statusline detail (terse, operator-facing), this is the
3256
+ * multi-line channel list (semantic, argument-free at `standard`) — see
3257
+ * tool-progress.ts for why the two labellers must not merge.
3258
+ */
3259
+ recordToolProgress(name, input) {
3260
+ const level = this.toolProgressLevel();
3261
+ if (level === "off")
3262
+ return;
3263
+ if (this.turnProgress.add(summarizeProgress(name, input, level))) {
3264
+ this.scheduleProgressBroadcast();
3265
+ }
3266
+ }
3267
+ /** Coalesce progress broadcasts: at most one per 3s, trailing edge kept. */
3268
+ scheduleProgressBroadcast() {
3269
+ const MIN_INTERVAL_MS = 3_000;
3270
+ const since = Date.now() - this.lastProgressBroadcastAt;
3271
+ if (since >= MIN_INTERVAL_MS) {
3272
+ this.broadcastToolProgress();
3273
+ return;
3274
+ }
3275
+ if (this.progressBroadcastTimer)
3276
+ return;
3277
+ this.progressBroadcastTimer = setTimeout(() => {
3278
+ this.progressBroadcastTimer = null;
3279
+ this.broadcastToolProgress();
3280
+ }, MIN_INTERVAL_MS - since);
3281
+ this.progressBroadcastTimer.unref?.();
3282
+ }
3283
+ broadcastToolProgress() {
3284
+ const rendered = this.turnProgress.render();
3285
+ if (rendered === this.lastProgressBroadcast)
3286
+ return;
3287
+ this.lastProgressBroadcast = rendered;
3288
+ this.lastProgressBroadcastAt = Date.now();
3289
+ this.ipcServer?.broadcast({
3290
+ type: "instance_progress",
3291
+ instanceName: this.name,
3292
+ progress: rendered,
3293
+ });
3294
+ }
3295
+ /** New turn (or turn over): drop the list and tell the fleet to clear it. */
3296
+ resetToolProgress() {
3297
+ if (this.progressBroadcastTimer) {
3298
+ clearTimeout(this.progressBroadcastTimer);
3299
+ this.progressBroadcastTimer = null;
3300
+ }
3301
+ this.turnProgress.reset();
3302
+ if (this.lastProgressBroadcast !== "") {
3303
+ this.lastProgressBroadcast = "";
3304
+ this.lastProgressBroadcastAt = Date.now();
3305
+ this.ipcServer?.broadcast({ type: "instance_progress", instanceName: this.name, progress: "" });
3306
+ }
3307
+ }
3308
+ /**
3309
+ * Options for every system-initiated paste (startup notice, session snapshot,
3310
+ * runtime-dialog keys).
3311
+ *
3312
+ * One place, so the three call sites cannot drift apart on it. Backends with a
3313
+ * native input queue do not get the retry Enter: on those, a second bare Enter
3314
+ * is not the no-op the retry assumed but a queue mutation — the same reason
3315
+ * `deliverMessage` refuses to probe for busy with one.
3316
+ */
3317
+ systemPasteOptions() {
3318
+ return { retryEnter: this.backend?.supportsQueuedInput?.() !== true };
3319
+ }
3320
+ summarizeTool(name, input) {
3321
+ const inp = input;
3322
+ if (!inp)
3323
+ return name;
3324
+ if (name === "Read")
3325
+ return `Read ${inp.file_path ?? ""}`;
3326
+ if (name === "Edit")
3327
+ return `Edit ${inp.file_path ?? ""}`;
3328
+ if (name === "Write")
3329
+ return `Write ${inp.file_path ?? ""}`;
3330
+ // Command name only — never the arguments. See shellCommandLabel.
3331
+ if (name === "Bash")
3332
+ return `$ ${shellCommandLabel(String(inp.command ?? ""))}`;
3333
+ if (name === "Glob")
3334
+ return `Glob ${inp.pattern ?? ""}`;
3335
+ if (name === "Grep")
3336
+ return `Grep ${inp.pattern ?? ""}`;
3337
+ if (name === "Agent")
3338
+ return "Agent (subagent)";
2452
3339
  if (name.startsWith("mcp__agend__"))
2453
3340
  return ""; // skip channel tools
2454
3341
  return name;
2455
3342
  }
3343
+ /**
3344
+ * The one place an inbound message grows its metadata wrapper ([user:]/
3345
+ * [from:] prefix, pending reactions, handoff metadata, reply instructions).
3346
+ * Both the normal queued path (pushChannelMessage) and /steer go through
3347
+ * here — a steered message must read EXACTLY like a queued one to the
3348
+ * agent, or the two drift apart in what the agent is told about replying.
3349
+ */
3350
+ formatInboundMessage(content, meta) {
3351
+ const user = meta.user || "unknown";
3352
+ const fromInstance = meta.from_instance;
3353
+ let formatted;
3354
+ if (fromInstance) {
3355
+ formatted = formatCrossInstanceInboundMessage(content, meta);
3356
+ }
3357
+ else {
3358
+ const via = meta.source ? ` via ${meta.source}` : "";
3359
+ const idTag = meta.user_id ? `, id:${meta.user_id}` : "";
3360
+ formatted = `[user:${user}${via}${idTag}] ${content}`;
3361
+ // Reactions queued since the last real message (#432). One leading line of
3362
+ // context, present only when something is pending — a reaction no longer
3363
+ // costs a turn of its own.
3364
+ if (meta.pending_reactions) {
3365
+ formatted = `[Recent reactions: ${meta.pending_reactions}]\n${formatted}`;
3366
+ }
3367
+ formatted += renderHandoffMetadata(meta);
3368
+ formatted += "\n(Reply using the reply tool — do NOT respond with direct text)";
3369
+ }
3370
+ if (meta.reply_to_text) {
3371
+ formatted += `\n(reply_to: "${meta.reply_to_text}")`;
3372
+ }
3373
+ return formatted;
3374
+ }
3375
+ /** Tell the fleet when an already-accepted cross-instance pane write failed. */
3376
+ reportCrossInstanceDeliveryFailure(meta, error) {
3377
+ if (!meta.from_instance)
3378
+ return;
3379
+ this.ipcServer?.broadcast({
3380
+ type: "cross_instance_delivery_failed",
3381
+ senderSession: meta.from_instance,
3382
+ targetInstance: this.name,
3383
+ correlationId: meta.correlation_id ?? "unknown",
3384
+ error: error ?? this.tmux?.getLastPasteError?.() ?? "target pane rejected the delivery",
3385
+ });
3386
+ }
3387
+ /**
3388
+ * /steer: interject into the CURRENT turn instead of queueing for idle.
3389
+ *
3390
+ * Differences from pushChannelMessage, and nothing else:
3391
+ * - serialized on steerLock, not pasteLock — it may overtake current work,
3392
+ * but never an ordinary delivery already queued ahead of it
3393
+ * - deliverMessage runs with { steer: true }, which takes the busy branch
3394
+ * that pastes immediately (the codex native-queue transaction) instead of
3395
+ * waiting for idle. Verification and silent-loss fallback are the ones
3396
+ * that path already has: pane-capture visibility check, then one
3397
+ * idle-gated redelivery — so on a TUI that swallows busy input the steer
3398
+ * degrades to "next message after this turn", never silently vanishes.
3399
+ *
3400
+ * The steered text goes through formatInboundMessage so the agent sees a
3401
+ * normal inbound message, with a steering notice prepended for context.
3402
+ */
3403
+ steerMessage(content, meta, deliveryEpoch = this.deliveryEpoch) {
3404
+ if (!this.isDeliveryEpochCurrent(deliveryEpoch))
3405
+ return;
3406
+ // The fleet facade prevents a steer from overtaking an item still in its
3407
+ // idle queue. Close the second race window here: an earlier item may already
3408
+ // have reached this daemon's paste queue by the time the steer IPC arrives.
3409
+ if (this.pasteQueueDepth > 0) {
3410
+ this.logger.info("Steer queued behind an earlier pane delivery to preserve message order");
3411
+ this.pushChannelMessage(content, meta, undefined, deliveryEpoch);
3412
+ return;
3413
+ }
3414
+ this.updateLastChat(meta.chat_id, meta.thread_id, meta.adapter_id);
3415
+ this.pendingWork.recordInbound();
3416
+ this.recordRecentUserMessage(content, meta);
3417
+ const formatted = "[STEERING — mid-task course correction. Fold this into the CURRENT work if one is active.]\n"
3418
+ + this.formatInboundMessage(content, meta);
3419
+ const chatId = meta.chat_id;
3420
+ const messageId = meta.message_id;
3421
+ const status = (chatId && messageId)
3422
+ ? { chatId: meta.thread_id || chatId, messageId }
3423
+ : undefined;
3424
+ this.steerLock = this.steerLock.then(async () => {
3425
+ if (!this.isDeliveryEpochCurrent(deliveryEpoch))
3426
+ return;
3427
+ await this.wake();
3428
+ if (!this.isDeliveryEpochCurrent(deliveryEpoch))
3429
+ return;
3430
+ if (await this.deliverMessage(formatted, status, { steer: true, deliveryEpoch })) {
3431
+ this.markTurnStarted(meta, formatted);
3432
+ }
3433
+ else if (this.isDeliveryEpochCurrent(deliveryEpoch)) {
3434
+ this.reportCrossInstanceDeliveryFailure(meta);
3435
+ }
3436
+ }).catch(err => {
3437
+ this.logger.warn({ err: err.message }, "steer delivery error");
3438
+ if (this.isDeliveryEpochCurrent(deliveryEpoch)) {
3439
+ this.reportCrossInstanceDeliveryFailure(meta, err.message);
3440
+ }
3441
+ });
3442
+ }
3443
+ /**
3444
+ * /btw: submit a separately-labelled side question immediately while asking
3445
+ * the agent not to change the active task. Claude's native `/btw` opens a
3446
+ * branchAndResume panel whose answer never reaches the channel or transcript,
3447
+ * so channel delivery intentionally uses a normal wrapped inbound instead.
3448
+ */
3449
+ btwMessage(content, meta, deliveryEpoch = this.deliveryEpoch) {
3450
+ if (!this.isDeliveryEpochCurrent(deliveryEpoch))
3451
+ return;
3452
+ this.updateLastChat(meta.chat_id, meta.thread_id, meta.adapter_id);
3453
+ this.pendingWork.recordInbound();
3454
+ this.recordRecentUserMessage(content, meta);
3455
+ const formatted = "[BTW — side question from the user. Answer it separately without changing or interrupting the CURRENT work.]\n"
3456
+ + this.formatInboundMessage(content, meta);
3457
+ const chatId = meta.chat_id;
3458
+ const messageId = meta.message_id;
3459
+ const status = (chatId && messageId)
3460
+ ? { chatId: meta.thread_id || chatId, messageId }
3461
+ : undefined;
3462
+ this.steerLock = this.steerLock.then(async () => {
3463
+ if (!this.isDeliveryEpochCurrent(deliveryEpoch))
3464
+ return;
3465
+ await this.wake();
3466
+ if (!this.isDeliveryEpochCurrent(deliveryEpoch))
3467
+ return;
3468
+ if (await this.deliverMessage(formatted, status, { steer: true, deliveryEpoch })) {
3469
+ this.markTurnStarted(meta, formatted);
3470
+ }
3471
+ }).catch(err => {
3472
+ this.logger.warn({ err: err.message }, "btw delivery error");
3473
+ });
3474
+ }
2456
3475
  /**
2457
3476
  * Push an inbound channel message to a specific MCP session.
2458
3477
  * If targetSession is provided, only send to the matching socket.
2459
3478
  * Otherwise send to the instance's own session (this.name).
2460
3479
  */
2461
- pushChannelMessage(content, meta, _targetSession) {
3480
+ pushChannelMessage(content, meta, _targetSession, deliveryEpoch = this.deliveryEpoch) {
3481
+ if (!this.isDeliveryEpochCurrent(deliveryEpoch)) {
3482
+ this.logger.info("Pending channel delivery dropped by user cancel");
3483
+ return;
3484
+ }
2462
3485
  if (!this.tmux) {
2463
3486
  this.logger.warn("Cannot push channel message: tmux not running");
2464
3487
  return;
@@ -2492,46 +3515,19 @@ export class Daemon extends EventEmitter {
2492
3515
  const rawText = content.slice(5);
2493
3516
  this.logger.info({ user }, "Raw paste from topic mode user");
2494
3517
  this.pasteLock = this.pasteLock.then(async () => {
2495
- if (await this.deliverMessage(rawText))
3518
+ if (!this.isDeliveryEpochCurrent(deliveryEpoch))
3519
+ return;
3520
+ if (await this.deliverMessage(rawText, undefined, { deliveryEpoch }))
2496
3521
  this.markTurnStarted(meta, rawText);
2497
3522
  }).catch(err => {
2498
3523
  this.logger.warn({ err: err.message }, "pasteLock raw delivery error");
2499
3524
  });
2500
3525
  return;
2501
3526
  }
2502
- let formatted;
2503
- if (fromInstance) {
2504
- // #77: show the sender's display name for readability, keeping the machine
2505
- // instance name in parens so the recipient's send_to_instance target is valid.
2506
- const fromLabel = meta.from_display ? `${meta.from_display} (${fromInstance})` : fromInstance;
2507
- formatted = `[from:${fromLabel}] ${content}`;
2508
- formatted += renderHandoffMetadata(meta);
2509
- // A delegated task that requires a reply must not read like a chatty FYI —
2510
- // the "you may stay silent" line is for the latter only.
2511
- formatted += meta.requires_reply === "true"
2512
- ? "\n(A reply IS required: use report_result with the correlation_id above — or send_to_instance. Not direct text.)"
2513
- : "\n(If you need to reply, use send_to_instance tool, NOT direct text. If there is nothing to add, you may stay silent.)";
2514
- }
2515
- else {
2516
- const via = meta.source ? ` via ${meta.source}` : "";
2517
- const idTag = meta.user_id ? `, id:${meta.user_id}` : "";
2518
- formatted = `[user:${user}${via}${idTag}] ${content}`;
2519
- // Reactions queued since the last real message (#432). One leading line of
2520
- // context, present only when something is pending — a reaction no longer
2521
- // costs a turn of its own.
2522
- if (meta.pending_reactions) {
2523
- formatted = `[Recent reactions: ${meta.pending_reactions}]\n${formatted}`;
2524
- }
2525
- formatted += renderHandoffMetadata(meta);
2526
- formatted += "\n(Reply using the reply tool — do NOT respond with direct text)";
2527
- }
2528
- if (meta.reply_to_text) {
2529
- formatted += `\n(reply_to: "${meta.reply_to_text}")`;
2530
- }
3527
+ const formatted = this.formatInboundMessage(content, meta);
2531
3528
  // Serialize deliveries: each message waits for the previous to complete,
2532
3529
  // and each waits for the CLI to be idle before pasting. Messages are never
2533
- // dropped for age — a long-busy CLI just queues them until it frees up
2534
- // (the user can press Cancel to interrupt and let the queue drain sooner).
3530
+ // dropped for age — only an explicit user Cancel invalidates queued work.
2535
3531
  const chatId = meta.chat_id;
2536
3532
  const messageId = meta.message_id;
2537
3533
  const wasQueued = this.pasteQueueDepth > 0;
@@ -2544,24 +3540,42 @@ export class Daemon extends EventEmitter {
2544
3540
  }
2545
3541
  this.pasteLock = this.pasteLock.then(async () => {
2546
3542
  try {
3543
+ if (!this.isDeliveryEpochCurrent(deliveryEpoch)) {
3544
+ this.logger.info("Pending channel delivery dropped by user cancel");
3545
+ return;
3546
+ }
2547
3547
  if (this.config.pre_task_command) {
2548
- await this.deliverMessage(this.config.pre_task_command);
3548
+ await this.deliverMessage(this.config.pre_task_command, undefined, { deliveryEpoch });
3549
+ if (!this.isDeliveryEpochCurrent(deliveryEpoch))
3550
+ return;
2549
3551
  }
2550
3552
  if (this.pendingInstructionsNotice) {
2551
3553
  this.pendingInstructionsNotice = false;
2552
- await this.deliverMessage(buildInstructionReloadNotice(this.backend?.binaryName ?? "unknown", this.name, this.instanceDir));
3554
+ await this.deliverMessage(buildInstructionReloadNotice(this.backend?.binaryName ?? "unknown", this.name, this.instanceDir), undefined, { deliveryEpoch });
3555
+ if (!this.isDeliveryEpochCurrent(deliveryEpoch))
3556
+ return;
2553
3557
  }
2554
3558
  const status = (chatId && messageId)
2555
3559
  ? { chatId: meta.thread_id || chatId, messageId }
2556
3560
  : undefined;
2557
- if (await this.deliverMessage(formatted, status))
3561
+ // A fresh delivery begins a fresh turn — its bubble must not inherit
3562
+ // the previous turn's tool list.
3563
+ this.resetToolProgress();
3564
+ if (await this.deliverMessage(formatted, status, { deliveryEpoch })) {
2558
3565
  this.markTurnStarted(meta, formatted);
3566
+ }
3567
+ else if (meta.from_instance && this.isDeliveryEpochCurrent(deliveryEpoch)) {
3568
+ this.reportCrossInstanceDeliveryFailure(meta);
3569
+ }
2559
3570
  }
2560
3571
  finally {
2561
3572
  this.pasteQueueDepth--;
2562
3573
  }
2563
3574
  }).catch(err => {
2564
3575
  this.logger.warn({ err: err.message }, "pasteLock delivery error — chain continues");
3576
+ if (this.isDeliveryEpochCurrent(deliveryEpoch)) {
3577
+ this.reportCrossInstanceDeliveryFailure(meta, err.message);
3578
+ }
2565
3579
  });
2566
3580
  this.logger.debug({ user: meta.user, text: content.slice(0, 100) }, "Queued channel message for delivery");
2567
3581
  }
@@ -2581,31 +3595,67 @@ export class Daemon extends EventEmitter {
2581
3595
  * immediately and own the application-level queue themselves. The pasteLock remains
2582
3596
  * serial in both cases so separate PTY writes can never overlap.
2583
3597
  */
2584
- async deliverMessage(formatted, status) {
3598
+ async deliverMessage(formatted, status, opts) {
3599
+ const cancelled = () => opts?.deliveryEpoch !== undefined
3600
+ && !this.isDeliveryEpochCurrent(opts.deliveryEpoch);
3601
+ if (cancelled() || this.stormWindow?.isStopped())
3602
+ return false;
2585
3603
  // Sanitize unclosed code fences — they cause CLI to wait for closure on Enter
2586
3604
  const fenceCount = (formatted.match(/```/g) || []).length;
2587
3605
  if (fenceCount % 2 !== 0) {
2588
3606
  // Odd number of fences = unclosed. Remove all code fences from the message.
2589
3607
  formatted = formatted.replace(/```/g, "");
2590
3608
  }
3609
+ // Direct paths such as /steer, /btw, /raw and schedules do not pass the
3610
+ // FleetManager idle gate. Hold them at the final common pane-write path so
3611
+ // a tmux storm cannot force any text into a half-booted CLI.
3612
+ if (this.stormWindow?.isDeliveryHeld(this.name)) {
3613
+ this.logger.warn("Pane delivery held during tmux server recovery");
3614
+ await this.stormWindow.waitForDeliveryAllowed(this.name);
3615
+ if (cancelled() || this.stormWindow.isStopped())
3616
+ return false;
3617
+ }
2591
3618
  // Before anything reads the window id: a spawn in progress is about to change it.
2592
3619
  await this.waitForSpawnToSettle();
3620
+ if (cancelled())
3621
+ return false;
3622
+ if (this.refuseFatalStartupDelivery(status))
3623
+ return false;
2593
3624
  let windowId = this.getWindowId();
2594
3625
  const supportsQueuedInput = this.backend?.supportsQueuedInput?.() === true;
2595
3626
  let handingOffToNativeQueue = false;
2596
3627
  // If the CLI is busy, either hand the complete submission to its native input
2597
3628
  // queue or wait for idle. Native queue support is an explicit backend capability:
2598
3629
  // normal Enter input has steering/interrupt semantics in several other CLIs.
2599
- if (windowId && this.controlClient && !this.controlClient.isIdle(windowId)) {
3630
+ let readiness = "ready";
3631
+ const gateWindowId = windowId;
3632
+ if (gateWindowId && this.controlClient)
3633
+ readiness = await this.paneReadinessForDelivery(gateWindowId);
3634
+ if (readiness !== "ready") {
2600
3635
  if (status)
2601
3636
  this.emit("message_queued", status);
2602
- if (supportsQueuedInput) {
3637
+ // Only a pane that is busy AND verifiably free of a blocking dialog may
3638
+ // be handed to the native input queue or steered — those paths paste and
3639
+ // press Enter at once. A dialog painted a moment ago is still "busy" to
3640
+ // the silence gate, so the probe is mandatory here; "dialog"/"unknown"
3641
+ // fall through to the wait.
3642
+ const canHandOff = (supportsQueuedInput || opts?.steer)
3643
+ && readiness === "busy"
3644
+ && (await this.probeBlockingDialog()).state === "clear";
3645
+ if (canHandOff) {
3646
+ // Native queue (codex), or an explicit /steer: hand the complete
3647
+ // paste+Enter transaction to the busy CLI now. For steer this is the
3648
+ // point — the user asked to interject, and the transaction's
3649
+ // visibility check + idle-gated fallback catch TUIs that swallow
3650
+ // busy input (see steerMessage).
2603
3651
  handingOffToNativeQueue = true;
2604
- this.logger.debug("CLI busy — handing message to backend-native input queue");
3652
+ this.logger.debug(opts?.steer ? "CLI busy — steering into the running turn" : "CLI busy — handing message to backend-native input queue");
2605
3653
  }
2606
3654
  else {
2607
3655
  this.logger.debug("CLI busy — queuing message until idle");
2608
- const becameIdle = await this.controlClient.waitUntilIdle(windowId);
3656
+ const becameIdle = await this.waitForPaneReadyForDelivery(gateWindowId);
3657
+ if (cancelled())
3658
+ return false;
2609
3659
  if (!becameIdle) {
2610
3660
  // The pane never freed up. Report the failure instead of pasting into a
2611
3661
  // wedged CLI (where the text would sit unsubmitted and the next message
@@ -2617,11 +3667,459 @@ export class Daemon extends EventEmitter {
2617
3667
  }
2618
3668
  }
2619
3669
  }
3670
+ // F3 (#kiro-enter-drop): an earlier delivery's Enter may have been dropped,
3671
+ // leaving that message in the input row. Pasting on top would submit both as
3672
+ // one. Submit the stranded text first (a bare Enter — a no-op on an empty
3673
+ // prompt, verified live), then wait for the prompt to come back before our
3674
+ // own paste. The wait happens OUTSIDE the pane lock like every other wait.
3675
+ if (windowId && this.controlClient && !handingOffToNativeQueue) {
3676
+ // Only a positively verified clear input row ("idle") lets the paste
3677
+ // proceed. After "submitted" (the stranded text's turn is running) or
3678
+ // "busy" (the pane changed under us, or could not be read) the prompt
3679
+ // must come back AND the check runs again — the stranded text may still
3680
+ // be there. "failed" (the Enter could not be sent) fails the delivery:
3681
+ // pasting now would submit both messages as one.
3682
+ for (let round = 0;; round++) {
3683
+ // Budget check BEFORE the next mutating call: each round may send one
3684
+ // recovery Enter, so exactly STRANDED_INPUT_MAX_ROUNDS of them go out.
3685
+ if (round >= STRANDED_INPUT_MAX_ROUNDS) {
3686
+ this.logger.error({ round }, "Input row still not clear after the stranded-text recovery budget — reporting delivery failure");
3687
+ if (status)
3688
+ this.emit("message_failed", status); // ❌
3689
+ return false;
3690
+ }
3691
+ const stranded = await this.submitStrandedInputIfAny(windowId);
3692
+ if (cancelled())
3693
+ return false;
3694
+ if (stranded === "idle")
3695
+ break;
3696
+ if (stranded === "failed") {
3697
+ this.logger.error({ round }, "Could not submit the stranded text — reporting delivery failure");
3698
+ if (status)
3699
+ this.emit("message_failed", status); // ❌
3700
+ return false;
3701
+ }
3702
+ const readyAgain = await this.waitForPaneReadyForDelivery(windowId);
3703
+ if (cancelled())
3704
+ return false;
3705
+ if (!readyAgain) {
3706
+ this.logger.error("Pane never returned to its prompt after submitting stranded input — reporting delivery failure");
3707
+ if (status)
3708
+ this.emit("message_failed", status); // ❌
3709
+ return false;
3710
+ }
3711
+ }
3712
+ }
2620
3713
  // Everything above is *waiting*; everything below *writes*. Only the write is
2621
3714
  // held under the pane lock — holding it across the idle wait (up to 30 min)
2622
3715
  // would starve the runtime-dialog dismisser, which is often the very thing
2623
3716
  // that would let the pane go idle again.
2624
- return this.paneWriteLock.run(() => this.writeMessageToPane(formatted, windowId, handingOffToNativeQueue, status));
3717
+ for (let round = 0;; round++) {
3718
+ const outcome = await this.paneWriteLock.run(async () => {
3719
+ if (cancelled())
3720
+ return false;
3721
+ if (this.refuseFatalStartupDelivery(status))
3722
+ return false;
3723
+ // TOCTOU: the probes above ran outside this lock, and the CLI repaints
3724
+ // whenever it likes — a resume prompt can be painted between "clear" and
3725
+ // this critical section. Re-probe here, right before the write; a
3726
+ // dialog or an unreadable pane means "not now", never "paste anyway".
3727
+ if (this.deliveryBlockingDialogs().length > 0) {
3728
+ const probe = await this.probeBlockingDialog();
3729
+ if (probe.state !== "clear")
3730
+ return "dialog";
3731
+ }
3732
+ return this.writeMessageToPane(formatted, windowId, handingOffToNativeQueue, status);
3733
+ });
3734
+ if (outcome !== "dialog")
3735
+ return outcome;
3736
+ // Wait OUTSIDE the lock (holding it would starve the runtime dismisser),
3737
+ // then try the write again — bounded, so a dialog nobody answers ends in
3738
+ // an honest failure rather than a message that never lands.
3739
+ if (round + 1 >= LATE_DIALOG_WRITE_ROUNDS) {
3740
+ this.logger.error({ rounds: round + 1 }, "A dialog kept appearing before the pane write — reporting delivery failure");
3741
+ if (status)
3742
+ this.emit("message_failed", status); // ❌
3743
+ return false;
3744
+ }
3745
+ this.logger.info("Dialog appeared before the pane write — waiting for it to clear");
3746
+ const clear = gateWindowId ? await this.waitForPaneReadyForDelivery(gateWindowId) : false;
3747
+ if (cancelled())
3748
+ return false;
3749
+ if (!clear) {
3750
+ if (status)
3751
+ this.emit("message_failed", status); // ❌
3752
+ return false;
3753
+ }
3754
+ }
3755
+ }
3756
+ /**
3757
+ * The runtime dialogs that hold delivery while on screen (`blocksDelivery`
3758
+ * or `holdOnly`). One canonical list and order, shared by the delivery gate
3759
+ * and the runtime scanner, so both observers agree on which dialog they see.
3760
+ * Startup-only patterns are deliberately excluded: several of them are loose
3761
+ * on purpose ("I trust", "Yes, continue") and would match ordinary prose.
3762
+ */
3763
+ deliveryBlockingDialogs() {
3764
+ return (this.backend?.getRuntimeDialogs?.() ?? []).filter(d => d.blocksDelivery || d.holdOnly);
3765
+ }
3766
+ static dialogKey(dialog) {
3767
+ return `${dialog.pattern.source}/${dialog.pattern.flags}`;
3768
+ }
3769
+ /** A dialog counts only when it is the CURRENT interactive region, if the backend can tell; else by pattern. */
3770
+ static dialogMatches(dialog, pane) {
3771
+ return dialog.isActive ? dialog.isActive(pane) : dialog.pattern.test(pane);
3772
+ }
3773
+ /**
3774
+ * Tri-state liveness. tmux's own isWindowAlive folds every query failure
3775
+ * into `false`, so a transient tmux hiccup would read as "the CLI died" —
3776
+ * and callers that clear the session on death must not act on that.
3777
+ * getPaneStatus returns null on a failed/ambiguous query and a definite
3778
+ * `{alive:false}` for a dead pane; only the latter is evidence.
3779
+ */
3780
+ async paneLiveness() {
3781
+ if (!this.tmux)
3782
+ return "unknown";
3783
+ const tmux = this.tmux;
3784
+ if (typeof tmux.getPaneStatus === "function") {
3785
+ try {
3786
+ const status = await tmux.getPaneStatus();
3787
+ if (status === null)
3788
+ return "unknown";
3789
+ return status.alive ? "alive" : "dead";
3790
+ }
3791
+ catch {
3792
+ return "unknown";
3793
+ }
3794
+ }
3795
+ try {
3796
+ return (await tmux.isWindowAlive()) ? "alive" : "dead";
3797
+ }
3798
+ catch {
3799
+ return "unknown";
3800
+ }
3801
+ }
3802
+ /**
3803
+ * Capture the pane and classify it. "unknown" (capture failed) is a distinct
3804
+ * outcome on purpose: the resume dialog is silent, so silence cannot vouch
3805
+ * for the pane, and treating an unreadable pane as clear is exactly the
3806
+ * fail-open that answers a dialog with its default. Callers retry unknown
3807
+ * within a bound and then fail loudly.
3808
+ */
3809
+ async probeBlockingDialog() {
3810
+ if (!this.tmux || !this.backend)
3811
+ return { state: "clear" };
3812
+ const dialogs = this.deliveryBlockingDialogs();
3813
+ if (dialogs.length === 0)
3814
+ return { state: "clear" };
3815
+ let pane;
3816
+ try {
3817
+ pane = await this.tmux.capturePane();
3818
+ }
3819
+ catch (err) {
3820
+ this.logger.debug({ err }, "capture-pane failed during the dialog probe — pane state unknown");
3821
+ return { state: "unknown" };
3822
+ }
3823
+ for (const dialog of dialogs) {
3824
+ if (Daemon.dialogMatches(dialog, pane)) {
3825
+ this.trackDialogParked(dialog);
3826
+ return { state: "dialog", dialog };
3827
+ }
3828
+ }
3829
+ this.trackDialogParked(null);
3830
+ return { state: "clear" };
3831
+ }
3832
+ /**
3833
+ * Remember how long a blocking dialog has been on screen. Once it has
3834
+ * outlived DIALOG_PARKED_NOTIFY_MS — the auto-dismiss did not take, or the
3835
+ * dialog is a hold-only variant — report it ONCE so a human answers it.
3836
+ * Never presses a key: the whole point is that we do not know which option
3837
+ * is safe. Identity is the pattern, so the delivery gate and the runtime
3838
+ * scanner (same canonical list) never reset each other's clock.
3839
+ */
3840
+ trackDialogParked(dialog) {
3841
+ if (!dialog) {
3842
+ if (this.dialogParkedSince !== 0)
3843
+ this.logger.info({ dialog: this.dialogParkedKey }, "Blocking dialog is gone from the pane");
3844
+ this.dialogParkedSince = 0;
3845
+ this.dialogParkedKey = null;
3846
+ this.dialogParkedReported = false;
3847
+ return;
3848
+ }
3849
+ const key = Daemon.dialogKey(dialog);
3850
+ if (this.dialogParkedSince === 0 || this.dialogParkedKey !== key) {
3851
+ this.dialogParkedSince = Date.now();
3852
+ this.dialogParkedKey = key;
3853
+ this.dialogParkedReported = false;
3854
+ return;
3855
+ }
3856
+ if (!this.dialogParkedReported && Date.now() - this.dialogParkedSince >= DIALOG_PARKED_NOTIFY_MS) {
3857
+ this.dialogParkedReported = true;
3858
+ this.logger.warn({ dialog: dialog.description, parkedForMs: Date.now() - this.dialogParkedSince }, "CLI dialog is still on screen — not auto-answering, reporting for a human");
3859
+ this.emit("dialog_parked", { name: this.name, description: dialog.description, holdOnly: dialog.holdOnly === true });
3860
+ }
3861
+ }
3862
+ /**
3863
+ * Bottom-anchored readiness for TUIs that drop Enter while busy (see
3864
+ * CliBackend.dropsEnterWhileBusy). Output silence is necessary but not
3865
+ * sufficient for them: Kiro paints nothing for the whole duration of a shell
3866
+ * or MCP tool call (and for each 10s backend retry), yet an Enter arriving
3867
+ * then is discarded. Reproduced deterministically: `sleep 9` tool, paste at
3868
+ * t+6s → text stranded in the prompt row, next message submitted both.
3869
+ * Other backends keep the silence-based gate byte-for-byte.
3870
+ */
3871
+ async isPaneReadyForDelivery(windowId) {
3872
+ return (await this.paneReadinessForDelivery(windowId)) === "ready";
3873
+ }
3874
+ /**
3875
+ * Why a pane is not deliverable matters. "busy" may be handed to a native
3876
+ * input queue or steered (after its own dialog probe); "dialog" and "unknown"
3877
+ * must never be — a paste + Enter into a dialog answers it with its default,
3878
+ * and an unreadable pane proves nothing. The silence gate comes first and
3879
+ * costs no capture, so a CLI that is simply working never pays for the probe.
3880
+ */
3881
+ async paneReadinessForDelivery(windowId) {
3882
+ if (!this.isPaneIdleForDelivery(windowId))
3883
+ return "busy";
3884
+ const probe = await this.probeBlockingDialog();
3885
+ if (probe.state !== "clear")
3886
+ return probe.state;
3887
+ if (this.backend?.dropsEnterWhileBusy?.() !== true)
3888
+ return "ready";
3889
+ const prompt = this.backend.getBottomReadyPattern?.();
3890
+ if (!prompt || !this.tmux)
3891
+ return "ready";
3892
+ try {
3893
+ const pane = await this.tmux.capturePane();
3894
+ if (this.backend.getBusyPattern?.()?.test(pane))
3895
+ return "busy";
3896
+ if (bottomRowIsReady(pane, prompt))
3897
+ return "ready";
3898
+ // FAIL CLOSED on any non-blank screen without the prompt at the bottom.
3899
+ // The obvious "unknown layout → fall back to silence" shortcut re-opened
3900
+ // the bug: a long tool output scrolls the turn's own `N% !>` row out of
3901
+ // the visible pane (capture-pane -p is the viewport only), the bottom row
3902
+ // is tool output, the pane is quiet — and a paste there strands the text
3903
+ // exactly as before. The prompt row is the only evidence Kiro accepts an
3904
+ // Enter; a modal or panel is handled by the dialog dismisser, and the
3905
+ // bounded wait reports a failure rather than pasting into a screen we do
3906
+ // not understand. That includes a completely BLANK capture: a clear /
3907
+ // redraw / alternate-screen transition paints nothing for a frame, tmux
3908
+ // still accepts a paste, and "nothing visible" is not evidence the TUI is
3909
+ // idle. The next 250ms poll sees the prompt if it is coming.
3910
+ return "busy";
3911
+ }
3912
+ catch (err) {
3913
+ // Same rule for a capture that fails: no positive proof, no paste. The
3914
+ // poll retries; a persistently unreadable pane ends in the bounded
3915
+ // timeout's ❌ rather than in a silently stranded message.
3916
+ this.logger.debug({ err }, "capture-pane failed during the readiness check — treating the pane as not ready");
3917
+ return "busy";
3918
+ }
3919
+ }
3920
+ /**
3921
+ * Like waitForPaneIdleForDelivery, plus the bottom-anchored prompt check for
3922
+ * Enter-dropping TUIs. Polls at a coarse cadence (each poll is a capture-pane)
3923
+ * only after the pane is already quiet, so the common fast-idle delivery still
3924
+ * costs a single capture.
3925
+ */
3926
+ async waitForPaneReadyForDelivery(windowId, timeoutMs = 30 * 60_000) {
3927
+ // Every backend polls: readiness now also means "no blocking dialog on
3928
+ // screen", and a dialog disappears without any output edge the silence
3929
+ // gate could see.
3930
+ const deadline = Date.now() + timeoutMs;
3931
+ const bottomGated = this.backend?.dropsEnterWhileBusy?.() === true;
3932
+ let unknownStreak = 0;
3933
+ for (;;) {
3934
+ const remaining = deadline - Date.now();
3935
+ if (remaining <= 0)
3936
+ return false;
3937
+ const idle = await this.waitForPaneIdleForDelivery(windowId, remaining);
3938
+ if (!idle)
3939
+ return false;
3940
+ // The idle wait just resolved, so re-reading the silence gate would only
3941
+ // add a stale-cache race; ask the pane the remaining questions directly.
3942
+ if (bottomGated) {
3943
+ if (await this.isPaneReadyForDelivery(windowId))
3944
+ return true;
3945
+ }
3946
+ else {
3947
+ const probe = await this.probeBlockingDialog();
3948
+ if (probe.state === "clear")
3949
+ return true;
3950
+ if (probe.state === "unknown") {
3951
+ if (++unknownStreak >= DIALOG_PROBE_UNKNOWN_MAX) {
3952
+ this.logger.error({ probes: unknownStreak }, "Pane stayed unreadable during the dialog probe — refusing to deliver blind");
3953
+ return false;
3954
+ }
3955
+ }
3956
+ else {
3957
+ unknownStreak = 0;
3958
+ }
3959
+ }
3960
+ await new Promise(r => setTimeout(r, BOTTOM_READY_POLL_MS));
3961
+ }
3962
+ }
3963
+ /**
3964
+ * F3: if an AgEnD message is sitting unsubmitted in the input row (a previous
3965
+ * delivery's Enter was dropped), submit it with one bare Enter under the pane
3966
+ * lock. Only acts on a positive signature — the `[user:`/`[from:` marker every
3967
+ * pasted message carries — never on Kiro's own placeholder hints.
3968
+ *
3969
+ * Outcomes: "idle" = positively verified nothing stranded (the caller may
3970
+ * paste); "submitted" = an Enter went out for the stranded text; "busy" = the
3971
+ * pane is not at its prompt or could not be read (the caller must wait for
3972
+ * the prompt and run this check AGAIN before pasting — never assume someone
3973
+ * else submitted it); "failed" = the Enter could not be sent (the delivery
3974
+ * must fail rather than paste on top of the stranded text).
3975
+ */
3976
+ async submitStrandedInputIfAny(windowId) {
3977
+ if (this.backend?.dropsEnterWhileBusy?.() !== true || !this.tmux)
3978
+ return "idle";
3979
+ const prompt = this.backend.getBottomReadyPattern?.();
3980
+ if (!prompt)
3981
+ return "idle";
3982
+ // Pre-check outside the lock so the common case (verified clear) never
3983
+ // contends with the dialog dismisser. Only a POSITIVE "clear" skips the
3984
+ // lock; an unreadable pane is re-examined under it, not waved through.
3985
+ const pre = await this.strandedInputState(prompt);
3986
+ if (pre === "clear")
3987
+ return "idle";
3988
+ if (pre === "busy")
3989
+ return "busy";
3990
+ if (pre === "stranded") {
3991
+ this.logger.warn({ windowId }, "Unsubmitted message found in the input row — submitting it before this delivery");
3992
+ }
3993
+ return this.paneWriteLock.run(async () => {
3994
+ // Re-check UNDER the lock right before the key goes out: between the
3995
+ // pre-check and acquiring the lock the CLI may have started generating
3996
+ // (the stranded text got submitted by someone else, or the dismisser
3997
+ // acted), or a fatal startup screen may have come up. A bare Enter is a
3998
+ // no-op at an empty prompt, but into a modal it is a confirmation.
3999
+ if (this.fatalStartupBlocked)
4000
+ return "busy";
4001
+ const state = await this.strandedInputState(prompt);
4002
+ if (state === "clear")
4003
+ return "idle";
4004
+ if (state !== "stranded")
4005
+ return "busy"; // busy, or unreadable: no evidence the prompt is free
4006
+ const sent = await this.sendDeliveryEnter("stranded-input-submit");
4007
+ return sent ? "submitted" : "failed";
4008
+ });
4009
+ }
4010
+ /**
4011
+ * "stranded": an AgEnD message sits in the input row; "clear": the prompt is
4012
+ * at the bottom with no stranded text; "busy": generating, or the prompt is
4013
+ * not at the bottom (incl. a blank frame); "unknown": the pane could not be
4014
+ * read. Only "clear" is positive evidence.
4015
+ */
4016
+ async strandedInputState(prompt) {
4017
+ let pane;
4018
+ try {
4019
+ pane = await this.tmux.capturePane();
4020
+ }
4021
+ catch {
4022
+ return "unknown";
4023
+ }
4024
+ if (this.backend?.getBusyPattern?.()?.test(pane))
4025
+ return "busy";
4026
+ if (strandedAgendMessageInInput(pane, prompt))
4027
+ return "stranded";
4028
+ return bottomRowIsReady(pane, prompt) ? "clear" : "busy";
4029
+ }
4030
+ /**
4031
+ * F2: for Enter-dropping TUIs, "some output after Enter" proves nothing — the
4032
+ * turn that made the pane look idle keeps producing output. Confirm the
4033
+ * SUBMISSION instead: the pasted text is gone from the input row ("absent"),
4034
+ * or the TUI's own busy marker is up. "unknown" = the pane could not be read;
4035
+ * it never counts as confirmed.
4036
+ */
4037
+ async pasteResidue(formatted) {
4038
+ const prompt = this.backend?.getBottomReadyPattern?.();
4039
+ if (!prompt || !this.tmux)
4040
+ return "absent";
4041
+ let pane;
4042
+ try {
4043
+ pane = await this.tmux.capturePane();
4044
+ }
4045
+ catch {
4046
+ return "unknown";
4047
+ }
4048
+ if (this.backend?.getBusyPattern?.()?.test(pane))
4049
+ return "absent"; // generating = submitted
4050
+ return pasteLeftInInput(pane, prompt, formatted) ? "present" : "absent";
4051
+ }
4052
+ /**
4053
+ * Submission = the legacy idle→busy output signal AND the pasted text has
4054
+ * verifiably left the input row. The first alone is what produced false ✅s:
4055
+ * the turn that made the pane look idle keeps emitting output after our
4056
+ * (dropped) Enter — and an unreadable pane is not a verified absence either.
4057
+ */
4058
+ async confirmSubmittedAfterEnter(windowId, enterAt, formatted) {
4059
+ const sawOutput = await this.confirmBusyAfterEnter(windowId, enterAt);
4060
+ const residue = await this.pasteResidue(formatted);
4061
+ if (residue === "unknown")
4062
+ this.logger.warn("Could not read the pane after Enter — not confirming the submission");
4063
+ return sawOutput && residue === "absent";
4064
+ }
4065
+ /**
4066
+ * While the CLI is parked on a fatal startup screen (see fatalStartupBlocked),
4067
+ * no pane write may happen: paste+Enter into the corrupt-config modal would
4068
+ * confirm "Exit and fix manually" and exit the CLI. The message is failed (❌)
4069
+ * rather than held — the accompanying config_error notification tells the user
4070
+ * what to fix, and holding would only build a queue that floods the pane the
4071
+ * moment the flag clears.
4072
+ */
4073
+ refuseFatalStartupDelivery(status) {
4074
+ if (!this.fatalStartupBlocked)
4075
+ return false;
4076
+ this.logger.error("Delivery refused — CLI is parked on a fatal startup screen");
4077
+ if (status)
4078
+ this.emit("message_failed", status); // ❌
4079
+ return true;
4080
+ }
4081
+ /**
4082
+ * A raw `%output` silence check is authoritative for ordinary TUIs. It is not
4083
+ * for a backend such as agy whose status-line hook periodically redraws an
4084
+ * identical screen. In that case trust an idle pane-state snapshot only when
4085
+ * it was captured after both the latest output and the latest control-client
4086
+ * observation reset.
4087
+ */
4088
+ isPaneIdleForDelivery(windowId) {
4089
+ if (!this.controlClient)
4090
+ return true;
4091
+ if (this.controlClient.isIdle(windowId))
4092
+ return true;
4093
+ if (this.backend?.hasPeriodicPaneRedraw?.() !== true || this.instanceState !== "idle")
4094
+ return false;
4095
+ const lastOutputAt = this.controlClient.getLastOutputAt(windowId);
4096
+ if (lastOutputAt === undefined || !this.instanceStateMachine)
4097
+ return false;
4098
+ const observedAt = this.instanceStateMachine.snapshot().observedAt;
4099
+ return observedAt >= lastOutputAt
4100
+ && observedAt >= this.controlClient.getObservationResetAt();
4101
+ }
4102
+ waitForPaneIdleForDelivery(windowId, timeoutMs = 30 * 60_000) {
4103
+ if (!this.controlClient)
4104
+ return Promise.resolve(true);
4105
+ if (this.backend?.hasPeriodicPaneRedraw?.() !== true) {
4106
+ return this.controlClient.waitUntilIdle(windowId, timeoutMs);
4107
+ }
4108
+ if (this.isPaneIdleForDelivery(windowId))
4109
+ return Promise.resolve(true);
4110
+ return new Promise(resolve => {
4111
+ const check = setInterval(() => {
4112
+ if (!this.isPaneIdleForDelivery(windowId))
4113
+ return;
4114
+ clearInterval(check);
4115
+ clearTimeout(timer);
4116
+ resolve(true);
4117
+ }, 50);
4118
+ const timer = setTimeout(() => {
4119
+ clearInterval(check);
4120
+ resolve(false);
4121
+ }, timeoutMs);
4122
+ });
2625
4123
  }
2626
4124
  /**
2627
4125
  * The write half of a delivery: paste → Enter → confirm, with retries.
@@ -2649,7 +4147,16 @@ export class Daemon extends EventEmitter {
2649
4147
  const pasteStartedAt = Date.now();
2650
4148
  const pasted = await this.tmux.pasteBuffer(formatted);
2651
4149
  if (!pasted) {
2652
- this.logger.warn({ attempt }, "pasteBuffer failed — recovering window and backing off");
4150
+ const tmuxError = this.tmux.getLastPasteError?.() ?? "unknown tmux paste failure";
4151
+ const recoverable = this.tmux.isLastPasteFailureRecoverable?.() ?? true;
4152
+ this.logger[recoverable ? "warn" : "error"]({ attempt, tmuxError, recoverable }, recoverable
4153
+ ? "pasteBuffer failed — recovering window and backing off"
4154
+ : "pasteBuffer failed — non-retryable tmux error");
4155
+ if (!recoverable) {
4156
+ if (status)
4157
+ this.emit("message_failed", status);
4158
+ return false;
4159
+ }
2653
4160
  windowId = (await this.recoverWindow()) ?? windowId;
2654
4161
  if (attempt < maxAttempts)
2655
4162
  await new Promise(r => setTimeout(r, 2000));
@@ -2731,7 +4238,9 @@ export class Daemon extends EventEmitter {
2731
4238
  }
2732
4239
  const repasted = await this.tmux.pasteBuffer(formatted);
2733
4240
  if (!repasted) {
2734
- this.logger.error("Idle-gated redelivery paste failed after native-queue silent loss");
4241
+ this.logger.error({
4242
+ tmuxError: this.tmux.getLastPasteError?.() ?? "unknown tmux paste failure",
4243
+ }, "Idle-gated redelivery paste failed after native-queue silent loss");
2735
4244
  if (status)
2736
4245
  this.emit("message_failed", status); // ❌
2737
4246
  return false;
@@ -2771,7 +4280,36 @@ export class Daemon extends EventEmitter {
2771
4280
  this.emit("message_failed", status); // ❌
2772
4281
  return false;
2773
4282
  }
2774
- if (windowId && this.controlClient) {
4283
+ if (windowId && this.controlClient && this.backend?.dropsEnterWhileBusy?.() === true) {
4284
+ // F2: output after Enter is necessary but not sufficient — the paste
4285
+ // must also have LEFT the input row.
4286
+ let submitted = await this.confirmSubmittedAfterEnter(windowId, enterAt, formatted);
4287
+ if (!submitted) {
4288
+ // The Enter landed while the TUI was still busy (it kept the text but
4289
+ // dropped the key). A retry is only useful once the prompt is back —
4290
+ // bounded, since we hold the pane lock here.
4291
+ this.logger.warn("Message not submitted after Enter — waiting for the prompt, then re-sending Enter once");
4292
+ const promptBack = await this.waitForPaneReadyForDelivery(windowId, STRANDED_RETRY_READY_WAIT_MS);
4293
+ const retryAt = Date.now();
4294
+ if (promptBack && !(await this.sendDeliveryEnter("stranded-text-retry"))) {
4295
+ if (status)
4296
+ this.emit("message_failed", status); // ❌
4297
+ return false;
4298
+ }
4299
+ submitted = promptBack && await this.confirmSubmittedAfterEnter(windowId, retryAt, formatted);
4300
+ }
4301
+ if (submitted) {
4302
+ if (status)
4303
+ this.emit("message_confirmed", status); // ✅
4304
+ }
4305
+ else {
4306
+ this.logger.error("Message pasted but never submitted (text still in the input row after Enter retry)");
4307
+ if (status)
4308
+ this.emit("message_failed", status); // ❌
4309
+ return false;
4310
+ }
4311
+ }
4312
+ else if (windowId && this.controlClient) {
2775
4313
  let becameBusy = await this.confirmBusyAfterEnter(windowId, enterAt);
2776
4314
  if (!becameBusy) {
2777
4315
  this.logger.warn("No idle→busy transition after Enter — re-sending Enter once");
@@ -2931,8 +4469,13 @@ export class Daemon extends EventEmitter {
2931
4469
  // for this turn: the agent proved it can still speak for itself.
2932
4470
  if (!error && result != null && TURN_REPLY_TOOLS.has(tool)) {
2933
4471
  this.turnOutboundDelivered = true;
4472
+ if (tool === "reply")
4473
+ this.turnReplyDelivered = true;
4474
+ }
4475
+ const sent = this.ipcServer?.send(socket, { requestId, result, error }) ?? false;
4476
+ if (!sent) {
4477
+ this.logger.warn({ tool, requestId }, "Tool result could not be returned — originating MCP socket is closed");
2934
4478
  }
2935
- this.ipcServer?.send(socket, { requestId, result, error });
2936
4479
  };
2937
4480
  // Repo checkout — handled locally in daemon (no fleet-manager)
2938
4481
  if (tool === "checkout_repo") {
@@ -2945,13 +4488,7 @@ export class Daemon extends EventEmitter {
2945
4488
  }
2946
4489
  if (tool === "set_display_name" || tool === "set_description") {
2947
4490
  const type = tool === "set_display_name" ? "fleet_set_display_name" : "fleet_set_description";
2948
- const fleetReqId = `${tool === "set_display_name" ? "dn" : "desc"}_${requestId}`;
2949
- this.ipcServer?.broadcast({
2950
- type,
2951
- payload: args,
2952
- meta: { instance_name: this.name },
2953
- fleetRequestId: fleetReqId,
2954
- });
4491
+ const fleetReqId = `${tool === "set_display_name" ? "dn" : "desc"}_${++this.fleetRequestSeq}_${requestId}`;
2955
4492
  const timeout = setTimeout(() => {
2956
4493
  this.pendingIpcRequests.delete(fleetReqId);
2957
4494
  respond(null, `${tool} timed out`);
@@ -2960,16 +4497,16 @@ export class Daemon extends EventEmitter {
2960
4497
  clearTimeout(timeout);
2961
4498
  respond(respMsg.result, respMsg.error);
2962
4499
  });
2963
- return;
2964
- }
2965
- if (tool === TASK_TOOL) {
2966
- const fleetReqId = `task_${requestId}`;
2967
4500
  this.ipcServer?.broadcast({
2968
- type: "fleet_task",
4501
+ type,
2969
4502
  payload: args,
2970
4503
  meta: { instance_name: this.name },
2971
4504
  fleetRequestId: fleetReqId,
2972
4505
  });
4506
+ return;
4507
+ }
4508
+ if (tool === TASK_TOOL) {
4509
+ const fleetReqId = `task_${++this.fleetRequestSeq}_${requestId}`;
2973
4510
  const timeout = setTimeout(() => {
2974
4511
  this.pendingIpcRequests.delete(fleetReqId);
2975
4512
  respond(null, `Task operation timed out after ${daemonBudgetMs(tool) / 1000}s`);
@@ -2978,6 +4515,12 @@ export class Daemon extends EventEmitter {
2978
4515
  clearTimeout(timeout);
2979
4516
  respond(respMsg.result, respMsg.error);
2980
4517
  });
4518
+ this.ipcServer?.broadcast({
4519
+ type: "fleet_task",
4520
+ payload: args,
4521
+ meta: { instance_name: this.name },
4522
+ fleetRequestId: fleetReqId,
4523
+ });
2981
4524
  return;
2982
4525
  }
2983
4526
  if (DECISION_TOOLS.has(tool)) {
@@ -2986,13 +4529,7 @@ export class Daemon extends EventEmitter {
2986
4529
  list_decisions: "fleet_decision_list",
2987
4530
  update_decision: "fleet_decision_update",
2988
4531
  };
2989
- const fleetReqId = `dec_${requestId}`;
2990
- this.ipcServer?.broadcast({
2991
- type: typeMap[tool],
2992
- payload: args,
2993
- meta: { instance_name: this.name, working_directory: this.config.working_directory },
2994
- fleetRequestId: fleetReqId,
2995
- });
4532
+ const fleetReqId = `dec_${++this.fleetRequestSeq}_${requestId}`;
2996
4533
  const timeout = setTimeout(() => {
2997
4534
  this.pendingIpcRequests.delete(fleetReqId);
2998
4535
  respond(null, `Decision operation timed out after ${daemonBudgetMs(tool) / 1000}s`);
@@ -3001,6 +4538,12 @@ export class Daemon extends EventEmitter {
3001
4538
  clearTimeout(timeout);
3002
4539
  respond(respMsg.result, respMsg.error);
3003
4540
  });
4541
+ this.ipcServer?.broadcast({
4542
+ type: typeMap[tool],
4543
+ payload: args,
4544
+ meta: { instance_name: this.name, working_directory: this.config.working_directory },
4545
+ fleetRequestId: fleetReqId,
4546
+ });
3004
4547
  return;
3005
4548
  }
3006
4549
  if (SCHEDULE_TOOLS.has(tool)) {
@@ -3012,13 +4555,7 @@ export class Daemon extends EventEmitter {
3012
4555
  };
3013
4556
  // Use fleetRequestId (not requestId) to avoid MCP server resolving the
3014
4557
  // pending tool call prematurely when it receives the broadcast.
3015
- const fleetReqId = `sched_${requestId}`;
3016
- this.ipcServer?.broadcast({
3017
- type: typeMap[tool],
3018
- payload: args,
3019
- meta: { chat_id: this.lastChatId, thread_id: this.lastThreadId, instance_name: this.name },
3020
- fleetRequestId: fleetReqId,
3021
- });
4558
+ const fleetReqId = `sched_${++this.fleetRequestSeq}_${requestId}`;
3022
4559
  // Wait for fleet_schedule_response via pending request map
3023
4560
  const timeout = setTimeout(() => {
3024
4561
  this.pendingIpcRequests.delete(fleetReqId);
@@ -3028,6 +4565,12 @@ export class Daemon extends EventEmitter {
3028
4565
  clearTimeout(timeout);
3029
4566
  respond(respMsg.result, respMsg.error);
3030
4567
  });
4568
+ this.ipcServer?.broadcast({
4569
+ type: typeMap[tool],
4570
+ payload: args,
4571
+ meta: { chat_id: this.lastChatId, thread_id: this.lastThreadId, instance_name: this.name },
4572
+ fleetRequestId: fleetReqId,
4573
+ });
3031
4574
  return;
3032
4575
  }
3033
4576
  if (CROSS_INSTANCE_TOOLS.has(tool)) {
@@ -3035,15 +4578,8 @@ export class Daemon extends EventEmitter {
3035
4578
  if (this.topicMode && this.ipcServer) {
3036
4579
  // Use fleetRequestId (not requestId) to avoid MCP server resolving the
3037
4580
  // pending tool call prematurely when it receives the broadcast.
3038
- const fleetReqId = `xmsg_${requestId}`;
4581
+ const fleetReqId = `xmsg_${++this.fleetRequestSeq}_${requestId}`;
3039
4582
  const senderSessionName = this.socketSessionNames.get(socket);
3040
- this.ipcServer.broadcast({
3041
- type: "fleet_outbound",
3042
- tool,
3043
- args,
3044
- fleetRequestId: fleetReqId,
3045
- senderSessionName,
3046
- });
3047
4583
  // Shared table, so delete_instance and the deployment tools get the same
3048
4584
  // budget here as the MCP client expects (the ad-hoc list omitted them).
3049
4585
  const crossTimeoutMs = daemonBudgetMs(tool);
@@ -3055,6 +4591,13 @@ export class Daemon extends EventEmitter {
3055
4591
  clearTimeout(timeout);
3056
4592
  respond(respMsg.result, respMsg.error);
3057
4593
  });
4594
+ this.ipcServer.broadcast({
4595
+ type: "fleet_outbound",
4596
+ tool,
4597
+ args,
4598
+ fleetRequestId: fleetReqId,
4599
+ senderSessionName,
4600
+ });
3058
4601
  }
3059
4602
  else {
3060
4603
  respond(null, "Cross-instance messaging requires topic mode");
@@ -3088,8 +4631,24 @@ export class Daemon extends EventEmitter {
3088
4631
  // The fleet manager's IPC client receives this and routes to shared adapter.
3089
4632
  // Use fleetRequestId (not requestId) to avoid other MCP sessions on this daemon
3090
4633
  // from prematurely resolving their pending requests when they receive the broadcast.
3091
- const fleetReqId = `tool_${requestId}`;
4634
+ // requestId is local to one MCP server process. Claude may keep multiple
4635
+ // MCP children attached (main session, resumed session, subagent); those
4636
+ // processes all emit requestId=1,2,... and previously overwrote this
4637
+ // shared pending map. The wrong caller could then receive another POST's
4638
+ // success + messageId while its own reply had never been confirmed.
4639
+ const fleetReqId = `tool_${++this.fleetRequestSeq}_${requestId}`;
3092
4640
  const outboundKey = fleetReqId;
4641
+ const timeout = setTimeout(() => {
4642
+ this.pendingIpcRequests.delete(outboundKey);
4643
+ respond(null, `Fleet outbound timed out after ${daemonBudgetMs(tool) / 1000}s`);
4644
+ }, daemonBudgetMs(tool));
4645
+ this.pendingIpcRequests.set(outboundKey, (respMsg) => {
4646
+ clearTimeout(timeout);
4647
+ respond(respMsg.result, respMsg.error);
4648
+ });
4649
+ // Register the waiter before broadcasting. A fast/mock adapter can settle
4650
+ // immediately; broadcasting first left a response-before-registration
4651
+ // window that degraded into a timeout despite a completed POST.
3093
4652
  this.ipcServer?.broadcast({
3094
4653
  type: "fleet_outbound",
3095
4654
  tool,
@@ -3100,14 +4659,6 @@ export class Daemon extends EventEmitter {
3100
4659
  // from a secondary world, and message ids are not portable across bots.
3101
4660
  adapterId: this.lastAdapterId,
3102
4661
  });
3103
- const timeout = setTimeout(() => {
3104
- this.pendingIpcRequests.delete(outboundKey);
3105
- respond(null, `Fleet outbound timed out after ${daemonBudgetMs(tool) / 1000}s`);
3106
- }, daemonBudgetMs(tool));
3107
- this.pendingIpcRequests.set(outboundKey, (respMsg) => {
3108
- clearTimeout(timeout);
3109
- respond(respMsg.result, respMsg.error);
3110
- });
3111
4662
  return;
3112
4663
  }
3113
4664
  const adapter = adapters[0];
@@ -3117,7 +4668,10 @@ export class Daemon extends EventEmitter {
3117
4668
  }
3118
4669
  /** Build config object for the CLI backend */
3119
4670
  buildBackendConfig() {
3120
- const isCliMode = this.config.agent_mode === "cli" || (this.config.agent_mode == null && this.config.backend === "antigravity");
4671
+ // Antigravity 1.1.17 has no per-workspace MCP config, but its global MCP
4672
+ // children inherit the CLI process environment. The backend installs one
4673
+ // environment-selecting launcher, so MCP is now the safe default here too.
4674
+ const isCliMode = this.config.agent_mode === "cli";
3121
4675
  const sockPath = join(this.instanceDir, "channel.sock");
3122
4676
  let serverJs = join(__dirname, "channel", "mcp-server.js");
3123
4677
  if (!existsSync(serverJs)) {
@@ -3376,9 +4930,35 @@ export class Daemon extends EventEmitter {
3376
4930
  throw new Error("No backend configured — cannot spawn CLI window");
3377
4931
  }
3378
4932
  const attemptedResume = !this.skipResume;
3379
- const alive = await this.trySpawn();
4933
+ // A resume launch may get a longer budget than a fresh one (kiro: the
4934
+ // conversation must come back from the backend before anything paints).
4935
+ const resumeBudget = attemptedResume ? this.startupBudgetFor(true) : undefined;
4936
+ let alive = await this.trySpawn(false, resumeBudget);
4937
+ if (!alive && attemptedResume) {
4938
+ // Resume failed. Before abandoning the session:
4939
+ // 1. If the backend is unreachable (fleet-level memory, or this very pane
4940
+ // shows the outage text), clearing the session would trade a
4941
+ // transient outage for permanent conversation loss — and the fresh
4942
+ // start would hang on the same backend. Fail the startup and keep the
4943
+ // session; the fleet's delayed retry comes back once it is up.
4944
+ // 2. Otherwise, for backends that ask for it, retry resume ONCE — the
4945
+ // first miss is usually slowness, not a broken session.
4946
+ await this.noteStartupPaneForBackendOutage();
4947
+ await this.failStartupIfBackendUnreachable();
4948
+ if (this.backend.retriesResumeOnStartupFailure?.() === true) {
4949
+ this.logger.warn("Resume startup failed — retrying resume once before abandoning the session");
4950
+ await this.killProcessTree();
4951
+ await this.tmux.killWindow();
4952
+ alive = await this.trySpawn(false, resumeBudget);
4953
+ if (!alive) {
4954
+ await this.noteStartupPaneForBackendOutage();
4955
+ await this.failStartupIfBackendUnreachable();
4956
+ }
4957
+ }
4958
+ }
3380
4959
  if (!alive) {
3381
- // First attempt failed (stale --resume, crash, rate limit, etc.)
4960
+ // Resume (or a fresh start) failed for a reason we do not recognise as an
4961
+ // outage (stale --resume, crash, rate limit, etc.).
3382
4962
  // Clean slate: clear session-id, skip resume, and retry once.
3383
4963
  this.logger.warn("CLI startup failed — clearing session-id and retrying without resume");
3384
4964
  const sidFile = join(this.instanceDir, "session-id");
@@ -3389,7 +4969,7 @@ export class Daemon extends EventEmitter {
3389
4969
  this.skipResume = true;
3390
4970
  await this.killProcessTree();
3391
4971
  await this.tmux.killWindow();
3392
- const retryAlive = await this.trySpawn();
4972
+ const retryAlive = await this.trySpawn(false, this.startupBudgetFor(false));
3393
4973
  if (!retryAlive) {
3394
4974
  await this.killProcessTree();
3395
4975
  await this.tmux.killWindow();
@@ -3398,6 +4978,9 @@ export class Daemon extends EventEmitter {
3398
4978
  }
3399
4979
  else if (attemptedResume) {
3400
4980
  resumedSuccessfully = true;
4981
+ // A resume needs the backend: its success is positive proof the backend
4982
+ // is reachable again (a fresh prompt is local and proves nothing).
4983
+ this.backendOutage?.clear(this.backendKey());
3401
4984
  }
3402
4985
  this.lastSpawnAt = Date.now();
3403
4986
  this.skipResume = false; // CLI started successfully — reset for next spawn
@@ -3408,6 +4991,122 @@ export class Daemon extends EventEmitter {
3408
4991
  }
3409
4992
  return resumedSuccessfully;
3410
4993
  }
4994
+ /**
4995
+ * Startup budget for this launch: the backend's override (resume-aware), never
4996
+ * below a user-configured startup_timeout_ms; undefined = trySpawn's default.
4997
+ */
4998
+ startupBudgetFor(resume) {
4999
+ const override = this.backend?.getStartupBudgetMs?.({ resume });
5000
+ if (override == null)
5001
+ return undefined;
5002
+ const configured = this.config.startup_timeout_ms;
5003
+ return configured != null ? Math.max(configured, override) : override;
5004
+ }
5005
+ /** Effective wake budget = max(caller timeout, backend resume override, configured startup_timeout_ms). */
5006
+ wakeBudgetMs(timeoutMs) {
5007
+ return Math.max(timeoutMs, this.startupBudgetFor(!this.skipResume) ?? 0, this.config.startup_timeout_ms ?? 0);
5008
+ }
5009
+ /** The fleet-level backend key this instance runs on (matches the lifecycle's backendOf). */
5010
+ backendKey() {
5011
+ return this.runtimeIdentity?.backend ?? this.config.backend ?? this.backend?.binaryName ?? "unknown";
5012
+ }
5013
+ /**
5014
+ * After a failed launch, read the pane for the backend's fleet-wide outage
5015
+ * text (e.g. kiro's `dispatch failure (timeout) … kiro.dev`) and record it.
5016
+ * The lifecycle only learns about outages from RUNNING instances; during a
5017
+ * post-update herd nothing is running yet, so the startup path must report
5018
+ * what it sees or the first instance in would clear its session for nothing.
5019
+ */
5020
+ async noteStartupPaneForBackendOutage() {
5021
+ if (!this.backendOutage || !this.backend || !this.tmux)
5022
+ return;
5023
+ const outagePattern = this.backend.getErrorPatterns?.().find(p => p.fleetWide && p.type === "network");
5024
+ if (!outagePattern)
5025
+ return;
5026
+ try {
5027
+ const pane = await this.tmux.capturePane();
5028
+ if (!outagePattern.pattern.test(pane))
5029
+ return;
5030
+ const detail = this.resolveErrorMessage(pane, outagePattern);
5031
+ this.backendOutage.record(this.backendKey(), this.name, detail);
5032
+ this.logger.warn({ detail }, "Startup pane shows the backend outage text");
5033
+ }
5034
+ catch { /* capture failed — nothing to learn */ }
5035
+ }
5036
+ /**
5037
+ * Fail the startup WITHOUT touching the session while the backend is known to
5038
+ * be down. The failed window is torn down first: the alternative — a live but
5039
+ * never-ready CLI left behind while the daemon reports `crashed` — would keep
5040
+ * the health monitor from ever seeing a dead pane and retrying.
5041
+ */
5042
+ async failStartupIfBackendUnreachable() {
5043
+ if (!this.backendOutage?.isActive(this.backendKey()))
5044
+ return;
5045
+ this.logger.warn("Backend unreachable — keeping the session and failing startup for a delayed retry");
5046
+ await this.killProcessTree();
5047
+ await this.tmux.killWindow();
5048
+ throw new BackendUnreachableStartupError(this.backendKey());
5049
+ }
5050
+ /**
5051
+ * Crash-respawn ran into the backend outage: the ordinary path would leave
5052
+ * `crashed` + a paused monitor + nothing scheduled (and count it as a crash).
5053
+ * Hand the instance to the fleet instead — the lifecycle stops this daemon
5054
+ * and the fleet's delayed startup retry re-creates it later with the session
5055
+ * intact. Returns true when someone took the hand-off.
5056
+ */
5057
+ handOffBackendUnreachableRespawn(err) {
5058
+ if (!(err instanceof BackendUnreachableStartupError))
5059
+ return false;
5060
+ if (this.listenerCount("startup_backend_unreachable") === 0)
5061
+ return false;
5062
+ this.logger.warn({ backend: err.backend }, "Respawn blocked by backend outage — handing off to the fleet's delayed startup retry");
5063
+ this.healthCheckPaused = true;
5064
+ this.emit("startup_backend_unreachable", { name: this.name, backend: err.backend });
5065
+ return true;
5066
+ }
5067
+ /**
5068
+ * Dispose a Daemon whose start() rejected. The lifecycle registers a daemon
5069
+ * only after start() resolves, so nothing else owns this object — and by the
5070
+ * time the spawn fails it has already bound its IPC server (channel.sock),
5071
+ * written daemon.pid and possibly created a window. Left alone, each failed
5072
+ * start leaks one live-but-unreachable server handle (the next start unlinks
5073
+ * the socket path and binds a new one), multiplied by the automatic startup
5074
+ * retries. Idempotent, and fast: no graceful CLI quit, the CLI never reached
5075
+ * its prompt.
5076
+ */
5077
+ async abortStartup() {
5078
+ if (this.startupAborted)
5079
+ return;
5080
+ this.startupAborted = true;
5081
+ this.freezeRuntimeMonitors();
5082
+ this.pendingIpcRequests.clear();
5083
+ try {
5084
+ await this.killProcessTree();
5085
+ }
5086
+ catch { /* nothing running */ }
5087
+ if (this.tmux) {
5088
+ const windowId = this.tmux.getWindowId();
5089
+ try {
5090
+ await this.tmux.killWindow();
5091
+ }
5092
+ catch { /* window may not exist */ }
5093
+ if (windowId)
5094
+ this.controlClient?.unregisterWindow(windowId);
5095
+ }
5096
+ try {
5097
+ await this.ipcServer?.close();
5098
+ }
5099
+ catch (err) {
5100
+ this.logger.debug({ err }, "IPC server close failed during startup abort");
5101
+ }
5102
+ this.ipcServer = null;
5103
+ for (const file of ["daemon.pid", "window-id"]) {
5104
+ try {
5105
+ unlinkSync(join(this.instanceDir, file));
5106
+ }
5107
+ catch { /* absent */ }
5108
+ }
5109
+ }
3411
5110
  /** Kill the entire process tree of the current tmux pane (CLI + MCP server). */
3412
5111
  async killProcessTree(signal = "SIGTERM") {
3413
5112
  if (!this.tmux)
@@ -3428,6 +5127,15 @@ export class Daemon extends EventEmitter {
3428
5127
  * Returns true if CLI is ready, false if it failed or got stuck.
3429
5128
  */
3430
5129
  async trySpawn(reuseWindow = false, startupTimeoutMs) {
5130
+ if (!this.spawnGate)
5131
+ return this.trySpawnInsideGate(reuseWindow, startupTimeoutMs);
5132
+ return this.spawnGate.run({
5133
+ instanceName: this.name,
5134
+ workingDirectory: this.config.working_directory,
5135
+ reason: reuseWindow ? "wake" : this.lastSpawnAt > 0 ? "recovery" : "startup",
5136
+ }, () => this.trySpawnInsideGate(reuseWindow, startupTimeoutMs));
5137
+ }
5138
+ async trySpawnInsideGate(reuseWindow = false, startupTimeoutMs) {
3431
5139
  const backendConfig = this.buildBackendConfig();
3432
5140
  // Compare freshly-built instructions against the last value the agent was
3433
5141
  // told about. Computed for ALL backends (not gated by
@@ -3492,6 +5200,9 @@ export class Daemon extends EventEmitter {
3492
5200
  const cmd = `${envPrefix} ` + this.backend.buildCommand(backendConfig);
3493
5201
  // Ensure tmux session exists (may have been destroyed if all windows died)
3494
5202
  await TmuxManager.ensureSession(this.tmuxSessionName);
5203
+ if (this.stormWindow?.observeServerAlive(await TmuxManager.getServerPid(this.tmuxSessionName))) {
5204
+ this.emit("tmux_server_crash", this.name);
5205
+ }
3495
5206
  let windowId;
3496
5207
  if (reuseWindow) {
3497
5208
  this.controlClient?.unregisterWindow(this.tmux.getWindowId());
@@ -3543,18 +5254,47 @@ export class Daemon extends EventEmitter {
3543
5254
  // Dismiss confirmation dialogs and verify CLI reached prompt.
3544
5255
  // With remain-on-exit, isWindowAlive() returns true even for dead panes,
3545
5256
  // but a startup crash would already be caught by waitForOutput/waitForIdle above.
3546
- if (!await this.tmux.isWindowAlive())
5257
+ return this.finishStartupScan(STARTUP_DIALOG_BUDGET_MS);
5258
+ }
5259
+ /**
5260
+ * The last leg of a spawn: confirm the pane is not positively dead, then run
5261
+ * the dialog scan. The old pre-check used tmux's isWindowAlive(), which folds
5262
+ * every query failure into `false` — so one transient tmux error right here
5263
+ * read as "the CLI died", the caller abandoned --resume and cleared the
5264
+ * session, before the tri-state scanner ever got a look. Only a POSITIVE
5265
+ * dead pane fails; an unknown liveness is the scanner's problem (bounded,
5266
+ * session-preserving).
5267
+ */
5268
+ async finishStartupScan(budgetMs) {
5269
+ if ((await this.paneLiveness()) === "dead")
3547
5270
  return false;
3548
- const ready = await this.dismissDialogsUntilReady(3);
3549
- if (ready)
5271
+ const ready = await this.dismissDialogsUntilReady(budgetMs);
5272
+ // A fatal startup screen returns true (the spawn is over — respawning
5273
+ // would only re-show the same screen) but is NOT ready-for-delivery:
5274
+ // fatalStartupBlocked gates every pane write in deliverMessage.
5275
+ if (ready && !this.fatalStartupBlocked)
3550
5276
  this.firstDeliveryDelay.recordReady();
3551
5277
  return ready;
3552
5278
  }
3553
5279
  /**
3554
- * Repeatedly check pane content, dismiss any confirmation dialogs,
3555
- * and return true once CLI reaches a ready prompt.
5280
+ * Poll the pane within a wall-clock budget, dismiss the confirmation dialogs
5281
+ * the backend knows, and return true once the CLI shows a ready prompt with
5282
+ * no dialog on screen for two consecutive polls.
5283
+ *
5284
+ * Polls are spaced by `pollMs` and every key sequence / render wait counts
5285
+ * against the budget: Claude paints its session-resume prompt only after
5286
+ * loading the session, i.e. AFTER the first quiet moment, and three
5287
+ * back-to-back captures used to miss it — the scan then "assumed ok", the
5288
+ * `❯` inside the dialog satisfied the ready pattern, and the instance sat on
5289
+ * the dialog with deliveries free to press Enter into it.
5290
+ *
5291
+ * Only hard evidence returns false: a dead window or "command not found".
5292
+ * A transient capture failure is logged and retried; exhausting the budget is
5293
+ * logged at WARN and returns true — false makes the caller abandon --resume
5294
+ * and clear the session, which is the loss this code exists to prevent. The
5295
+ * delivery gate keeps refusing pastes while a blocking dialog is visible.
3556
5296
  */
3557
- async dismissDialogsUntilReady(maxAttempts) {
5297
+ async dismissDialogsUntilReady(budgetMs, pollMs = STARTUP_DIALOG_POLL_MS) {
3558
5298
  // Backend-specific startup dialogs, with hardcoded fallback for backward compat
3559
5299
  const startupDialogs = this.backend?.getStartupDialogs?.() ?? [
3560
5300
  { pattern: /[❯›]\s*\d+\.\s*No/m, keys: ["Down", "Enter"], description: "Confirmation dialog — navigate past No" },
@@ -3562,14 +5302,76 @@ export class Daemon extends EventEmitter {
3562
5302
  { pattern: /No, exit|No, quit|Don't trust|I accept|I trust|Yes, continue|Trust folder/i, keys: ["Enter"], description: "Generic confirmation dialog" },
3563
5303
  { pattern: /Resume Session/i, keys: ["Escape"], description: "Resume session picker — start fresh" },
3564
5304
  ];
3565
- for (let i = 0; i < maxAttempts; i++) {
5305
+ const deadline = Date.now() + budgetMs;
5306
+ const remaining = () => deadline - Date.now();
5307
+ const sleep = async () => { if (remaining() > 0)
5308
+ await new Promise(r => setTimeout(r, Math.min(pollMs, Math.max(remaining(), 0)))); };
5309
+ let cleanReadyPolls = 0;
5310
+ let lastDialog = null;
5311
+ let captureFailures = 0;
5312
+ let attempts = 0;
5313
+ do {
5314
+ attempts++;
5315
+ let pane;
5316
+ try {
5317
+ pane = await this.tmux.capturePane();
5318
+ }
5319
+ catch (err) {
5320
+ // Transient tmux trouble is not evidence about the CLI. Returning false
5321
+ // here would clear the session; retry within the budget instead.
5322
+ captureFailures++;
5323
+ this.logger.warn({ err, captureFailures }, "capture-pane failed during the startup dialog scan — retrying");
5324
+ // Only a POSITIVE dead pane ends the scan; an unanswerable liveness
5325
+ // query is the same transient trouble and must not clear the session.
5326
+ if ((await this.paneLiveness()) === "dead")
5327
+ return false;
5328
+ await sleep();
5329
+ continue;
5330
+ }
3566
5331
  try {
3567
- const pane = await this.tmux.capturePane();
3568
5332
  // Try each startup dialog pattern before checking ready state
3569
5333
  let matched = false;
5334
+ lastDialog = null;
3570
5335
  for (const dialog of startupDialogs) {
3571
- if (dialog.pattern.test(pane)) {
3572
- this.logger.debug(`Dismissing startup dialog: ${dialog.description}`);
5336
+ if (Daemon.dialogMatches(dialog, pane)) {
5337
+ lastDialog = dialog;
5338
+ cleanReadyPolls = 0;
5339
+ // Start the parked clock for every delivery-blocking dialog, exact
5340
+ // ones included: if the auto-dismiss keeps failing here, the human
5341
+ // report should count from first sight, not from the runtime scan.
5342
+ if (!dialog.fatal && this.deliveryBlockingDialogs().some(d => Daemon.dialogKey(d) === Daemon.dialogKey(dialog))) {
5343
+ this.trackDialogParked(dialog);
5344
+ }
5345
+ // A fatal startup screen cannot be dismissed by keypresses (e.g. a
5346
+ // corrupt claude.json modal offering only "exit" / "reset config").
5347
+ // It must be caught HERE and not fall through to the ready check:
5348
+ // the modal contains the `❯` selector the ready pattern matches, so
5349
+ // it would otherwise pass as ready and queued messages would be
5350
+ // typed into the modal. Report the error (pause stops delivery) and
5351
+ // stop the retry loop — like the sign-in screen case below, only a
5352
+ // human fixing the config ends this state.
5353
+ if (dialog.fatal) {
5354
+ // Block deliveries BEFORE the emit: the pty_error pause handler
5355
+ // is async, and a queued message would otherwise win the race
5356
+ // and be pasted into the modal.
5357
+ this.fatalStartupBlocked = true;
5358
+ if (!this.fatalStartupReported) {
5359
+ this.fatalStartupReported = true;
5360
+ this.logger.warn({ description: dialog.description }, "Fatal startup screen detected — reporting instead of dismissing");
5361
+ this.emit("pty_error", { name: this.name, ...dialog.fatal });
5362
+ }
5363
+ return true;
5364
+ }
5365
+ if (dialog.holdOnly) {
5366
+ // Recognised but deliberately not answered (see RuntimeDialog.holdOnly).
5367
+ if (this.dialogParkedKey !== Daemon.dialogKey(dialog)) {
5368
+ this.logger.warn({ description: dialog.description }, "Startup dialog held — not auto-answering, deliveries stay blocked");
5369
+ }
5370
+ this.trackDialogParked(dialog);
5371
+ matched = true;
5372
+ break;
5373
+ }
5374
+ this.logger.info(`Dismissing startup dialog: ${dialog.description}`);
3573
5375
  // Restart is exactly when inbound messages pile up, and nothing gates
3574
5376
  // delivery on `spawning`. Take the pane lock for the key sequence so a
3575
5377
  // queued message cannot be pasted into a half-dismissed trust dialog.
@@ -3584,35 +5386,82 @@ export class Daemon extends EventEmitter {
3584
5386
  await new Promise(r => setTimeout(r, 200));
3585
5387
  }
3586
5388
  });
3587
- // Wait for next screen to render
5389
+ // Wait for next screen to render — bounded by what is left of the budget.
5390
+ const renderWait = Math.max(0, Math.min(10_000, remaining()));
3588
5391
  if (this.controlClient) {
3589
5392
  const wid = readFileSync(join(this.instanceDir, "window-id"), "utf-8").trim();
3590
- await this.controlClient.waitForIdle(wid, 10_000);
5393
+ await this.controlClient.waitForIdle(wid, renderWait);
3591
5394
  }
3592
5395
  else {
3593
- await new Promise(r => setTimeout(r, 3_000));
5396
+ await new Promise(r => setTimeout(r, Math.min(3_000, renderWait)));
3594
5397
  }
3595
- if (!await this.tmux.isWindowAlive())
5398
+ if ((await this.paneLiveness()) === "dead")
3596
5399
  return false;
3597
5400
  matched = true;
3598
5401
  break;
3599
5402
  }
3600
5403
  }
3601
- if (matched)
5404
+ if (matched) {
5405
+ if (lastDialog?.holdOnly)
5406
+ await sleep();
3602
5407
  continue;
3603
- // CLI is ready (pattern defined by each backend)
3604
- if (this.backend.getReadyPattern().test(pane))
5408
+ }
5409
+ if (this.dialogParkedSince !== 0 && !this.deliveryBlockingDialogs().some(d => Daemon.dialogMatches(d, pane)))
5410
+ this.trackDialogParked(null);
5411
+ // CLI is ready (pattern defined by each backend). Require it on two
5412
+ // consecutive polls: the first ready frame is also the moment a late
5413
+ // dialog is about to be painted over it.
5414
+ if (this.backend.getReadyPattern().test(pane)) {
5415
+ cleanReadyPolls++;
5416
+ if (cleanReadyPolls >= 2 || remaining() <= 0) {
5417
+ // A real ready prompt (with no fatal dialog on screen — those are
5418
+ // matched above, before this check) means the fatal screen is gone,
5419
+ // e.g. the user fixed the config and the instance respawned.
5420
+ this.fatalStartupBlocked = false;
5421
+ return true;
5422
+ }
5423
+ await sleep();
5424
+ continue;
5425
+ }
5426
+ cleanReadyPolls = 0;
5427
+ // A CLI parked at its own sign-in screen is an AUTH incident: no dialog
5428
+ // key can dismiss it and no ready pattern will ever appear. Report it as
5429
+ // an auth error (the lifecycle double-checks with the token-free probe)
5430
+ // instead of letting it decay into crash/hang/MCP-died noise, and stop
5431
+ // the retry loop — only re-login (see /login) ends this state.
5432
+ const loginScreen = LOGIN_FLOWS[this.config.backend ?? "claude-code"]?.loginScreenPattern;
5433
+ if (loginScreen?.test(pane) && !this.loginScreenReported) {
5434
+ this.loginScreenReported = true;
5435
+ this.authFailureUnresolved = true;
5436
+ this.logger.warn({ backend: this.config.backend }, "CLI is waiting at its sign-in screen — reporting auth error");
5437
+ this.emit("pty_error", {
5438
+ name: this.name,
5439
+ type: "auth_error",
5440
+ action: "pause",
5441
+ message: "CLI is waiting at its sign-in screen — credentials are missing or expired",
5442
+ });
3605
5443
  return true;
5444
+ }
3606
5445
  // Fatal: command not found (must match full phrase to avoid false positives
3607
5446
  // like Kiro's "agent X not found, using default")
3608
5447
  if (/command not found|: not found$/m.test(pane))
3609
5448
  return false;
3610
5449
  }
3611
- catch {
3612
- return false;
5450
+ catch (err) {
5451
+ // Key sends / isWindowAlive failing: same rule — log, retry within the budget.
5452
+ this.logger.warn({ err }, "startup dialog scan step failed — retrying");
5453
+ if ((await this.paneLiveness()) === "dead")
5454
+ return false;
3613
5455
  }
5456
+ await sleep();
5457
+ } while (remaining() > 0);
5458
+ // Budget exhausted. Never silently: say which way it went.
5459
+ if (lastDialog) {
5460
+ this.logger.warn({ description: lastDialog.description, attempts, budgetMs }, "Startup scan exhausted with a dialog still on screen — reporting the CLI alive; deliveries stay blocked until it is answered");
5461
+ }
5462
+ else {
5463
+ this.logger.warn({ attempts, budgetMs, captureFailures }, "Startup scan exhausted without a ready prompt or a known dialog — assuming ready (unknown CLI screen)");
3614
5464
  }
3615
- // Exhausted attempts — assume ok for unknown CLI prompts
3616
5465
  return true;
3617
5466
  }
3618
5467
  /**