@songsid/agend 2.1.4-beta.9 → 2.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (166) hide show
  1. package/README.md +1 -1
  2. package/README.zh-TW.md +1 -1
  3. package/dist/agent-cli-instructions.md +2 -2
  4. package/dist/agent-endpoint.js +1 -1
  5. package/dist/agent-endpoint.js.map +1 -1
  6. package/dist/backend/antigravity.d.ts +12 -8
  7. package/dist/backend/antigravity.js +189 -42
  8. package/dist/backend/antigravity.js.map +1 -1
  9. package/dist/backend/claude-code.d.ts +87 -23
  10. package/dist/backend/claude-code.js +361 -46
  11. package/dist/backend/claude-code.js.map +1 -1
  12. package/dist/backend/codex.d.ts +18 -0
  13. package/dist/backend/codex.js +139 -5
  14. package/dist/backend/codex.js.map +1 -1
  15. package/dist/backend/gemini-cli.js +2 -2
  16. package/dist/backend/gemini-cli.js.map +1 -1
  17. package/dist/backend/grok.js +2 -2
  18. package/dist/backend/grok.js.map +1 -1
  19. package/dist/backend/kiro.d.ts +52 -2
  20. package/dist/backend/kiro.js +267 -12
  21. package/dist/backend/kiro.js.map +1 -1
  22. package/dist/backend/opencode.js +17 -4
  23. package/dist/backend/opencode.js.map +1 -1
  24. package/dist/backend/types.d.ts +122 -3
  25. package/dist/backend/types.js +5 -1
  26. package/dist/backend/types.js.map +1 -1
  27. package/dist/backend-outage.d.ts +61 -0
  28. package/dist/backend-outage.js +71 -0
  29. package/dist/backend-outage.js.map +1 -0
  30. package/dist/channel/adapters/discord.d.ts +44 -2
  31. package/dist/channel/adapters/discord.js +523 -90
  32. package/dist/channel/adapters/discord.js.map +1 -1
  33. package/dist/channel/adapters/telegram.d.ts +30 -0
  34. package/dist/channel/adapters/telegram.js +221 -19
  35. package/dist/channel/adapters/telegram.js.map +1 -1
  36. package/dist/channel/agy-mcp-launcher.d.ts +2 -0
  37. package/dist/channel/agy-mcp-launcher.js +28 -0
  38. package/dist/channel/agy-mcp-launcher.js.map +1 -0
  39. package/dist/channel/ipc-bridge.d.ts +2 -1
  40. package/dist/channel/ipc-bridge.js +33 -4
  41. package/dist/channel/ipc-bridge.js.map +1 -1
  42. package/dist/channel/markdown-chunk.d.ts +42 -0
  43. package/dist/channel/markdown-chunk.js +205 -0
  44. package/dist/channel/markdown-chunk.js.map +1 -0
  45. package/dist/channel/mcp-server.js +34 -5
  46. package/dist/channel/mcp-server.js.map +1 -1
  47. package/dist/channel/mcp-tools.js +5 -3
  48. package/dist/channel/mcp-tools.js.map +1 -1
  49. package/dist/channel/message-queue.js +64 -8
  50. package/dist/channel/message-queue.js.map +1 -1
  51. package/dist/channel/types.d.ts +31 -1
  52. package/dist/classic-channel-manager.d.ts +65 -0
  53. package/dist/classic-channel-manager.js +172 -3
  54. package/dist/classic-channel-manager.js.map +1 -1
  55. package/dist/cli.js +124 -83
  56. package/dist/cli.js.map +1 -1
  57. package/dist/config-validator.js +13 -0
  58. package/dist/config-validator.js.map +1 -1
  59. package/dist/config.d.ts +4 -0
  60. package/dist/config.js +12 -1
  61. package/dist/config.js.map +1 -1
  62. package/dist/cross-instance-envelope.d.ts +6 -0
  63. package/dist/cross-instance-envelope.js +30 -0
  64. package/dist/cross-instance-envelope.js.map +1 -0
  65. package/dist/daemon.d.ts +491 -14
  66. package/dist/daemon.js +2121 -276
  67. package/dist/daemon.js.map +1 -1
  68. package/dist/doctor.d.ts +41 -0
  69. package/dist/doctor.js +267 -0
  70. package/dist/doctor.js.map +1 -0
  71. package/dist/fleet-context.d.ts +63 -0
  72. package/dist/fleet-context.js.map +1 -1
  73. package/dist/fleet-manager.d.ts +402 -11
  74. package/dist/fleet-manager.js +2594 -422
  75. package/dist/fleet-manager.js.map +1 -1
  76. package/dist/fleet-yaml-slim.d.ts +10 -0
  77. package/dist/fleet-yaml-slim.js +59 -0
  78. package/dist/fleet-yaml-slim.js.map +1 -0
  79. package/dist/general-knowledge/skills/cross-instance-messaging/SKILL.md +1 -0
  80. package/dist/general-knowledge/skills/model-discovery/SKILL.md +63 -16
  81. package/dist/general-knowledge/skills/session-management/SKILL.md +171 -8
  82. package/dist/instance-lifecycle.d.ts +114 -1
  83. package/dist/instance-lifecycle.js +404 -14
  84. package/dist/instance-lifecycle.js.map +1 -1
  85. package/dist/instructions.js +42 -4
  86. package/dist/instructions.js.map +1 -1
  87. package/dist/locale.js +1024 -320
  88. package/dist/locale.js.map +1 -1
  89. package/dist/login-flows.d.ts +91 -0
  90. package/dist/login-flows.js +170 -0
  91. package/dist/login-flows.js.map +1 -0
  92. package/dist/login-manager.d.ts +63 -0
  93. package/dist/login-manager.js +134 -0
  94. package/dist/login-manager.js.map +1 -0
  95. package/dist/network-family.d.ts +18 -0
  96. package/dist/network-family.js +20 -0
  97. package/dist/network-family.js.map +1 -0
  98. package/dist/outbound-handlers.d.ts +16 -1
  99. package/dist/outbound-handlers.js +174 -40
  100. package/dist/outbound-handlers.js.map +1 -1
  101. package/dist/outbound-schemas.d.ts +10 -3
  102. package/dist/outbound-schemas.js +10 -0
  103. package/dist/outbound-schemas.js.map +1 -1
  104. package/dist/pane-input-residue.d.ts +52 -0
  105. package/dist/pane-input-residue.js +107 -0
  106. package/dist/pane-input-residue.js.map +1 -0
  107. package/dist/process-memory.d.ts +6 -0
  108. package/dist/process-memory.js +45 -0
  109. package/dist/process-memory.js.map +1 -1
  110. package/dist/reply-dedup.d.ts +7 -8
  111. package/dist/reply-dedup.js +0 -0
  112. package/dist/reply-dedup.js.map +1 -1
  113. package/dist/restart-progress.d.ts +2 -0
  114. package/dist/restart-progress.js +3 -0
  115. package/dist/restart-progress.js.map +1 -1
  116. package/dist/scheduler/db.d.ts +12 -0
  117. package/dist/scheduler/db.js +59 -0
  118. package/dist/scheduler/db.js.map +1 -1
  119. package/dist/service-installer.d.ts +11 -0
  120. package/dist/service-installer.js +84 -18
  121. package/dist/service-installer.js.map +1 -1
  122. package/dist/settings-api.js +1 -1
  123. package/dist/settings-api.js.map +1 -1
  124. package/dist/setup-wizard.js +2 -2
  125. package/dist/setup-wizard.js.map +1 -1
  126. package/dist/spawn-gate.d.ts +30 -0
  127. package/dist/spawn-gate.js +79 -0
  128. package/dist/spawn-gate.js.map +1 -0
  129. package/dist/steer-capability.d.ts +1 -0
  130. package/dist/steer-capability.js +6 -0
  131. package/dist/steer-capability.js.map +1 -0
  132. package/dist/storm-window.d.ts +83 -0
  133. package/dist/storm-window.js +251 -0
  134. package/dist/storm-window.js.map +1 -0
  135. package/dist/tips.d.ts +44 -0
  136. package/dist/tips.js +355 -0
  137. package/dist/tips.js.map +1 -0
  138. package/dist/tmux-manager.d.ts +16 -0
  139. package/dist/tmux-manager.js +110 -25
  140. package/dist/tmux-manager.js.map +1 -1
  141. package/dist/topic-commands.d.ts +31 -7
  142. package/dist/topic-commands.js +342 -70
  143. package/dist/topic-commands.js.map +1 -1
  144. package/dist/transcript-sources.d.ts +35 -10
  145. package/dist/transcript-sources.js +190 -38
  146. package/dist/transcript-sources.js.map +1 -1
  147. package/dist/types.d.ts +6 -1
  148. package/dist/ui/dashboard.html +55 -32
  149. package/dist/ui/settings.html +200 -60
  150. package/dist/ui/view.html +147 -30
  151. package/dist/usage/format-rich.d.ts +1 -1
  152. package/dist/usage/format-rich.js +28 -24
  153. package/dist/usage/format-rich.js.map +1 -1
  154. package/dist/usage/i18n-keys.d.ts +7 -0
  155. package/dist/usage/i18n-keys.js +34 -0
  156. package/dist/usage/i18n-keys.js.map +1 -0
  157. package/dist/usage/i18n.d.ts +5 -0
  158. package/dist/usage/i18n.js +27 -0
  159. package/dist/usage/i18n.js.map +1 -0
  160. package/dist/usage/providers.d.ts +21 -0
  161. package/dist/usage/providers.js +153 -75
  162. package/dist/usage/providers.js.map +1 -1
  163. package/dist/usage/usage-api.d.ts +11 -3
  164. package/dist/usage/usage-api.js +61 -24
  165. package/dist/usage/usage-api.js.map +1 -1
  166. package/package.json +2 -2
package/dist/daemon.js CHANGED
@@ -9,6 +9,7 @@ import { clearPausedMarker, writePausedMarker } from "./pause-marker.js";
9
9
  import { TmuxManager, resolveTmuxLogicalSize } from "./tmux-manager.js";
10
10
  import { TranscriptMonitor } from "./transcript-monitor.js";
11
11
  import { createTranscriptSource } from "./transcript-sources.js";
12
+ import { LOGIN_FLOWS } from "./login-flows.js";
12
13
  import { ProgressAccumulator, summarizeProgress } from "./tool-progress.js";
13
14
  import { ContextGuardian } from "./context-guardian.js";
14
15
  import { IpcServer } from "./channel/ipc-bridge.js";
@@ -21,6 +22,8 @@ import { HangDetector } from "./hang-detector.js";
21
22
  import { writeSecretFile } from "./secret-file.js";
22
23
  import { PaneWriteLock } from "./pane-write-lock.js";
23
24
  import { buildFleetInstructions } from "./instructions.js";
25
+ import { formatCrossInstanceInboundMessage, renderCrossInstanceHandoffMetadata, } from "./cross-instance-envelope.js";
26
+ import { bottomRowIsReady, pasteLeftInInput, strandedAgendMessageInInput } from "./pane-input-residue.js";
24
27
  const __filename = fileURLToPath(import.meta.url);
25
28
  const __dirname = dirname(__filename);
26
29
  // Tool routing sets — module-level to avoid re-creation on every handleToolCall
@@ -47,6 +50,10 @@ export function buildInstructionReloadNotice(binaryName, instanceName, instanceD
47
50
  export const DEFAULT_STUCK_TIMEOUT_MS = 10 * 60_000;
48
51
  export const DEFAULT_STATE_IDLE_DEBOUNCE_MS = 2_000;
49
52
  export const DEFAULT_STATE_SAFETY_SWEEP_MS = 60_000;
53
+ /** Coalesce one cosmetic-redraw burst before comparing the visible pane. */
54
+ export const PERIODIC_REDRAW_PROBE_MS = 25;
55
+ /** A foreground server/port-forward should hand control back or be acknowledged. */
56
+ export const DEFAULT_BLOCKING_PROCESS_GRACE_MS = 2 * 60_000;
50
57
  const LAST_INBOUND_FILE = "last-inbound-at";
51
58
  /** Minimum gap between "health check is failing" notifications for one instance. */
52
59
  const HEALTH_ERROR_NOTIFY_INTERVAL_MS = 10 * 60_000;
@@ -133,19 +140,7 @@ export function writeLastInboundAt(instanceDir, timestamp) {
133
140
  * message_id line.
134
141
  */
135
142
  export function renderHandoffMetadata(meta) {
136
- const rows = [];
137
- const add = (label, value) => {
138
- if (value && value.trim())
139
- rows.push(`${label}: ${value.trim()}`);
140
- };
141
- add("message_id", meta.message_id);
142
- add("correlation_id", meta.correlation_id);
143
- add("request_kind", meta.request_kind);
144
- add("task_summary", meta.task_summary);
145
- add("working_directory", meta.working_directory);
146
- add("branch", meta.branch);
147
- add("attachment_file_id", meta.attachment_file_id);
148
- return rows.length ? `\n(${rows.join(" | ")})` : "";
143
+ return renderCrossInstanceHandoffMetadata(meta);
149
144
  }
150
145
  /** Headless inactivity timer used by the daemon and unit tests. */
151
146
  export class AutoPauseController {
@@ -159,6 +154,11 @@ export class AutoPauseController {
159
154
  recordActivity(now = Date.now()) {
160
155
  this.lastActivityAt = now;
161
156
  }
157
+ /** Apply a new idle threshold and start a fresh countdown from this update. */
158
+ reconfigure(thresholdMs, now = Date.now()) {
159
+ this.thresholdMs = Math.max(0, thresholdMs);
160
+ this.lastActivityAt = now;
161
+ }
162
162
  observe(state, now = Date.now()) {
163
163
  if (this.pausedAt !== null || this.thresholdMs <= 0)
164
164
  return false;
@@ -281,9 +281,11 @@ export class PaneStateMachine {
281
281
  * working" and a finished turn would not be seen as idle until the next 60s
282
282
  * safety sweep. Output timestamps are the honest liveness signal here, and
283
283
  * the caller has them.
284
+ * @param opts.forceBusy the backend has positively identified a still-running
285
+ * foreground tool even though its TUI keeps an old ready footer visible.
284
286
  */
285
287
  observe(pane, now = Date.now(), opts = {}) {
286
- const { settled = false, changeAt = now } = opts;
288
+ const { settled = false, changeAt = now, forceBusy = false } = opts;
287
289
  const paneHash = createHash("sha256").update(pane).digest("hex");
288
290
  const firstObservation = this.lastPaneHash === null;
289
291
  const paneChanged = this.lastPaneHash !== paneHash;
@@ -292,7 +294,10 @@ export class PaneStateMachine {
292
294
  this.lastPaneChangeAt = changeAt;
293
295
  }
294
296
  this.lastObservedAt = now;
295
- const ready = this.isReady(pane);
297
+ // Some TUIs keep their ready footer visible while a foreground tool owns
298
+ // stdin. Backends can positively identify that tool from the pane; in that
299
+ // case the old prompt must not clear pending work or retire Cancel.
300
+ const ready = !forceBusy && this.isReady(pane);
296
301
  const nextState = firstObservation
297
302
  ? ready ? "idle" : "working"
298
303
  : paneChanged && !settled
@@ -354,6 +359,33 @@ export class PendingWorkTracker {
354
359
  }
355
360
  }
356
361
  const NORMAL_ENTER_SETTLE_MS = 500;
362
+ /** Bottom-ready re-poll cadence for Enter-dropping TUIs once the pane is quiet. */
363
+ const BOTTOM_READY_POLL_MS = 250;
364
+ /** Consecutive unreadable pane probes tolerated by the delivery gate before the delivery is failed (≈10s at the poll cadence). */
365
+ const DIALOG_PROBE_UNKNOWN_MAX = 40;
366
+ /** Startup dialog scan: poll cadence and wall-clock budget (keys and waits included). */
367
+ const STARTUP_DIALOG_POLL_MS = 500;
368
+ const STARTUP_DIALOG_BUDGET_MS = 30_000;
369
+ /** A blocking dialog still on screen this long after first being seen is reported for a human. */
370
+ const DIALOG_PARKED_NOTIFY_MS = 60_000;
371
+ /** How many "dialog painted just before the write → wait → retry" rounds a delivery tolerates. */
372
+ const LATE_DIALOG_WRITE_ROUNDS = 3;
373
+ /**
374
+ * Startup failed because the CLI's backend is unreachable (see backend-outage.ts).
375
+ * The session-id is deliberately KEPT; the fleet schedules a delayed retry.
376
+ */
377
+ export class BackendUnreachableStartupError extends Error {
378
+ backend;
379
+ constructor(backend) {
380
+ super(`CLI failed to start: the ${backend} backend is unreachable — session kept for a later retry`);
381
+ this.backend = backend;
382
+ this.name = "BackendUnreachableStartupError";
383
+ }
384
+ }
385
+ /** Bounded wait (under the pane lock) for the prompt to return before retrying a dropped Enter. */
386
+ const STRANDED_RETRY_READY_WAIT_MS = 30_000;
387
+ /** Max "stranded text → submit → wait for prompt → re-check" rounds per delivery; each may send one recovery Enter. */
388
+ const STRANDED_INPUT_MAX_ROUNDS = 3;
357
389
  const FIRST_ENTER_SETTLE_MS = 1_750;
358
390
  const FIRST_DELIVERY_WINDOW_MS = 5_000;
359
391
  /** After busy native-queue paste+Enter, wait before checking the pane for silent loss. */
@@ -531,6 +563,66 @@ export function extractProxyReplyText(pane, opts = {}) {
531
563
  // The end of the turn is the answer — when over budget, keep the tail.
532
564
  return text.length > maxChars ? text.slice(text.length - maxChars) : text;
533
565
  }
566
+ /**
567
+ * Find the narrow Claude XML failure signature reported in #648.
568
+ *
569
+ * This intentionally runs only at the daemon's busy→idle edge (the caller
570
+ * enforces that) and only scans output after the current inbound marker. A
571
+ * partial tool call while Claude is still streaming is therefore never acted
572
+ * on. The extracted argument goes through the same credential redaction as the
573
+ * dead-MCP pane proxy before it can leave the host.
574
+ */
575
+ export function detectMalformedClaudeToolCall(pane, opts = {}) {
576
+ let clean = pane
577
+ .replace(/\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])/g, "")
578
+ .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "");
579
+ const marker = opts.inboundMarker?.trim();
580
+ if (marker) {
581
+ const markerAt = clean.lastIndexOf(marker);
582
+ if (markerAt >= 0)
583
+ clean = clean.slice(markerAt + marker.length);
584
+ }
585
+ const lines = clean.split(/\r?\n/).slice(-(opts.maxLines ?? 120));
586
+ const tail = lines.join("\n");
587
+ const closingTags = [...tail.matchAll(/<\/(?:parameter|invoke|function_calls)>/gi)];
588
+ if (closingTags.length === 0)
589
+ return null;
590
+ const parameterOpen = /<parameter\b[^>]*\bname\s*=\s*["']text["'][^>]*>/gi;
591
+ const parameterOpens = [...tail.matchAll(parameterOpen)];
592
+ const open = parameterOpens.at(-1);
593
+ if (!open || open.index === undefined) {
594
+ // Base stale suppression on the XML artifact itself, not unrelated pane
595
+ // redraws appended after it. The occurrence counts still distinguish a
596
+ // genuinely new identical malformed call while both remain visible.
597
+ const signature = createHash("sha256")
598
+ .update(`0:${closingTags.length}:${closingTags.map(tag => tag[0].toLowerCase()).join(":")}`)
599
+ .digest("hex");
600
+ return { signature, text: null };
601
+ }
602
+ const contentAt = open.index + open[0].length;
603
+ const closeAt = tail.slice(contentAt).search(/<\/parameter>/i);
604
+ if (closeAt < 0) {
605
+ const signature = createHash("sha256")
606
+ .update(`${parameterOpens.length}:${closingTags.length}:${open[0]}`)
607
+ .digest("hex");
608
+ return { signature, text: null };
609
+ }
610
+ const fragment = tail.slice(contentAt, contentAt + closeAt);
611
+ // Fingerprint only the malformed XML artifact; never put its contents in logs.
612
+ const signature = createHash("sha256")
613
+ .update(`${parameterOpens.length}:${closingTags.length}:${open[0]}${fragment}</parameter>`)
614
+ .digest("hex");
615
+ let text = sanitizePaneTail(fragment, opts.maxLines ?? 120)
616
+ .join("\n")
617
+ .replace(/\n{3,}/g, "\n\n")
618
+ .trim();
619
+ if (text.replace(/[^\p{L}\p{N}]/gu, "").length < 2)
620
+ return { signature, text: null };
621
+ const maxChars = opts.maxChars ?? 12_000;
622
+ if (text.length > maxChars)
623
+ text = text.slice(0, maxChars);
624
+ return { signature, text };
625
+ }
534
626
  const INTERACTIVE_PROMPT_PATTERNS = [
535
627
  { kind: "sudo_password", pattern: /^\s*\[sudo\]\s+password\s+for\s+[^:\n]+:\s*$/im },
536
628
  // macOS sudo and several package installers use this exact no-echo prompt.
@@ -593,6 +685,69 @@ export class InteractivePromptDetector {
593
685
  this.notifiedSignature = null;
594
686
  }
595
687
  }
688
+ const BLOCKING_PROCESS_PATTERNS = [
689
+ // kubectl port-forward (including a backgrounded child whose inherited
690
+ // stdout keeps the parent shell tool open).
691
+ /^\s*(?:Forwarding from\s+(?:127\.0\.0\.1|localhost|\[::1\]):\d+\s+->\s+\d+|Handling connection for\s+\d+)\s*$/im,
692
+ // Common development servers. Keep these line-shaped and require an address
693
+ // or port so prose such as "the server is running" does not arm the detector.
694
+ /^\s*(?:(?:INFO:\s*)?Uvicorn running on|Serving HTTP on|Listening on|Server (?:is )?(?:listening|running) (?:at|on)|Application startup complete[^\n]*(?:port|https?:\/\/))[^\n]*(?:https?:\/\/|\b(?:localhost|127\.0\.0\.1|0\.0\.0\.0|\[::\]|port)\b)[^\n]*$/im,
695
+ ];
696
+ /**
697
+ * Detect a foreground shell tool which has become a long-lived server.
698
+ *
699
+ * Unlike the generic pane-stuck clock, this clock deliberately survives new
700
+ * output: access logs and `Handling connection` lines are exactly why a
701
+ * foreground server can remain blocked forever without ever looking silent.
702
+ * Once a server marker is seen, the still-running tool identity is the stable
703
+ * signal; the marker may scroll out of the 32-row pane.
704
+ */
705
+ export class BlockingProcessDetector {
706
+ graceMs;
707
+ activity = null;
708
+ evidence = null;
709
+ detectedAt = 0;
710
+ notified = false;
711
+ constructor(graceMs = DEFAULT_BLOCKING_PROCESS_GRACE_MS) {
712
+ this.graceMs = graceMs;
713
+ }
714
+ observe(pane, activity, now = Date.now()) {
715
+ const shellActivity = activity && /^(?:shell|bash|exec_command|terminal|command)(?::|$)/i.test(activity);
716
+ if (!shellActivity) {
717
+ this.reset();
718
+ return null;
719
+ }
720
+ if (activity !== this.activity) {
721
+ this.reset();
722
+ this.activity = activity;
723
+ }
724
+ if (!this.evidence) {
725
+ const tail = sanitizePaneTail(pane, 40).join("\n");
726
+ for (const pattern of BLOCKING_PROCESS_PATTERNS) {
727
+ const match = tail.match(pattern);
728
+ if (!match)
729
+ continue;
730
+ this.evidence = match[0].trim().slice(0, 200);
731
+ this.detectedAt = now;
732
+ break;
733
+ }
734
+ }
735
+ if (!this.evidence || this.notified || now - this.detectedAt < this.graceMs)
736
+ return null;
737
+ this.notified = true;
738
+ return {
739
+ activity,
740
+ evidence: this.evidence,
741
+ blockedForMs: now - this.detectedAt,
742
+ };
743
+ }
744
+ reset() {
745
+ this.activity = null;
746
+ this.evidence = null;
747
+ this.detectedAt = 0;
748
+ this.notified = false;
749
+ }
750
+ }
596
751
  export class Daemon extends EventEmitter {
597
752
  name;
598
753
  config;
@@ -601,6 +756,9 @@ export class Daemon extends EventEmitter {
601
756
  backend;
602
757
  controlClient;
603
758
  runtimeIdentity;
759
+ spawnGate;
760
+ stormWindow;
761
+ backendOutage;
604
762
  logger;
605
763
  tmuxSessionName;
606
764
  tmux = null;
@@ -610,6 +768,13 @@ export class Daemon extends EventEmitter {
610
768
  guardian = null;
611
769
  adapter = null;
612
770
  pendingIpcRequests = new Map();
771
+ /**
772
+ * Daemon-global request sequence. MCP request ids are only process-local:
773
+ * Claude can have more than one MCP child connected to the same daemon and
774
+ * each starts at 1, so using requestId alone lets one caller steal another's
775
+ * response.
776
+ */
777
+ fleetRequestSeq = 0;
613
778
  // Track chatId/threadId from inbound messages for automatic outbound routing
614
779
  lastChatId;
615
780
  lastThreadId;
@@ -620,10 +785,12 @@ export class Daemon extends EventEmitter {
620
785
  currentActivity = null;
621
786
  // Session identity: map IPC socket → sessionName (from mcp_ready)
622
787
  socketSessionNames = new Map();
623
- // Crash recovery
624
- static tmuxServerCrashTimestamps = [];
625
- static tmuxServerPaused = false;
626
- static tmuxServerRecoveryTimer = null;
788
+ /** pid each connected MCP server announced in mcp_ready (or pong); keyed like socketSessionNames. */
789
+ socketPids = new Map();
790
+ mcpPingSeq = 0;
791
+ mcpPingWaiters = new Map();
792
+ // Crash recovery is fleet-owned; all daemon objects share the coordinators
793
+ // injected by FleetManager below.
627
794
  healthCheckTimer = null;
628
795
  crashCount = 0;
629
796
  lastCrashAt = 0;
@@ -644,6 +811,12 @@ export class Daemon extends EventEmitter {
644
811
  resolveSpawnSettled = null;
645
812
  spawnDepth = 0;
646
813
  skipResume = false;
814
+ startupAborted = false;
815
+ /** First time the current on-screen blocking dialog was seen (0 = none); drives the parked report. */
816
+ dialogParkedSince = 0;
817
+ /** Identity of that dialog: its pattern, not its description (two tables may describe one screen differently). */
818
+ dialogParkedKey = null;
819
+ dialogParkedReported = false;
647
820
  backgroundSessionRecoveryAttempted = false;
648
821
  /** Whether the last spawn started a fresh session (not resumed). */
649
822
  isNewSession = false;
@@ -657,6 +830,7 @@ export class Daemon extends EventEmitter {
657
830
  /** Fallback safety sweep used only when no shared tmux control client exists. */
658
831
  instanceStateMonitorTimer = null;
659
832
  instanceStateIdleTimer = null;
833
+ instanceStateOutputProbeTimer = null;
660
834
  instanceStateStuckTimer = null;
661
835
  instanceStateOutputListener = null;
662
836
  instanceStateOutputEventName = null;
@@ -675,8 +849,12 @@ export class Daemon extends EventEmitter {
675
849
  // between, the agent's answer exists only on screen — the daemon relays it.
676
850
  turnHadInbound = false;
677
851
  turnOutboundDelivered = false;
852
+ /** Successful `reply` specifically; cross-instance tools do not satisfy #648. */
853
+ turnReplyDelivered = false;
678
854
  turnCorrelationId;
679
855
  turnInboundMarker;
856
+ /** Prevent a visible stale XML fragment from being recovered on later turns. */
857
+ lastMalformedToolCallSignature;
680
858
  proxyReplySeq = 0;
681
859
  autoPauseController;
682
860
  pauseRequested = false;
@@ -687,6 +865,35 @@ export class Daemon extends EventEmitter {
687
865
  * would silently no-op exactly when we most need to stop feeding the CLI.
688
866
  */
689
867
  pausePending = false;
868
+ /**
869
+ * An auth failure was detected and nothing has resolved it yet.
870
+ *
871
+ * Separate from `pausePending` because it must outlive it: the deferred pause
872
+ * clears `pausePending` as soon as it fires, but if that pause is refused the
873
+ * instance is still auth-broken and must not resume raising alarms.
874
+ */
875
+ authFailureUnresolved = false;
876
+ /** One login-screen auth report per spawn — the screen persists across polls. */
877
+ loginScreenReported = false;
878
+ /** A fatal startup screen (see StartupDialog.fatal) was already reported this run. */
879
+ fatalStartupReported = false;
880
+ /**
881
+ * The CLI is parked on a fatal startup screen and every pane write must be
882
+ * refused. This is deliberately separate from the pause that the reported
883
+ * pty_error triggers: the pause is applied by an async handler, and a queued
884
+ * delivery would race it into the modal (where paste+Enter can confirm
885
+ * "Exit and fix manually"). Set synchronously at dialog-match time; cleared
886
+ * when a startup scan reaches a real ready prompt, or on wake().
887
+ */
888
+ fatalStartupBlocked = false;
889
+ /** model_error seen mid-turn: notify only if it survives to the idle screen. */
890
+ pendingModelErrorKey = null;
891
+ /**
892
+ * Let the next pause() proceed from a stuck pane. Set only for the auth-deferred
893
+ * pause — see the pausePending consumption site for why waiting for idle there
894
+ * is waiting forever.
895
+ */
896
+ pauseAllowStuck = false;
690
897
  pauseWakeState = "active";
691
898
  pauseWakeTransition = null;
692
899
  // Model failover: override model on next spawn when rate-limited
@@ -700,6 +907,12 @@ export class Daemon extends EventEmitter {
700
907
  * and ⏳/👀/✅ reactions live here). It does NOT cover the other writers that
701
908
  * reach the pane — {@link paneWriteLock} does. */
702
909
  pasteLock = Promise.resolve();
910
+ /**
911
+ * Fleet cancellation epoch. Queue entries retain the epoch they arrived with;
912
+ * advancing it invalidates only work received before the cancel, never a new
913
+ * user message which arrives immediately afterwards.
914
+ */
915
+ deliveryEpoch = 0;
703
916
  /** Mutual exclusion for *every* write into the pane, whichever subsystem it
704
917
  * comes from. See PaneWriteLock for why interleaving is destructive. */
705
918
  paneWriteLock = new PaneWriteLock();
@@ -727,7 +940,9 @@ export class Daemon extends EventEmitter {
727
940
  // PTY error pattern monitoring
728
941
  errorMonitorTimer = null;
729
942
  interactivePromptDetector = new InteractivePromptDetector();
943
+ blockingProcessDetector = new BlockingProcessDetector();
730
944
  /** Same 5-min gate the error monitor uses, so a dead MCP server alerts once. */
945
+ static MCP_PING_TIMEOUT_MS = 2_000;
731
946
  static MCP_DEATH_COOLDOWN_MS = 5 * 60_000;
732
947
  lastMcpDeathNotifiedAt = 0;
733
948
  mcpDeathNotifiedForPid = null;
@@ -738,6 +953,16 @@ export class Daemon extends EventEmitter {
738
953
  * past this point interrupting the turn costs less than staying mute.
739
954
  */
740
955
  static MCP_RESTART_MAX_WAIT_MS = 30 * 60_000;
956
+ /**
957
+ * How long the revival restart waits, at the point of firing, for a
958
+ * CLI-respawned MCP server to announce itself.
959
+ *
960
+ * Codex <0.146 tears down and recreates ALL MCP connections on an auth/config
961
+ * change — healthy ones included — so the daemon sees the old pid die while
962
+ * the CLI is fine and a replacement is seconds away. Module load measured
963
+ * ~0.36s, so 2s is roughly 5x headroom without making a genuine loss wait.
964
+ */
965
+ static MCP_REPLACEMENT_GRACE_MS = 2_000;
741
966
  /**
742
967
  * Sticky "restart to revive the MCP server as soon as the pane idles" — same
743
968
  * shape as pausePending, because the death is usually detected mid-turn when
@@ -746,6 +971,25 @@ export class Daemon extends EventEmitter {
746
971
  */
747
972
  mcpRestartPending = false;
748
973
  mcpRestartStaleTimer = null;
974
+ /** In-flight replacement grace; also the guard against three triggers stacking. */
975
+ mcpRestartGraceTimer = null;
976
+ /**
977
+ * Which trigger the in-flight grace will commit under.
978
+ *
979
+ * The single-timer guard would otherwise SWALLOW a stale_timeout that lands
980
+ * during an idle-gated grace — and because the stale timer is one-shot and
981
+ * already cleared itself, the 30-minute backstop would be gone for good on an
982
+ * instance that never idles, which is the exact case it exists for.
983
+ */
984
+ mcpRestartGraceTrigger = null;
985
+ /**
986
+ * A dead-MCP sighting held back once, pending a possible replacement.
987
+ *
988
+ * Deliberately NOT the same field as `mcpDeathNotifiedForPid`: deferring must
989
+ * not consume the once-per-pid budget, or a suspicion that turns out to be a
990
+ * real loss could never be reported at all.
991
+ */
992
+ mcpDeathDeferredForPid = null;
749
993
  /** Prevent in-flight monitor callbacks from re-arming after a pause. */
750
994
  runtimeMonitorsFrozen = false;
751
995
  errorWaitingForRecovery = false; // true = error detected, waiting for ready pattern
@@ -754,7 +998,7 @@ export class Daemon extends EventEmitter {
754
998
  activeErrorPatternKey = null;
755
999
  /** Whether this instance is in an abnormal error state (auto-pause is normal). */
756
1000
  get isErrorState() {
757
- return this.errorWaitingForRecovery || (this.healthCheckPaused && !this.isPaused) || Daemon.tmuxServerPaused;
1001
+ return this.errorWaitingForRecovery || (this.healthCheckPaused && !this.isPaused) || this.stormWindow?.isActive() === true;
758
1002
  }
759
1003
  get isPaused() { return this.pauseWakeState !== "active"; }
760
1004
  /** Current CLI process state for status surfaces and recovery commands. */
@@ -794,7 +1038,9 @@ export class Daemon extends EventEmitter {
794
1038
  this.errorRecoveryDeadlineAt = 0;
795
1039
  this.activeErrorPatternKey = null;
796
1040
  }
797
- constructor(name, config, instanceDir, topicMode = false, backend, controlClient, rootLogger, runtimeIdentity) {
1041
+ constructor(name, config, instanceDir, topicMode = false, backend, controlClient, rootLogger, runtimeIdentity, spawnGate, stormWindow,
1042
+ /** Fleet-level "is this CLI's backend down?" memory — see backend-outage.ts. */
1043
+ backendOutage) {
798
1044
  super();
799
1045
  this.name = name;
800
1046
  this.config = config;
@@ -803,6 +1049,9 @@ export class Daemon extends EventEmitter {
803
1049
  this.backend = backend;
804
1050
  this.controlClient = controlClient;
805
1051
  this.runtimeIdentity = runtimeIdentity;
1052
+ this.spawnGate = spawnGate;
1053
+ this.stormWindow = stormWindow;
1054
+ this.backendOutage = backendOutage;
806
1055
  if (!rootLogger)
807
1056
  throw new Error("Daemon requires a shared root logger");
808
1057
  this.runtimeIdentity ??= {
@@ -891,20 +1140,10 @@ export class Daemon extends EventEmitter {
891
1140
  this.handleToolCall(msg, socket);
892
1141
  }
893
1142
  else if (msg.type === "mcp_ready") {
894
- const sessionName = msg.sessionName;
895
- if (sessionName) {
896
- this.socketSessionNames.set(socket, sessionName);
897
- socket.on("close", () => {
898
- this.socketSessionNames.delete(socket);
899
- // Notify fleet manager so it can clean up sessionRegistry
900
- if (sessionName !== this.name) {
901
- this.ipcServer?.broadcast({ type: "session_disconnected", sessionName });
902
- }
903
- });
904
- }
905
- this.logger.debug({ sessionName }, "MCP channel server connected and ready");
906
- // Notify FleetManager's IPC client that MCP is ready
907
- this.ipcServer?.broadcast({ type: "mcp_ready", sessionName });
1143
+ this.handleMcpReady(msg, socket);
1144
+ }
1145
+ else if (msg.type === "pong") {
1146
+ this.handleMcpPong(msg, socket);
908
1147
  }
909
1148
  else if (msg.type === "query_sessions") {
910
1149
  // Fleet manager asks for all registered session names (catches sessions
@@ -924,47 +1163,38 @@ export class Daemon extends EventEmitter {
924
1163
  // Fleet manager routed a message to us (topic mode)
925
1164
  const meta = msg.meta;
926
1165
  const targetSession = msg.targetSession;
1166
+ const deliveryEpoch = this.captureDeliveryEpoch(msg.delivery_epoch);
927
1167
  void this.wake().then(() => {
928
- this.pushChannelMessage(msg.content, meta, targetSession);
1168
+ this.pushChannelMessage(msg.content, meta, targetSession, deliveryEpoch);
929
1169
  }).catch(err => {
930
1170
  this.logger.error({ err: err.message }, "Wake failed for inbound delivery");
931
1171
  });
932
1172
  }
933
1173
  else if (msg.type === "raw_paste") {
934
1174
  // Paste raw text directly to CLI without [user:] wrapping.
935
- if (this.tmux) {
936
- const rawText = msg.content;
937
- this.pasteLock = this.pasteLock.then(async () => {
938
- await this.wake();
939
- await this.deliverMessage(rawText);
940
- this.logger.debug({ text: rawText.slice(0, 100) }, "Raw paste delivered");
941
- }).catch(err => {
942
- this.logger.warn({ err: err.message }, "raw_paste delivery error");
943
- });
944
- }
1175
+ this.queueRawPaste(msg.content, this.captureDeliveryEpoch(msg.delivery_epoch), msg.confirm_clear === true);
1176
+ }
1177
+ else if (msg.type === "config_update") {
1178
+ this.applyConfigUpdate(msg.config);
945
1179
  }
946
1180
  else if (msg.type === "steer") {
947
1181
  const meta = (msg.meta ?? {});
948
- this.steerMessage(msg.content, meta);
1182
+ this.steerMessage(msg.content, meta, this.captureDeliveryEpoch(msg.delivery_epoch));
1183
+ }
1184
+ else if (msg.type === "btw") {
1185
+ const meta = (msg.meta ?? {});
1186
+ this.btwMessage(msg.content, meta, this.captureDeliveryEpoch(msg.delivery_epoch));
949
1187
  }
950
1188
  else if (msg.type === "fleet_schedule_trigger") {
951
1189
  const payload = msg.payload;
952
1190
  const meta = msg.meta;
953
- void this.wake().then(() => this.pushChannelMessage(payload.message, meta)).catch(err => {
1191
+ const deliveryEpoch = this.captureDeliveryEpoch(msg.delivery_epoch);
1192
+ void this.wake().then(() => this.pushChannelMessage(payload.message, meta, undefined, deliveryEpoch)).catch(err => {
954
1193
  this.logger.error({ err: err.message }, "Wake failed for scheduled delivery");
955
1194
  });
956
1195
  }
957
1196
  else if (msg.type === "query_instance_state") {
958
- const snapshot = this.getInstanceStateSnapshot();
959
- this.ipcServer?.send(socket, {
960
- type: "instance_state_response",
961
- requestId: msg.requestId,
962
- instanceName: this.name,
963
- ...snapshot,
964
- state: this.isPaused ? "paused" : snapshot.state,
965
- processStatus: this.processStatus,
966
- pausedAt: this.lastPausedAt,
967
- });
1197
+ void this.respondToInstanceStateQuery(msg, socket);
968
1198
  }
969
1199
  });
970
1200
  // 2. Tmux — ensure session, create window if not alive
@@ -1133,30 +1363,119 @@ export class Daemon extends EventEmitter {
1133
1363
  * process would have no client reading it. Only the CLI can restore its own
1134
1364
  * tools — hence notify, and let the operator decide about restarting.
1135
1365
  */
1366
+ /**
1367
+ * An MCP server has connected and announced itself.
1368
+ *
1369
+ * Beyond the existing session bookkeeping this is the earliest evidence that
1370
+ * tools are back — roughly 30s ahead of the next health tick, which is
1371
+ * exactly the window #663's false restarts fired in.
1372
+ */
1373
+ handleMcpReady(msg, socket) {
1374
+ const sessionName = msg.sessionName;
1375
+ if (typeof msg.pid === "number")
1376
+ this.socketPids.set(socket, msg.pid);
1377
+ if (sessionName) {
1378
+ this.socketSessionNames.set(socket, sessionName);
1379
+ socket.on("close", () => {
1380
+ this.socketSessionNames.delete(socket);
1381
+ this.socketPids.delete(socket);
1382
+ // Notify fleet manager so it can clean up sessionRegistry
1383
+ if (sessionName !== this.name) {
1384
+ this.ipcServer?.broadcast({ type: "session_disconnected", sessionName });
1385
+ }
1386
+ });
1387
+ }
1388
+ this.logger.debug({ sessionName, pid: msg.pid }, "MCP channel server connected and ready");
1389
+ // Gated on ownership: this ipcServer also accepts EXTERNAL sessions (a
1390
+ // `claude` run in another directory registers as `external-<cwd>-<pid>`).
1391
+ // Without the gate an unrelated session could stand down a restart armed
1392
+ // for THIS instance's genuinely dead server. The liveness re-read below is
1393
+ // a second guard, but ownership is the one that is semantically right.
1394
+ if (sessionName === this.name) {
1395
+ // A server for THIS instance just connected: that is proof of life at
1396
+ // the IPC layer, whatever channel.mcp.pid says (a replacement or a
1397
+ // sub-agent's server may have left a dead pid in the slot).
1398
+ this.noteMcpProofOfLife("mcp_ready", typeof msg.pid === "number" ? msg.pid : undefined);
1399
+ }
1400
+ this.ipcServer?.broadcast({ type: "mcp_ready", sessionName });
1401
+ }
1136
1402
  checkMcpServerAlive() {
1137
1403
  if (this.isPaused)
1138
1404
  return;
1139
1405
  const status = mcpServerState(this.instanceDir);
1140
- if (status.state === "unknown")
1141
- return;
1142
1406
  if (status.state === "alive") {
1407
+ if (this.mcpDeathNotifiedForPid != null)
1408
+ this.emitMcpRecovered("pid", status.pid);
1143
1409
  this.mcpDeathNotifiedForPid = null; // the CLI respawned it — re-arm
1410
+ this.mcpDeathDeferredForPid = null;
1144
1411
  this.clearMcpRestartRequest(); // tools are back — stand down a pending auto-restart
1145
1412
  return;
1146
1413
  }
1414
+ // The pid slot does not name a live process — dead, or missing. It is a
1415
+ // single last-writer slot shared by every mcp-server process of this
1416
+ // instance (a CLI-driven replacement, a sub-agent's server), so "that pid
1417
+ // is gone" is not "no server is serving", and "the last writer exited
1418
+ // cleanly and unlinked its slot" is not "nothing ever started". Ask the
1419
+ // IPC layer — the one thing every serving mcp-server must hold — before
1420
+ // deciding. Reproduced deterministically: two servers, SIGKILL the last
1421
+ // writer, the other keeps serving while the slot reads DEAD forever; or
1422
+ // the last writer exits cleanly and the slot stays MISSING, so the
1423
+ // survivor's later real death would never be seen.
1424
+ const sockets = this.liveMcpSockets();
1425
+ if (sockets.length > 0) {
1426
+ const livePid = this.socketPids.get(sockets[sockets.length - 1]);
1427
+ const stalePid = status.state === "dead" ? status.pid : undefined;
1428
+ this.logger.info({ slot: status.state, stalePid, livePid, connections: sockets.length }, "channel.mcp.pid does not name a live process, but a live MCP connection still serves this instance — not an incident");
1429
+ this.repairMcpPidSlot(livePid, status.state === "dead" ? "stale pid slot" : "missing pid slot");
1430
+ this.pingMcpSockets(sockets);
1431
+ this.noteMcpProofOfLife("connection", livePid);
1432
+ return;
1433
+ }
1434
+ if (status.state === "unknown")
1435
+ return; // never started, or nothing left to watch — as before
1147
1436
  // Dead: report once per pid, and at most once per cooldown window.
1148
1437
  if (this.mcpDeathNotifiedForPid === status.pid)
1149
1438
  return;
1439
+ // An auth-broken CLI cannot keep an MCP server alive, and restarting it
1440
+ // only loops back to the sign-in screen — hold the auto-restart until a
1441
+ // re-login (which restarts the instance and replaces this daemon anyway).
1442
+ const authSuspected = this.authFailureUnresolved;
1443
+ const autoRestart = this.config.mcp_auto_restart !== false && !authSuspected;
1444
+ // Codex <0.146 kills and recreates its MCP connections wholesale, so the
1445
+ // FIRST sighting of a dead pid under a live Codex is more likely a swap
1446
+ // than a loss. Hold the alarm for one tick — only the alarm: the restart is
1447
+ // still armed, and the chokepoint's re-read cancels it if the replacement
1448
+ // arrives. Not generalised to other backends, where a dead MCP under a live
1449
+ // CLI is a genuine incident that silence would mask.
1450
+ //
1451
+ // Crucially this does NOT touch mcpDeathNotifiedForPid or the cooldown
1452
+ // clock: if the suspicion is wrong the incident must still be reportable.
1453
+ if (this.suspectsTransientMcpReplacement() && this.mcpDeathDeferredForPid !== status.pid) {
1454
+ this.mcpDeathDeferredForPid = status.pid;
1455
+ this.logger.warn({ pid: status.pid }, "MCP server pid is gone under a live Codex — holding the alarm one tick for a possible replacement");
1456
+ if (autoRestart)
1457
+ this.armMcpRestartWhenIdle();
1458
+ return;
1459
+ }
1150
1460
  if (Date.now() - this.lastMcpDeathNotifiedAt < Daemon.MCP_DEATH_COOLDOWN_MS)
1151
1461
  return;
1152
1462
  this.mcpDeathNotifiedForPid = status.pid;
1463
+ this.mcpDeathDeferredForPid = null;
1153
1464
  this.lastMcpDeathNotifiedAt = Date.now();
1154
- const autoRestart = this.config.mcp_auto_restart !== false;
1155
- this.logger.error({ pid: status.pid, autoRestart }, "MCP server process is gone — instance has no agend tools");
1156
- this.emit("mcp_died", { name: this.name, pid: status.pid, autoRestart });
1465
+ this.logger.error({ pid: status.pid, autoRestart, authSuspected }, "MCP server process is gone — instance has no agend tools");
1466
+ this.emit("mcp_died", { name: this.name, pid: status.pid, autoRestart, authSuspected });
1157
1467
  if (autoRestart)
1158
1468
  this.armMcpRestartWhenIdle();
1159
1469
  }
1470
+ /**
1471
+ * Whether a dead MCP pid on this backend is more likely a CLI-driven
1472
+ * replacement than a real loss. Codex only, and deliberately narrow: it is
1473
+ * the backend whose released fix (0.146) confirms it tore down healthy MCP
1474
+ * connections alongside the one it meant to restart.
1475
+ */
1476
+ suspectsTransientMcpReplacement() {
1477
+ return (this.config.backend ?? this.backend?.binaryName) === "codex";
1478
+ }
1160
1479
  /**
1161
1480
  * Arm an automatic instance restart to revive a dead MCP server. Fires on the
1162
1481
  * next busy→idle edge — or immediately when the pane is already idle, which is
@@ -1179,21 +1498,210 @@ export class Daemon extends EventEmitter {
1179
1498
  this.fireMcpRestartRequest("already_idle");
1180
1499
  }
1181
1500
  }
1501
+ /**
1502
+ * Single chokepoint for all three revival triggers (already_idle, idle_edge,
1503
+ * stale_timeout) — and therefore the one place worth re-checking reality.
1504
+ *
1505
+ * The pending flag is armed by a health tick that saw the OLD pid die, and
1506
+ * was previously fired without ever looking again. When a CLI legitimately
1507
+ * respawns its own MCP child, the replacement is already serving by the time
1508
+ * the turn ends — but the idle edge fires inside the ~30s before the next
1509
+ * health tick can stand the request down, restarting a healthy instance. One
1510
+ * external user was restarted 18 times in two days this way (#663).
1511
+ *
1512
+ * Re-reading here is safe for every backend: worst case it clears a pending
1513
+ * restart that should not have existed.
1514
+ */
1182
1515
  fireMcpRestartRequest(trigger) {
1183
- // A recovery is already reshaping the CLI: crash-loop handling
1184
- // (healthCheckPaused), a respawn in flight (spawning), or a pause
1185
- // (isPaused / frozen monitors). Each of those paths ends in a fresh CLI —
1186
- // and with it a fresh MCP server — or in supervision ending, where a
1187
- // restart on top would fight the recovery. Either way the revival restart
1188
- // is moot: cancel it rather than defer it.
1189
- if (this.healthCheckPaused || this.spawning || this.isPaused || this.runtimeMonitorsFrozen) {
1190
- this.logger.warn({ trigger }, "MCP revival restart cancelled — instance is already pausing/spawning/recovering");
1516
+ if (this.mcpRestartCancelledByRecovery(trigger))
1517
+ return;
1518
+ // Authoritative re-read — the CLI may have replaced the server since the
1519
+ // health tick that armed this.
1520
+ if (this.mcpServerAlive().alive) {
1521
+ this.logger.info({ trigger }, "MCP revival restart stood down — a live MCP server is already serving");
1522
+ this.clearMcpRestartRequest();
1523
+ return;
1524
+ }
1525
+ // Not back yet: give a CLI-driven replacement a bounded moment rather than
1526
+ // restarting the instance out from under it. One timer covers all three
1527
+ // triggers; a second trigger during the wait is a no-op.
1528
+ if (this.mcpRestartGraceTimer) {
1529
+ // stale_timeout outranks the idle-gated triggers: it is the backstop for
1530
+ // a turn that never ends, so it must not be lost to an in-flight grace.
1531
+ if (trigger === "stale_timeout")
1532
+ this.mcpRestartGraceTrigger = "stale_timeout";
1533
+ return;
1534
+ }
1535
+ this.mcpRestartGraceTrigger = trigger;
1536
+ this.mcpRestartGraceTimer = setTimeout(() => {
1537
+ this.mcpRestartGraceTimer = null;
1538
+ const committed = this.mcpRestartGraceTrigger ?? trigger;
1539
+ this.mcpRestartGraceTrigger = null;
1540
+ if (!this.mcpRestartPending || this.runtimeMonitorsFrozen)
1541
+ return;
1542
+ if (this.mcpRestartCancelledByRecovery(committed))
1543
+ return;
1544
+ if (this.mcpServerAlive().alive) {
1545
+ this.logger.info({ trigger: committed }, "MCP revival restart stood down — replacement came up during the grace window");
1546
+ this.clearMcpRestartRequest();
1547
+ return;
1548
+ }
1549
+ // Re-check the idle gate. The grace turned what used to be a synchronous
1550
+ // decision into a 2s window, and "idle edge, then a queued message lands"
1551
+ // is the NORMAL ordering for queued delivery — so without this the
1552
+ // restart tears down a turn that started during the wait. Worse than the
1553
+ // race it replaced: the #485 proxy reply only runs at turn end, so an
1554
+ // in-flight answer would be swallowed with no fallback. Only the
1555
+ // 30-minute stale timer may interrupt a live turn, by design.
1556
+ //
1557
+ // Deliberately does NOT clear pending: the MCP really is gone, so the
1558
+ // request must survive to the next idle edge (which re-enters here), and
1559
+ // the stale timer keeps running underneath as the backstop.
1560
+ if (committed !== "stale_timeout" && !(this.instanceState === "idle" && this.pasteQueueDepth === 0)) {
1561
+ this.logger.info({ trigger: committed, instanceState: this.instanceState, pasteQueueDepth: this.pasteQueueDepth }, "MCP revival restart deferred — the pane went busy during the grace window");
1562
+ return;
1563
+ }
1564
+ // The suspicion was wrong: a real loss. Report it BEFORE asking for the
1565
+ // restart — instance-lifecycle's suppression path documents that it
1566
+ // assumes the mcp_died notification has already gone out, and this daemon
1567
+ // does not survive the restart to send it afterwards.
1568
+ this.reportDeferredMcpDeath();
1191
1569
  this.clearMcpRestartRequest();
1570
+ this.logger.warn({ trigger: committed }, "Requesting instance restart to revive its MCP server");
1571
+ this.emit("mcp_restart_requested", { name: this.name, trigger: committed });
1572
+ }, Daemon.MCP_REPLACEMENT_GRACE_MS);
1573
+ this.mcpRestartGraceTimer.unref?.();
1574
+ }
1575
+ /**
1576
+ * A recovery is already reshaping the CLI: crash-loop handling
1577
+ * (healthCheckPaused), a respawn in flight (spawning), or a pause
1578
+ * (isPaused / frozen monitors). Each ends in a fresh CLI — and with it a
1579
+ * fresh MCP server — or in supervision ending, where a restart on top would
1580
+ * fight the recovery. Either way the revival restart is moot.
1581
+ */
1582
+ mcpRestartCancelledByRecovery(trigger) {
1583
+ if (!(this.healthCheckPaused || this.spawning || this.isPaused || this.runtimeMonitorsFrozen || this.stormWindow?.isActive()))
1584
+ return false;
1585
+ this.logger.warn({ trigger }, "MCP revival restart cancelled — instance is already pausing/spawning/recovering");
1586
+ this.clearMcpRestartRequest();
1587
+ return true;
1588
+ }
1589
+ /**
1590
+ * Emit a dead-MCP report that was held back as a suspected replacement, once
1591
+ * it is clear no replacement is coming. Consumes the once-per-pid budget
1592
+ * here — deferring was to avoid spending it on a false alarm, not to drop
1593
+ * the incident.
1594
+ */
1595
+ reportDeferredMcpDeath() {
1596
+ const pid = this.mcpDeathDeferredForPid;
1597
+ if (pid == null || this.mcpDeathNotifiedForPid === pid)
1598
+ return;
1599
+ if (this.mcpServerAlive().alive) {
1600
+ this.logger.info({ pid }, "Deferred MCP death not reported — a live MCP server is serving");
1601
+ this.mcpDeathDeferredForPid = null;
1192
1602
  return;
1193
1603
  }
1604
+ const authSuspected = this.authFailureUnresolved;
1605
+ const autoRestart = this.config.mcp_auto_restart !== false && !authSuspected;
1606
+ this.mcpDeathNotifiedForPid = pid;
1607
+ this.lastMcpDeathNotifiedAt = Date.now();
1608
+ this.logger.error({ pid, autoRestart, authSuspected }, "MCP server process is gone — no replacement arrived, instance has no agend tools");
1609
+ this.emit("mcp_died", { name: this.name, pid, autoRestart, authSuspected });
1610
+ }
1611
+ /** Socket-layer answer to pingMcpSockets(); records the pid the server reports. */
1612
+ handleMcpPong(msg, socket) {
1613
+ const waiter = this.mcpPingWaiters.get(msg.requestId);
1614
+ if (waiter) {
1615
+ this.mcpPingWaiters.delete(msg.requestId);
1616
+ waiter(true);
1617
+ }
1618
+ if (typeof msg.pid === "number")
1619
+ this.socketPids.set(socket, msg.pid);
1620
+ }
1621
+ /** Live IPC sockets of mcp-server processes that announced THIS instance's session. */
1622
+ liveMcpSockets() {
1623
+ const out = [];
1624
+ for (const [socket, sessionName] of this.socketSessionNames) {
1625
+ if (sessionName === this.name && !socket.destroyed)
1626
+ out.push(socket);
1627
+ }
1628
+ return out;
1629
+ }
1630
+ /**
1631
+ * MCP liveness with the pid slot as the first opinion and the IPC layer as
1632
+ * the authority: a live connection from a server announcing our session is
1633
+ * proof that tools are being served, whatever channel.mcp.pid says.
1634
+ */
1635
+ mcpServerAlive() {
1636
+ const status = mcpServerState(this.instanceDir);
1637
+ if (status.state === "alive")
1638
+ return { alive: true, source: "pid", pid: status.pid };
1639
+ const sockets = this.liveMcpSockets();
1640
+ if (sockets.length > 0)
1641
+ return { alive: true, source: "connection", pid: this.socketPids.get(sockets[sockets.length - 1]) };
1642
+ return { alive: false, source: "none", pid: status.state === "dead" ? status.pid : undefined };
1643
+ }
1644
+ /** Point the pid slot at the server that is actually serving, so pid-based readers agree with the IPC layer. */
1645
+ repairMcpPidSlot(pid, reason) {
1646
+ if (!pid)
1647
+ return;
1648
+ try {
1649
+ writeFileSync(join(this.instanceDir, "channel.mcp.pid"), String(pid));
1650
+ this.logger.info({ pid, reason }, "channel.mcp.pid repaired to the live MCP server");
1651
+ }
1652
+ catch (err) {
1653
+ this.logger.debug({ err }, "could not repair channel.mcp.pid");
1654
+ }
1655
+ }
1656
+ /**
1657
+ * Active, LLM-free confirmation: ask each live socket to answer a ping at
1658
+ * the socket layer. Only logged — the open connection is already the
1659
+ * decision; a missing pong means an older mcp-server build or a wedged
1660
+ * process, and a truly dead socket closes on its own and drops out of
1661
+ * liveMcpSockets() by the next tick.
1662
+ */
1663
+ pingMcpSockets(sockets) {
1664
+ for (const socket of sockets) {
1665
+ const requestId = ++this.mcpPingSeq;
1666
+ const timer = setTimeout(() => {
1667
+ if (!this.mcpPingWaiters.delete(requestId))
1668
+ return;
1669
+ this.logger.warn({ pid: this.socketPids.get(socket) }, "connected MCP server did not answer ping within 2s (older mcp-server build, or wedged)");
1670
+ }, Daemon.MCP_PING_TIMEOUT_MS);
1671
+ timer.unref?.();
1672
+ this.mcpPingWaiters.set(requestId, () => {
1673
+ clearTimeout(timer);
1674
+ this.logger.debug({ pid: this.socketPids.get(socket) }, "MCP server answered ping");
1675
+ });
1676
+ if (!this.ipcServer?.send(socket, { type: "ping", requestId })) {
1677
+ this.mcpPingWaiters.delete(requestId);
1678
+ clearTimeout(timer);
1679
+ }
1680
+ }
1681
+ }
1682
+ /**
1683
+ * Evidence that a server for this instance is serving (mcp_ready, a tool
1684
+ * call, a live connection). Stands down any pending revival restart and,
1685
+ * if an alarm had already gone out, retracts it.
1686
+ */
1687
+ noteMcpProofOfLife(source, pid) {
1688
+ const hadAlarm = this.mcpDeathNotifiedForPid != null;
1689
+ const hadPending = this.mcpRestartPending || this.mcpDeathDeferredForPid != null;
1690
+ if (!hadAlarm && !hadPending)
1691
+ return;
1692
+ if (hadPending)
1693
+ this.logger.info({ source, pid }, "MCP server is serving again — standing down the pending revival restart");
1694
+ if (hadAlarm)
1695
+ this.emitMcpRecovered(source, pid);
1696
+ this.mcpDeathNotifiedForPid = null;
1697
+ this.mcpDeathDeferredForPid = null;
1194
1698
  this.clearMcpRestartRequest();
1195
- this.logger.warn({ trigger }, "Requesting instance restart to revive its MCP server");
1196
- this.emit("mcp_restart_requested", { name: this.name, trigger });
1699
+ if (pid && mcpServerState(this.instanceDir).state !== "alive")
1700
+ this.repairMcpPidSlot(pid, source);
1701
+ }
1702
+ emitMcpRecovered(source, pid) {
1703
+ this.logger.info({ source, pid }, "MCP server recovered — retracting the earlier death report");
1704
+ this.emit("mcp_recovered", { name: this.name, source, pid });
1197
1705
  }
1198
1706
  clearMcpRestartRequest() {
1199
1707
  this.mcpRestartPending = false;
@@ -1201,6 +1709,11 @@ export class Daemon extends EventEmitter {
1201
1709
  clearTimeout(this.mcpRestartStaleTimer);
1202
1710
  this.mcpRestartStaleTimer = null;
1203
1711
  }
1712
+ if (this.mcpRestartGraceTimer) {
1713
+ clearTimeout(this.mcpRestartGraceTimer);
1714
+ this.mcpRestartGraceTimer = null;
1715
+ }
1716
+ this.mcpRestartGraceTrigger = null;
1204
1717
  }
1205
1718
  startHealthCheck() {
1206
1719
  if (this.runtimeMonitorsFrozen || this.healthCheckTimer)
@@ -1234,7 +1747,7 @@ export class Daemon extends EventEmitter {
1234
1747
  this.emitSupervisionEnded("instance directory was removed from disk", "Recreate the instance, or delete it from fleet.yaml.");
1235
1748
  return;
1236
1749
  }
1237
- if (!this.tmux || this.spawning || this.healthCheckPaused || Daemon.tmuxServerPaused) {
1750
+ if (!this.tmux || this.spawning || this.healthCheckPaused) {
1238
1751
  scheduleNext();
1239
1752
  return;
1240
1753
  }
@@ -1292,28 +1805,29 @@ export class Daemon extends EventEmitter {
1292
1805
  let nullReason;
1293
1806
  if (!paneStatus) {
1294
1807
  const serverAlive = await TmuxManager.sessionExists(this.tmuxSessionName);
1295
- if (!serverAlive) {
1808
+ // A server may have already restarted before this daemon's health
1809
+ // tick. The new PID is the durable generation boundary: treat it as
1810
+ // a fleet storm even though `has-session` is true again.
1811
+ const generationChanged = serverAlive
1812
+ ? this.stormWindow?.observeServerAlive(await TmuxManager.getServerPid(this.tmuxSessionName)) === true
1813
+ : false;
1814
+ if (generationChanged)
1815
+ this.emit("tmux_server_crash", this.name);
1816
+ if (!serverAlive || generationChanged || this.stormWindow?.needsRecovery(this.name)) {
1296
1817
  crashType = "server";
1297
- nullReason = "server_gone";
1298
- this.logger.error(`tmux server died — all ${cliLabel} windows lost`);
1299
- // Fleet-level circuit breaker: pause all instances on repeated tmux server crashes
1300
- Daemon.tmuxServerCrashTimestamps.push(Date.now());
1301
- const cutoff = Date.now() - 5 * 60_000;
1302
- Daemon.tmuxServerCrashTimestamps = Daemon.tmuxServerCrashTimestamps.filter(t => t > cutoff);
1303
- if (Daemon.tmuxServerCrashTimestamps.length >= 2 && !Daemon.tmuxServerPaused) {
1304
- Daemon.tmuxServerPaused = true;
1305
- this.logger.error("Fleet-level tmux server circuit breaker triggered — pausing all respawns for 30s");
1306
- this.emit("tmux_server_crash", this.name);
1307
- if (!Daemon.tmuxServerRecoveryTimer) {
1308
- Daemon.tmuxServerRecoveryTimer = setTimeout(() => {
1309
- Daemon.tmuxServerRecoveryTimer = null;
1310
- Daemon.tmuxServerPaused = false;
1311
- }, 30_000);
1312
- }
1313
- scheduleNext();
1314
- return;
1818
+ nullReason = serverAlive ? "server_storm_window_loss" : "server_gone";
1819
+ if (!serverAlive) {
1820
+ this.logger.error(`tmux server died — all ${cliLabel} windows lost`);
1821
+ const distinct = this.stormWindow?.recordServerDead(this.name, [this.name]) ?? true;
1822
+ if (distinct)
1823
+ this.emit("tmux_server_crash", this.name);
1824
+ }
1825
+ else {
1826
+ this.stormWindow?.addAffected(this.name);
1315
1827
  }
1316
- await new Promise(r => setTimeout(r, 2_000)); // let session stabilize
1828
+ // The fleet breaker owns the delay and extends it on every new
1829
+ // server generation. Do not schedule a competing fixed timer.
1830
+ await this.stormWindow?.waitForSpawnAllowed();
1317
1831
  }
1318
1832
  else {
1319
1833
  // null but server alive: window-level disappearance. Probe whether
@@ -1409,6 +1923,14 @@ export class Daemon extends EventEmitter {
1409
1923
  }
1410
1924
  // Append to crash history
1411
1925
  this.appendCrashHistory({ exitCode, lastOutput, crashType, reason: nullReason });
1926
+ // A fleet-wide server loss is one infrastructure incident, not N
1927
+ // independent CLI crash loops. StormWindow owns its escalation; do
1928
+ // not let the same three server generations permanently trip every
1929
+ // daemon's per-instance 3-in-5m breaker.
1930
+ if (crashType === "server") {
1931
+ this.crashTimestamps = [];
1932
+ this.crashCount = 0;
1933
+ }
1412
1934
  if (max_retries <= 0) {
1413
1935
  this.healthCheckPaused = true;
1414
1936
  this.logger.warn(`${cliLabel} window died — automatic restart is disabled`);
@@ -1495,9 +2017,15 @@ export class Daemon extends EventEmitter {
1495
2017
  this.setProcessStatus("running");
1496
2018
  this.logger.info({ resumed }, `Respawned ${cliLabel} window after crash`);
1497
2019
  this.emit("crash_respawn", this.name);
2020
+ // Emit the per-instance audit event while the storm is still active
2021
+ // so its chat notification is suppressed into the fleet summary.
2022
+ if (crashType === "server")
2023
+ this.stormWindow?.markRecovered(this.name);
1498
2024
  }
1499
2025
  catch (err) {
1500
- this.logger.error({ err }, `Failed to respawn ${cliLabel} window`);
2026
+ if (!this.handOffBackendUnreachableRespawn(err)) {
2027
+ this.logger.error({ err }, `Failed to respawn ${cliLabel} window`);
2028
+ }
1501
2029
  }
1502
2030
  }
1503
2031
  catch (err) {
@@ -1606,10 +2134,37 @@ export class Daemon extends EventEmitter {
1606
2134
  // A prompt is not an error and must not enter the PTY recovery gate.
1607
2135
  // Continue scanning real errors in this same snapshot.
1608
2136
  }
1609
- // Auto-dismiss runtime dialogs (e.g. Codex rate limit model switch)
2137
+ // If a backend can derive activity from this exact pane, trust its
2138
+ // explicit null (tool completed) instead of falling back to a possibly
2139
+ // stale transcript activity from the previous scan.
2140
+ const paneActivity = this.backend?.getPaneActivity
2141
+ ? this.backend.getPaneActivity(pane)
2142
+ : this.currentActivity;
2143
+ const hasPendingWork = this.pendingWork.hasPendingWork();
2144
+ if (!hasPendingWork)
2145
+ this.blockingProcessDetector.reset();
2146
+ const blockingProcess = hasPendingWork
2147
+ ? this.blockingProcessDetector.observe(pane, paneActivity, Date.now())
2148
+ : null;
2149
+ if (blockingProcess) {
2150
+ // Reuse the hang-notification bridge: it offers explicit Restart/Wait
2151
+ // choices and, unlike pty_error, does not retire a still-useful Cancel
2152
+ // button merely because a foreground process owns stdin.
2153
+ this.logger.warn(blockingProcess, "Foreground process is blocking the agent input loop");
2154
+ this.hangDetector?.emit("hang", { unchangedForMs: blockingProcess.blockedForMs });
2155
+ }
2156
+ // Auto-dismiss runtime dialogs (e.g. Codex rate limit model switch, or
2157
+ // Claude's session-resume prompt painted after the startup scan moved
2158
+ // on). Runtime table only: startup patterns are loose on purpose and
2159
+ // must never send keys into an ordinary transcript.
2160
+ let blockingSeen = null;
1610
2161
  for (const dialog of dialogs) {
1611
- if (!dialog.pattern.test(pane))
2162
+ if (!Daemon.dialogMatches(dialog, pane))
1612
2163
  continue;
2164
+ if (dialog.blocksDelivery || dialog.holdOnly)
2165
+ blockingSeen = dialog;
2166
+ if (dialog.holdOnly)
2167
+ break; // recognised, deliberately not answered; trackDialogParked reports it
1613
2168
  // These keys go straight into the pane. Sent while a message delivery is
1614
2169
  // mid-transaction, an `Escape` wipes the pasted text and an `Enter`
1615
2170
  // submits it half-composed — the user sees a message that vanished. Skip
@@ -1629,10 +2184,14 @@ export class Daemon extends EventEmitter {
1629
2184
  }
1630
2185
  });
1631
2186
  if (!dismissed) {
1632
- this.logger.debug({ dialog: dialog.description }, "Dialog dismissal deferred — pane write in flight");
2187
+ this.logger.info({ dialog: dialog.description }, "Dialog dismissal deferred — pane write in flight");
1633
2188
  }
2189
+ this.trackDialogParked(blockingSeen);
1634
2190
  return; // Dialog handled (or deliberately deferred): skip error checks this cycle
1635
2191
  }
2192
+ this.trackDialogParked(blockingSeen);
2193
+ if (blockingSeen)
2194
+ return; // held dialog: skip error checks this cycle
1636
2195
  this.evaluateErrorPatterns(pane, patterns, readyPattern, Date.now(), busyPattern);
1637
2196
  }
1638
2197
  catch {
@@ -1678,6 +2237,7 @@ export class Daemon extends EventEmitter {
1678
2237
  if (seen > 0)
1679
2238
  this.lastErrorCount.set(Daemon.errorPatternKey(ep), seen);
1680
2239
  }
2240
+ this.authFailureUnresolved = false;
1681
2241
  this.clearErrorRecoveryGate();
1682
2242
  this.logger.info({ downtime_s: downtime }, "PTY error recovered — agent is ready again");
1683
2243
  this.emit("pty_recovered", { name: this.name, downtime_s: downtime });
@@ -1698,6 +2258,19 @@ export class Daemon extends EventEmitter {
1698
2258
  this.logger.warn({ errorType: this.lastDetectedErrorType }, "PTY error recovery deadline expired — resuming error scan");
1699
2259
  this.clearErrorRecoveryGate();
1700
2260
  }
2261
+ // A model_error deferred mid-turn: the turn is over (or the pane left
2262
+ // "working"), so decide now — still on screen means the CLI gave up and
2263
+ // the user must act; gone means a retry got past it and nobody needs to know.
2264
+ if (this.pendingModelErrorKey && this.instanceState !== "working") {
2265
+ const key = this.pendingModelErrorKey;
2266
+ this.pendingModelErrorKey = null;
2267
+ const ep = patterns.find(candidate => Daemon.errorPatternKey(candidate) === key);
2268
+ if (ep && ep.pattern.test(pane)) {
2269
+ this.emitErrorPattern(ep, key, pane, now);
2270
+ return;
2271
+ }
2272
+ this.logger.debug({ errorKey: key }, "Deferred model_error cleared before idle — dropped");
2273
+ }
1701
2274
  // State: monitoring — count-based new-error detection over the full pane.
1702
2275
  for (const ep of patterns) {
1703
2276
  const key = Daemon.errorPatternKey(ep);
@@ -1725,25 +2298,49 @@ export class Daemon extends EventEmitter {
1725
2298
  continue;
1726
2299
  }
1727
2300
  this.lastErrorCount.set(key, count);
1728
- // skipRecoveryWait: this error self-recovers (e.g. a timeout — Kiro is
1729
- // back at its prompt immediately). Keep monitoring so the next occurrence
1730
- // can fire without waiting for a possibly startup-only ready pattern.
1731
- if (!ep.skipRecoveryWait) {
1732
- this.errorWaitingForRecovery = true;
1733
- this.errorDetectedAt = now;
1734
- this.errorRecoveryDeadlineAt = now + Daemon.ERROR_RECOVERY_TIMEOUT_MS;
1735
- this.activeErrorPatternKey = key;
1736
- this.lastDetectedErrorType = ep.type;
1737
- }
1738
- this.lastErrorNotifiedAt.set(key, now);
1739
- if (ep.action === "failover")
1740
- this.lastFailoverAt = now;
1741
- const message = this.resolveErrorMessage(pane, ep);
1742
- this.logger.warn({ errorType: ep.type, action: ep.action }, `PTY error detected: ${message}`);
1743
- this.emit("pty_error", { name: this.name, ...ep, message });
2301
+ // Kiro's `auto` model picks a concrete model per turn and reroutes around
2302
+ // a temporarily unavailable one on its own — telling the user to run
2303
+ // /model would ask them to fix something the CLI is already handling.
2304
+ if (ep.type === "model_error" && ep.action === "notify" && this.config.model === "auto") {
2305
+ this.logger.debug({ errorType: ep.type }, "model_error suppressed — model 'auto' reroutes on its own");
2306
+ continue;
2307
+ }
2308
+ // A mid-turn model error is often a transient the CLI retries past. Hold
2309
+ // the notification until the turn ends and notify only if the error is
2310
+ // still on the idle screen (see the deferred check above).
2311
+ if (ep.type === "model_error" && ep.action === "notify" && this.instanceState === "working") {
2312
+ this.pendingModelErrorKey = key;
2313
+ this.logger.debug({ errorType: ep.type }, "model_error deferred — instance is mid-turn, re-checking at idle");
2314
+ continue;
2315
+ }
2316
+ this.emitErrorPattern(ep, key, pane, now);
1744
2317
  break; // Only handle first unsuppressed new error per scan
1745
2318
  }
1746
2319
  }
2320
+ /** Arm the recovery gates and emit one detected error pattern. */
2321
+ emitErrorPattern(ep, key, pane, now) {
2322
+ // skipRecoveryWait: this error self-recovers (e.g. a timeout — Kiro is
2323
+ // back at its prompt immediately). Keep monitoring so the next occurrence
2324
+ // can fire without waiting for a possibly startup-only ready pattern.
2325
+ if (!ep.skipRecoveryWait) {
2326
+ this.errorWaitingForRecovery = true;
2327
+ this.errorDetectedAt = now;
2328
+ this.errorRecoveryDeadlineAt = now + Daemon.ERROR_RECOVERY_TIMEOUT_MS;
2329
+ this.activeErrorPatternKey = key;
2330
+ this.lastDetectedErrorType = ep.type;
2331
+ }
2332
+ this.lastErrorNotifiedAt.set(key, now);
2333
+ if (ep.action === "failover")
2334
+ this.lastFailoverAt = now;
2335
+ const message = this.resolveErrorMessage(pane, ep);
2336
+ // Remember an auth failure for as long as it is unresolved. An auth-broken
2337
+ // CLI cannot finish the turn it is holding, so every other detector in this
2338
+ // scan would otherwise keep reporting the same dead instance forever.
2339
+ if (ep.type === "auth_error")
2340
+ this.authFailureUnresolved = true;
2341
+ this.logger.warn({ errorType: ep.type, action: ep.action }, `PTY error detected: ${message}`);
2342
+ this.emit("pty_error", { name: this.name, ...ep, message });
2343
+ }
1747
2344
  /**
1748
2345
  * Notification text for a detected pattern. `formatMessage` patterns build it
1749
2346
  * from the match so the user gets the specifics (e.g. which period and what
@@ -1782,6 +2379,97 @@ export class Daemon extends EventEmitter {
1782
2379
  const cancelKey = this.backend?.getCancelKey() ?? "Escape";
1783
2380
  await this.tmux?.sendSpecialKey(cancelKey);
1784
2381
  }
2382
+ /**
2383
+ * Drop every not-yet-started pane delivery which predates a user cancel.
2384
+ * Promise chains cannot be removed, so entries become generation-checked
2385
+ * no-ops. An entry already past its check is allowed to finish its current PTY
2386
+ * transaction; the interrupt key still cancels the CLI generation.
2387
+ */
2388
+ clearPendingDeliveries(fleetEpoch) {
2389
+ this.deliveryEpoch = fleetEpoch === undefined
2390
+ ? this.deliveryEpoch + 1
2391
+ : Math.max(this.deliveryEpoch, fleetEpoch);
2392
+ this.logger.info({ deliveryEpoch: this.deliveryEpoch }, "Pending delivery queue cleared by user cancel");
2393
+ }
2394
+ captureDeliveryEpoch(value) {
2395
+ const epoch = typeof value === "number" && Number.isSafeInteger(value) && value >= 0
2396
+ ? value
2397
+ : this.deliveryEpoch;
2398
+ if (epoch > this.deliveryEpoch)
2399
+ this.deliveryEpoch = epoch;
2400
+ return epoch;
2401
+ }
2402
+ isDeliveryEpochCurrent(epoch) {
2403
+ return epoch === this.deliveryEpoch;
2404
+ }
2405
+ queueRawPaste(rawText, deliveryEpoch = this.deliveryEpoch, confirmClear = false) {
2406
+ if (!this.tmux || !this.isDeliveryEpochCurrent(deliveryEpoch))
2407
+ return;
2408
+ this.pasteLock = this.pasteLock.then(async () => {
2409
+ if (!this.isDeliveryEpochCurrent(deliveryEpoch)) {
2410
+ this.logger.info("Pending raw delivery dropped by user cancel");
2411
+ return;
2412
+ }
2413
+ await this.wake();
2414
+ if (!this.isDeliveryEpochCurrent(deliveryEpoch))
2415
+ return;
2416
+ await this.deliverMessage(rawText, undefined, { deliveryEpoch });
2417
+ if (confirmClear)
2418
+ await this.confirmBackendClearDialog();
2419
+ this.logger.debug({ text: rawText.slice(0, 100) }, "Raw paste delivered");
2420
+ }).catch(err => {
2421
+ this.logger.warn({ err: err.message }, "raw_paste delivery error");
2422
+ });
2423
+ }
2424
+ /**
2425
+ * Confirm a backend's terminal-level /clear prompt after the channel-level
2426
+ * admin confirmation has already succeeded. Poll instead of pasting `/clear\ny`
2427
+ * as one buffer: Kiro versions that clear immediately interpret that queued
2428
+ * `y` as a brand-new user message.
2429
+ */
2430
+ async confirmBackendClearDialog() {
2431
+ const dialog = this.backend?.getClearConfirmationDialog?.();
2432
+ if (!dialog || !this.tmux)
2433
+ return;
2434
+ const matches = (pane) => {
2435
+ dialog.pattern.lastIndex = 0;
2436
+ return dialog.pattern.test(pane);
2437
+ };
2438
+ const deadline = Date.now() + 3_000;
2439
+ while (Date.now() <= deadline) {
2440
+ const pane = await this.tmux.capturePane().catch(() => "");
2441
+ if (matches(pane)) {
2442
+ const sent = await this.paneWriteLock.run(async () => {
2443
+ // Re-check under the write lock so a prompt that disappeared between
2444
+ // capture and lock acquisition never receives a stray literal `y`.
2445
+ const currentPane = await this.tmux.capturePane().catch(() => "");
2446
+ if (!matches(currentPane))
2447
+ return false;
2448
+ const specialKeys = new Set(["Up", "Down", "Enter", "Escape", "Right", "Left"]);
2449
+ for (const key of dialog.keys) {
2450
+ const ok = specialKeys.has(key)
2451
+ ? await this.tmux.sendSpecialKey(key)
2452
+ : await this.tmux.sendKeys(key);
2453
+ if (!ok)
2454
+ return false;
2455
+ await new Promise(resolve => setTimeout(resolve, 100));
2456
+ }
2457
+ return true;
2458
+ });
2459
+ if (sent) {
2460
+ this.logger.info({ dialog: dialog.description }, "Auto-confirmed backend clear dialog");
2461
+ }
2462
+ else {
2463
+ this.logger.warn({ dialog: dialog.description }, "Backend clear dialog disappeared or confirmation keys failed");
2464
+ }
2465
+ return;
2466
+ }
2467
+ await new Promise(resolve => setTimeout(resolve, 100));
2468
+ }
2469
+ // Recent Kiro releases may clear immediately without a second prompt. That
2470
+ // is a successful no-op here, not an error.
2471
+ this.logger.debug({ dialog: dialog.description }, "Backend clear dialog did not appear");
2472
+ }
1785
2473
  /** Send the backend-specific graceful quit command/key sequence. */
1786
2474
  async sendQuitSequence() {
1787
2475
  if (!this.tmux || !this.backend)
@@ -1789,18 +2477,20 @@ export class Daemon extends EventEmitter {
1789
2477
  const quitCmd = this.backend.getQuitCommand();
1790
2478
  const quitKey = this.backend.getQuitKey?.();
1791
2479
  if (quitCmd) {
1792
- await this.tmux.sendKeys(quitCmd);
2480
+ if (!await this.tmux.sendKeys(quitCmd))
2481
+ return false;
1793
2482
  // Delay before Enter to prevent tmux server races when instances stop in
1794
2483
  // parallel (same pattern as pasteText).
1795
2484
  await new Promise(r => setTimeout(r, 150));
1796
- await this.tmux.sendSpecialKey("Enter");
1797
- return true;
2485
+ return this.tmux.sendSpecialKey("Enter");
1798
2486
  }
1799
2487
  if (!quitKey)
1800
2488
  return false;
1801
2489
  const presses = Math.max(1, Math.floor(this.backend.getQuitKeyPresses?.() ?? 1));
1802
2490
  for (let i = 0; i < presses; i++) {
1803
- await this.tmux.sendSpecialKey(quitKey);
2491
+ const sent = await this.tmux.sendSpecialKey(quitKey);
2492
+ if (!sent)
2493
+ return false;
1804
2494
  if (i + 1 < presses)
1805
2495
  await new Promise(r => setTimeout(r, 250));
1806
2496
  }
@@ -1833,8 +2523,23 @@ export class Daemon extends EventEmitter {
1833
2523
  }
1834
2524
  }
1835
2525
  }
1836
- if (!killed)
1837
- this.logger.warn("CLI did not exit gracefully within 3s, force killing window");
2526
+ if (!killed) {
2527
+ this.logger.warn({ quitSent }, "CLI did not exit gracefully within 3s — falling back to SIGTERM");
2528
+ await this.killProcessTree("SIGTERM");
2529
+ for (let i = 0; i < 5; i++) {
2530
+ await new Promise(r => setTimeout(r, 200));
2531
+ const status = await this.tmux.getPaneStatus();
2532
+ if (!status || !status.alive) {
2533
+ killed = true;
2534
+ break;
2535
+ }
2536
+ }
2537
+ }
2538
+ if (!killed) {
2539
+ this.logger.warn("CLI process tree survived SIGTERM — falling back to SIGKILL");
2540
+ await this.killProcessTree("SIGKILL");
2541
+ await new Promise(r => setTimeout(r, 200));
2542
+ }
1838
2543
  // Always kill window — remain-on-exit keeps dead panes around after CLI exits
1839
2544
  const stoppedWindowId = this.tmux.getWindowId();
1840
2545
  await this.tmux.killWindow();
@@ -1883,6 +2588,33 @@ export class Daemon extends EventEmitter {
1883
2588
  stateChangedAt: Date.now(),
1884
2589
  };
1885
2590
  }
2591
+ /**
2592
+ * Answer a FleetManager state query. Most callers only need the cached state,
2593
+ * but lifecycle decisions such as post-reply Cancel retirement need an
2594
+ * authoritative pane observation: after startup the last transition can stay
2595
+ * "working" even though the CLI has since settled without emitting another
2596
+ * control-mode output record.
2597
+ */
2598
+ async respondToInstanceStateQuery(msg, socket) {
2599
+ try {
2600
+ if (msg.refresh === true) {
2601
+ await this.captureAndEvaluateInstanceState("state_query");
2602
+ }
2603
+ const snapshot = this.getInstanceStateSnapshot();
2604
+ this.ipcServer?.send(socket, {
2605
+ type: "instance_state_response",
2606
+ requestId: msg.requestId,
2607
+ instanceName: this.name,
2608
+ ...snapshot,
2609
+ state: this.isPaused ? "paused" : snapshot.state,
2610
+ processStatus: this.processStatus,
2611
+ pausedAt: this.lastPausedAt,
2612
+ });
2613
+ }
2614
+ catch (err) {
2615
+ this.logger.debug({ err: err.message }, "Instance state query failed");
2616
+ }
2617
+ }
1886
2618
  /** Gracefully stop the CLI while keeping its remain-on-exit tmux window. */
1887
2619
  /**
1888
2620
  * Mark the instance to pause as soon as its pane is idle. Used for auth
@@ -1902,7 +2634,14 @@ export class Daemon extends EventEmitter {
1902
2634
  if (this.getPauseWakeState() !== "active")
1903
2635
  return;
1904
2636
  }
1905
- if (this.instanceState !== "idle" || this.pasteQueueDepth > 0) {
2637
+ // `pauseAllowStuck` is consumed here, not merely read: it authorises exactly
2638
+ // one pause from a stuck pane (the auth-deferred one) and must never leak
2639
+ // into an ordinary idle-timeout pause of a busy instance.
2640
+ const allowStuck = this.pauseAllowStuck;
2641
+ this.pauseAllowStuck = false;
2642
+ const pausableState = this.instanceState === "idle"
2643
+ || (allowStuck && this.instanceState === "stuck");
2644
+ if (!pausableState || this.pasteQueueDepth > 0) {
1906
2645
  this.pauseRequested = false;
1907
2646
  return;
1908
2647
  }
@@ -1966,6 +2705,10 @@ export class Daemon extends EventEmitter {
1966
2705
  // An explicit wake (e.g. after the user re-logs in) cancels a deferred
1967
2706
  // auth pause — otherwise the instance would pause again the moment it idles.
1968
2707
  this.pausePending = false;
2708
+ this.authFailureUnresolved = false;
2709
+ this.loginScreenReported = false;
2710
+ this.fatalStartupReported = false;
2711
+ this.fatalStartupBlocked = false;
1969
2712
  if (this.pauseWakeState === "active")
1970
2713
  return;
1971
2714
  if (this.pauseWakeState === "pausing")
@@ -1978,19 +2721,30 @@ export class Daemon extends EventEmitter {
1978
2721
  }
1979
2722
  this.pauseWakeState = "waking";
1980
2723
  this.beginSpawn();
2724
+ // A wake is a real resume launch: it gets at least the backend's resume
2725
+ // budget (kiro: 60s — the conversation must come back from the backend
2726
+ // before anything paints), never less than the caller's or the configured
2727
+ // startup_timeout_ms.
2728
+ const budgetMs = this.wakeBudgetMs(timeoutMs);
2729
+ // No outer race. trySpawn cannot be cancelled: racing it against a timer
2730
+ // and giving up left the daemon "paused" (monitors frozen) while the spawn
2731
+ // went on to revive the window behind its back. Every phase inside is
2732
+ // bounded on its own terms — the SpawnGate wait is deliberate storm /
2733
+ // concurrency coordination, first output + idle is `budgetMs`, the dialog
2734
+ // scan is STARTUP_DIALOG_BUDGET_MS — so the outcome is always trySpawn's
2735
+ // own verdict. A soft timer only makes a slow wake visible in the log.
1981
2736
  const transition = this.autoPauseController.wakeOnDeliver(async () => {
1982
- let timeout;
2737
+ const slow = setTimeout(() => {
2738
+ this.logger.warn({ budgetMs }, "Wake is exceeding its budget — still waiting for the spawn (it cannot be cancelled)");
2739
+ }, budgetMs + STARTUP_DIALOG_BUDGET_MS);
2740
+ slow.unref?.();
1983
2741
  try {
1984
- const ready = await Promise.race([
1985
- this.trySpawn(true, timeoutMs),
1986
- new Promise(resolve => { timeout = setTimeout(() => resolve(false), timeoutMs); }),
1987
- ]);
2742
+ const ready = await this.trySpawn(true, budgetMs);
1988
2743
  if (!ready)
1989
- throw new Error(`Wake timed out before CLI became ready (${timeoutMs}ms)`);
2744
+ throw new Error(`Wake failed: the CLI did not become ready (budget ${budgetMs}ms)`);
1990
2745
  }
1991
2746
  finally {
1992
- if (timeout)
1993
- clearTimeout(timeout);
2747
+ clearTimeout(slow);
1994
2748
  }
1995
2749
  });
1996
2750
  this.pauseWakeTransition = transition;
@@ -2062,8 +2816,13 @@ export class Daemon extends EventEmitter {
2062
2816
  if (snapshot.state !== "idle")
2063
2817
  this.pauseRequested = false;
2064
2818
  // A pause deferred by an auth failure: retry the moment the pane settles.
2065
- if (this.pausePending && snapshot.state === "idle" && this.pasteQueueDepth === 0) {
2819
+ // "stuck" counts as settled here, and that is the whole point — an expired
2820
+ // session cannot complete the turn it is holding, so the pane never returns
2821
+ // to idle and waiting for idle meant waiting forever. The instance stayed
2822
+ // live, kept its warm slot, and re-raised a hang alert on every scan.
2823
+ if (this.pausePending && (snapshot.state === "idle" || snapshot.state === "stuck") && this.pasteQueueDepth === 0) {
2066
2824
  this.pausePending = false;
2825
+ this.pauseAllowStuck = snapshot.state === "stuck";
2067
2826
  this.emit("auto_pause_requested", { name: this.name, idleSince: snapshot.stateChangedAt });
2068
2827
  return;
2069
2828
  }
@@ -2093,6 +2852,7 @@ export class Daemon extends EventEmitter {
2093
2852
  return;
2094
2853
  this.turnHadInbound = true;
2095
2854
  this.turnOutboundDelivered = false;
2855
+ this.turnReplyDelivered = false;
2096
2856
  this.turnCorrelationId = meta.correlation_id || undefined;
2097
2857
  // The last non-empty line of what we pasted: everything on screen after it
2098
2858
  // is the agent's own output.
@@ -2108,13 +2868,38 @@ export class Daemon extends EventEmitter {
2108
2868
  maybeProxyReplyOnTurnEnd(pane) {
2109
2869
  const hadInbound = this.turnHadInbound;
2110
2870
  const delivered = this.turnOutboundDelivered;
2871
+ const replyDelivered = this.turnReplyDelivered;
2111
2872
  const correlationId = this.turnCorrelationId;
2112
2873
  const inboundMarker = this.turnInboundMarker;
2113
2874
  this.turnHadInbound = false;
2114
2875
  this.turnOutboundDelivered = false;
2876
+ this.turnReplyDelivered = false;
2115
2877
  this.turnCorrelationId = undefined;
2116
2878
  this.turnInboundMarker = undefined;
2117
- if (!hadInbound || delivered || this.isPaused)
2879
+ if (!hadInbound || this.isPaused)
2880
+ return;
2881
+ // Claude sometimes prints a broken XML tool call as plain text and returns
2882
+ // idle without ever invoking `reply`. This is independent of MCP liveness:
2883
+ // the model malformed the call before the server could receive it.
2884
+ if (!replyDelivered && this.isClaudeCodeBackend() && pane) {
2885
+ const malformed = detectMalformedClaudeToolCall(pane, { inboundMarker });
2886
+ if (malformed) {
2887
+ const stale = malformed.signature === this.lastMalformedToolCallSignature;
2888
+ this.lastMalformedToolCallSignature = malformed.signature;
2889
+ if (!stale) {
2890
+ const recovered = malformed.text != null;
2891
+ this.logger.warn({ correlationId, recovered }, recovered
2892
+ ? "Malformed Claude tool call detected — attempting to recover reply text"
2893
+ : "Malformed Claude tool call detected — reply text could not be extracted");
2894
+ this.emit("malformed_tool_call", { name: this.name, correlationId, recovered });
2895
+ if (malformed.text)
2896
+ this.sendRecoveredMalformedReply(malformed.text, correlationId);
2897
+ }
2898
+ // Never let the broader dead-MCP fallback relay the XML/chrome too.
2899
+ return;
2900
+ }
2901
+ }
2902
+ if (delivered)
2118
2903
  return;
2119
2904
  // Opt-in: raw pane text can carry secrets past the regex redaction.
2120
2905
  if (this.config.mcp_proxy_reply !== true)
@@ -2124,6 +2909,11 @@ export class Daemon extends EventEmitter {
2124
2909
  return;
2125
2910
  void this.sendProxyReply(pane, inboundMarker, correlationId);
2126
2911
  }
2912
+ isClaudeCodeBackend() {
2913
+ return this.runtimeIdentity?.backend === "claude-code"
2914
+ || this.config.backend === "claude-code"
2915
+ || this.backend?.binaryName === "claude";
2916
+ }
2127
2917
  /** Relay the pane's final text to the channel, marked as a daemon proxy reply. */
2128
2918
  async sendProxyReply(pane, inboundMarker, correlationId) {
2129
2919
  try {
@@ -2141,59 +2931,96 @@ export class Daemon extends EventEmitter {
2141
2931
  let body = `⚠️ [MCP unavailable — proxy reply]\n\n${text}`;
2142
2932
  if (correlationId)
2143
2933
  body += `\n\n(correlation_id: ${correlationId})`;
2144
- const args = { text: body };
2145
- // Same context-bound routing as the reply tool in handleToolCall.
2146
- if (this.lastChatId) {
2147
- args.chat_id = this.lastChatId;
2148
- if (this.lastThreadId)
2149
- args.thread_id = this.lastThreadId;
2150
- }
2151
2934
  this.logger.warn({ correlationId }, "MCP server dead and the turn sent no reply — relaying the pane text as a proxy reply");
2152
2935
  this.emit("mcp_proxy_reply", { name: this.name, correlationId });
2153
- const adapters = this.messageBus.getAllAdapters();
2154
- if (adapters.length > 0) {
2155
- routeToolCall(adapters[0], "reply", args, this.lastThreadId, (_result, error) => {
2156
- if (error)
2157
- this.logger.error({ error }, "Dead-MCP proxy reply failed");
2158
- });
2159
- return;
2160
- }
2161
- if (!this.ipcServer)
2162
- return;
2163
- const fleetReqId = `proxyreply_${++this.proxyReplySeq}`;
2164
- this.ipcServer.broadcast({
2165
- type: "fleet_outbound",
2166
- tool: "reply",
2167
- args,
2168
- fleetRequestId: fleetReqId,
2169
- adapterId: this.lastAdapterId,
2170
- });
2171
- const timeout = setTimeout(() => {
2172
- this.pendingIpcRequests.delete(fleetReqId);
2173
- this.logger.error("Dead-MCP proxy reply timed out waiting for the fleet manager");
2174
- }, daemonBudgetMs("reply"));
2175
- timeout.unref?.();
2176
- this.pendingIpcRequests.set(fleetReqId, (respMsg) => {
2177
- clearTimeout(timeout);
2178
- if (respMsg.error)
2179
- this.logger.error({ error: respMsg.error }, "Dead-MCP proxy reply failed");
2180
- else
2181
- this.logger.info("Dead-MCP proxy reply delivered");
2182
- });
2936
+ this.deliverDaemonReply(body, "proxyreply", "Dead-MCP proxy reply");
2183
2937
  }
2184
2938
  catch (err) {
2185
2939
  this.logger.error({ err: err.message }, "Dead-MCP proxy reply attempt failed");
2186
2940
  }
2187
2941
  }
2942
+ /** Send a narrowly extracted #648 reply; operator notification is a separate event. */
2943
+ sendRecoveredMalformedReply(text, correlationId) {
2944
+ try {
2945
+ this.deliverDaemonReply(text, "malformedreply", "Malformed tool-call recovery");
2946
+ }
2947
+ catch (err) {
2948
+ this.logger.error({ err: err.message, correlationId }, "Malformed tool-call recovery attempt failed");
2949
+ }
2950
+ }
2951
+ /** Context-bound reply path shared by dead-MCP and malformed-call recovery. */
2952
+ deliverDaemonReply(body, requestPrefix, logLabel) {
2953
+ const args = { text: body };
2954
+ if (this.lastChatId) {
2955
+ args.chat_id = this.lastChatId;
2956
+ if (this.lastThreadId)
2957
+ args.thread_id = this.lastThreadId;
2958
+ }
2959
+ const adapters = this.messageBus.getAllAdapters();
2960
+ if (adapters.length > 0) {
2961
+ routeToolCall(adapters[0], "reply", args, this.lastThreadId, (_result, error) => {
2962
+ if (error)
2963
+ this.logger.error({ error }, `${logLabel} failed`);
2964
+ else
2965
+ this.logger.info(`${logLabel} delivered`);
2966
+ });
2967
+ return;
2968
+ }
2969
+ if (!this.ipcServer) {
2970
+ this.logger.error(`${logLabel} failed — no adapter or fleet IPC route`);
2971
+ return;
2972
+ }
2973
+ const fleetReqId = `${requestPrefix}_${++this.proxyReplySeq}`;
2974
+ const timeout = setTimeout(() => {
2975
+ this.pendingIpcRequests.delete(fleetReqId);
2976
+ this.logger.error(`${logLabel} timed out waiting for the fleet manager`);
2977
+ }, daemonBudgetMs("reply"));
2978
+ timeout.unref?.();
2979
+ // Register before broadcast: an in-process/fast adapter can answer in the
2980
+ // same tick, and a waiter installed afterwards would miss that response.
2981
+ this.pendingIpcRequests.set(fleetReqId, (respMsg) => {
2982
+ clearTimeout(timeout);
2983
+ if (respMsg.error)
2984
+ this.logger.error({ error: respMsg.error }, `${logLabel} failed`);
2985
+ else
2986
+ this.logger.info(`${logLabel} delivered`);
2987
+ });
2988
+ this.ipcServer.broadcast({
2989
+ type: "fleet_outbound",
2990
+ tool: "reply",
2991
+ args,
2992
+ fleetRequestId: fleetReqId,
2993
+ adapterId: this.lastAdapterId,
2994
+ });
2995
+ }
2188
2996
  clearInstanceStateIdleTimer() {
2189
2997
  if (this.instanceStateIdleTimer)
2190
2998
  clearTimeout(this.instanceStateIdleTimer);
2191
2999
  this.instanceStateIdleTimer = null;
2192
3000
  }
2193
- clearInstanceStateStuckTimer() {
2194
- if (this.instanceStateStuckTimer)
2195
- clearTimeout(this.instanceStateStuckTimer);
2196
- this.instanceStateStuckTimer = null;
3001
+ clearInstanceStateOutputProbeTimer() {
3002
+ if (this.instanceStateOutputProbeTimer)
3003
+ clearTimeout(this.instanceStateOutputProbeTimer);
3004
+ this.instanceStateOutputProbeTimer = null;
3005
+ }
3006
+ /**
3007
+ * Periodic status-line hooks can generate `%output` without changing a single
3008
+ * visible cell. Probe the pane once per burst: a changed capture is real work;
3009
+ * an identical ready capture remains idle and emits no state edge.
3010
+ */
3011
+ scheduleInstanceStateOutputProbe() {
3012
+ if (this.instanceStateOutputProbeTimer)
3013
+ return;
3014
+ this.instanceStateOutputProbeTimer = setTimeout(() => {
3015
+ this.instanceStateOutputProbeTimer = null;
3016
+ void this.captureAndEvaluateInstanceState("output_probe", this.instanceStateLastOutputAt);
3017
+ }, PERIODIC_REDRAW_PROBE_MS);
3018
+ this.instanceStateOutputProbeTimer.unref?.();
3019
+ }
3020
+ clearInstanceStateStuckTimer() {
3021
+ if (this.instanceStateStuckTimer)
3022
+ clearTimeout(this.instanceStateStuckTimer);
3023
+ this.instanceStateStuckTimer = null;
2197
3024
  }
2198
3025
  scheduleInstanceStateStuckDeadline(changeAt) {
2199
3026
  // Keep one deadline timer per instance. High-volume TUIs can emit hundreds
@@ -2234,6 +3061,8 @@ export class Daemon extends EventEmitter {
2234
3061
  if (this.statePollInFlight) {
2235
3062
  if (reason === "idle_debounce")
2236
3063
  this.scheduleInstanceStateIdleCapture();
3064
+ if (reason === "output_probe")
3065
+ this.scheduleInstanceStateOutputProbe();
2237
3066
  return;
2238
3067
  }
2239
3068
  this.statePollInFlight = true;
@@ -2242,9 +3071,19 @@ export class Daemon extends EventEmitter {
2242
3071
  const pane = await this.tmux.capturePane();
2243
3072
  // Output received while capture-pane was in flight makes this snapshot
2244
3073
  // stale. Its output handler has already armed a new debounce.
2245
- if ((expectedOutputAt > 0 && this.instanceStateLastOutputAt > expectedOutputAt)
2246
- || (this.instanceStateLastOutputAt > 0 && this.instanceStateLastOutputAt >= captureStartedAt))
3074
+ const outputMovedDuringCapture = (expectedOutputAt > 0 && this.instanceStateLastOutputAt > expectedOutputAt)
3075
+ || (this.instanceStateLastOutputAt > 0 && this.instanceStateLastOutputAt >= captureStartedAt);
3076
+ if (outputMovedDuringCapture) {
3077
+ if (reason === "output_probe") {
3078
+ // A second burst overtook the probe. Do not bless its stale capture as
3079
+ // idle: continuous output is the safer working signal, while the next
3080
+ // quiet probe/debounce can still prove that it was only repainting.
3081
+ this.applyInstanceStateSnapshot(this.instanceStateMachine.recordOutput(this.instanceStateLastOutputAt));
3082
+ this.scheduleInstanceStateIdleCapture();
3083
+ this.scheduleInstanceStateStuckDeadline(this.instanceStateLastOutputAt);
3084
+ }
2247
3085
  return;
3086
+ }
2248
3087
  const observedChangeAt = expectedOutputAt || captureStartedAt;
2249
3088
  // One observation per capture. This used to call observe() twice with the
2250
3089
  // same pane: the second call always saw an unchanged pane, so the
@@ -2253,6 +3092,7 @@ export class Daemon extends EventEmitter {
2253
3092
  // mismatched spinner frame was enough to report idle mid-turn.
2254
3093
  const settled = this.instanceStateLastOutputAt === 0
2255
3094
  || captureStartedAt - this.instanceStateLastOutputAt >= this.instanceStateIdleDebounceMs;
3095
+ const paneActivity = this.backend?.getPaneActivity?.(pane) ?? null;
2256
3096
  // The two times are genuinely different and both matter: the content
2257
3097
  // changed when tmux reported output (observedChangeAt), but idle/stuck are
2258
3098
  // decisions about *now*. The old double-observe expressed that by calling
@@ -2260,6 +3100,7 @@ export class Daemon extends EventEmitter {
2260
3100
  const snapshot = this.instanceStateMachine.observe(pane, Date.now(), {
2261
3101
  settled,
2262
3102
  changeAt: observedChangeAt,
3103
+ forceBusy: paneActivity !== null,
2263
3104
  });
2264
3105
  this.applyInstanceStateSnapshot(snapshot, pane);
2265
3106
  // Backends without a transcript feed can still say what they are doing, if
@@ -2267,7 +3108,7 @@ export class Daemon extends EventEmitter {
2267
3108
  // extra tmux call. Cadence is the capture cadence (idle debounce + the 60s
2268
3109
  // safety sweep), which matches the progress ticker's own 60s interval.
2269
3110
  if (this.backend?.getPaneActivity) {
2270
- this.publishActivity(snapshot.state === "idle" ? null : this.backend.getPaneActivity(pane));
3111
+ this.publishActivity(snapshot.state === "idle" ? null : paneActivity);
2271
3112
  }
2272
3113
  if (snapshot.state === "idle") {
2273
3114
  this.clearInstanceStateStuckTimer();
@@ -2294,6 +3135,14 @@ export class Daemon extends EventEmitter {
2294
3135
  if (!windowId || event.windowId !== windowId || !this.instanceStateMachine)
2295
3136
  return;
2296
3137
  this.instanceStateLastOutputAt = event.at;
3138
+ if (this.backend?.hasPeriodicPaneRedraw?.() === true && this.instanceState === "idle") {
3139
+ // Do not emit idle→working solely because agy repainted an identical
3140
+ // footer. The short capture probe below still observes genuine pane
3141
+ // changes promptly and moves the state to working.
3142
+ this.scheduleInstanceStateOutputProbe();
3143
+ this.scheduleInstanceStateIdleCapture();
3144
+ return;
3145
+ }
2297
3146
  const snapshot = this.instanceStateMachine.recordOutput(event.at);
2298
3147
  this.applyInstanceStateSnapshot(snapshot);
2299
3148
  this.scheduleInstanceStateIdleCapture();
@@ -2344,6 +3193,7 @@ export class Daemon extends EventEmitter {
2344
3193
  stopInstanceStateMonitor() {
2345
3194
  this.instanceStateMonitorActive = false;
2346
3195
  this.clearInstanceStateIdleTimer();
3196
+ this.clearInstanceStateOutputProbeTimer();
2347
3197
  this.clearInstanceStateStuckTimer();
2348
3198
  if (this.instanceStateMonitorTimer)
2349
3199
  clearInterval(this.instanceStateMonitorTimer);
@@ -2378,6 +3228,15 @@ export class Daemon extends EventEmitter {
2378
3228
  this.logger.debug(diagnostic, "Suppressing stuck notification without pending work");
2379
3229
  return;
2380
3230
  }
3231
+ // An auth-broken instance IS stuck, and stays stuck: it cannot finish the
3232
+ // turn it is holding, so pendingWork never clears and this fired on every
3233
+ // scan forever. The notification is also useless — it offers Restart/Wait
3234
+ // and asks General for help, but only the user re-logging in fixes this, and
3235
+ // they have already been told exactly that by the auth alert.
3236
+ if (this.authFailureUnresolved) {
3237
+ this.logger.debug(diagnostic, "Suppressing stuck notification — unresolved auth failure (re-login required)");
3238
+ return;
3239
+ }
2381
3240
  this.logger.warn(diagnostic, "Instance pane stuck with pending work");
2382
3241
  this.hangDetector?.emit("hang", { unchangedForMs: snapshot.unchangedForMs });
2383
3242
  }
@@ -2397,6 +3256,7 @@ export class Daemon extends EventEmitter {
2397
3256
  // fire into a stopped daemon).
2398
3257
  this.clearMcpRestartRequest();
2399
3258
  this.interactivePromptDetector.reset();
3259
+ this.blockingProcessDetector.reset();
2400
3260
  this.stopInstanceStateMonitor();
2401
3261
  this.transcriptMonitor?.stop();
2402
3262
  this.guardian?.stop();
@@ -2422,9 +3282,9 @@ export class Daemon extends EventEmitter {
2422
3282
  * Tell the fleet manager what this instance is doing right now, for the live
2423
3283
  * progress line on the cancel button.
2424
3284
  *
2425
- * Purely cosmetic — nothing decides anything from it, so it is fine that only
2426
- * backends with a transcript feed report at all (claude-code today). For the
2427
- * rest the progress line keeps showing just elapsed time.
3285
+ * Primarily cosmetic. The foreground-process detector also uses a current
3286
+ * shell activity as one of several positive signals, but never makes a state
3287
+ * decision from an arbitrary activity label alone.
2428
3288
  *
2429
3289
  * Repeats are dropped: the ticker only edits the channel message when the text
2430
3290
  * changes, and a stream of identical broadcasts would defeat that.
@@ -2440,10 +3300,84 @@ export class Daemon extends EventEmitter {
2440
3300
  activity: next,
2441
3301
  });
2442
3302
  }
2443
- /** Effective tool_progress level, hardened against junk config values. */
3303
+ /** Effective tool_progress level, hardened against junk config values.
3304
+ * Off unless explicitly enabled: an update must never make bubbles start
3305
+ * listing tool activity the operator did not ask to broadcast. */
2444
3306
  toolProgressLevel() {
2445
3307
  const raw = this.config.tool_progress;
2446
- return raw === "off" || raw === "verbose" ? raw : "standard";
3308
+ return raw === "standard" || raw === "verbose" ? raw : "off";
3309
+ }
3310
+ /**
3311
+ * Apply a tool-progress setting without restarting the CLI session.
3312
+ *
3313
+ * Drop the current accumulator at the boundary: verbose entries may contain
3314
+ * command previews which must not survive a downgrade to standard/off, and a
3315
+ * newly enabled level should start with a clean, internally consistent list.
3316
+ */
3317
+ updateToolProgress(level) {
3318
+ const previous = this.toolProgressLevel();
3319
+ this.config.tool_progress = level;
3320
+ if (this.toolProgressLevel() !== previous)
3321
+ this.resetToolProgress();
3322
+ }
3323
+ /** Snapshot the configuration actually owned by this live daemon. */
3324
+ getConfigSnapshot() {
3325
+ return structuredClone(this.config);
3326
+ }
3327
+ /**
3328
+ * Apply the whitelisted hot configuration carried by FleetManager IPC.
3329
+ * null removes an optional value; invalid/unlisted fields are ignored rather
3330
+ * than allowing another client on the per-instance socket to mutate cold
3331
+ * process/session settings.
3332
+ */
3333
+ applyConfigUpdate(value) {
3334
+ if (!value || typeof value !== "object" || Array.isArray(value))
3335
+ return;
3336
+ const update = value;
3337
+ if (update.tool_progress === null || ["off", "standard", "verbose"].includes(String(update.tool_progress))) {
3338
+ this.updateToolProgress(update.tool_progress === null ? undefined : update.tool_progress);
3339
+ }
3340
+ if (typeof update.mcp_proxy_reply === "boolean")
3341
+ this.config.mcp_proxy_reply = update.mcp_proxy_reply;
3342
+ else if (update.mcp_proxy_reply === null)
3343
+ delete this.config.mcp_proxy_reply;
3344
+ if (typeof update.warm_cap === "number" && Number.isInteger(update.warm_cap) && update.warm_cap >= 0)
3345
+ this.config.warm_cap = update.warm_cap;
3346
+ else if (update.warm_cap === null)
3347
+ delete this.config.warm_cap;
3348
+ const nextAutoPause = typeof update.auto_pause_after === "number"
3349
+ && Number.isFinite(update.auto_pause_after)
3350
+ && update.auto_pause_after >= 0
3351
+ ? update.auto_pause_after
3352
+ : update.auto_pause_after === null ? 0 : undefined;
3353
+ if (nextAutoPause !== undefined) {
3354
+ const previousAutoPause = typeof this.config.auto_pause_after === "number" ? this.config.auto_pause_after : 0;
3355
+ if (update.auto_pause_after === null)
3356
+ delete this.config.auto_pause_after;
3357
+ else
3358
+ this.config.auto_pause_after = nextAutoPause;
3359
+ if (nextAutoPause !== previousAutoPause) {
3360
+ const isGeneral = this.config.general_topic === true || this.name === "general";
3361
+ this.autoPauseController.reconfigure((isGeneral ? 0 : nextAutoPause) * 60_000);
3362
+ }
3363
+ }
3364
+ for (const key of ["display_name", "description"]) {
3365
+ if (typeof update[key] === "string")
3366
+ this.config[key] = update[key];
3367
+ else if (update[key] === null)
3368
+ delete this.config[key];
3369
+ }
3370
+ if (Array.isArray(update.tags) && update.tags.every(tag => typeof tag === "string"))
3371
+ this.config.tags = [...update.tags];
3372
+ else if (update.tags === null)
3373
+ delete this.config.tags;
3374
+ if (["trace", "debug", "info", "warn", "error"].includes(String(update.log_level))) {
3375
+ const level = update.log_level;
3376
+ if (this.config.log_level !== level) {
3377
+ this.config.log_level = level;
3378
+ this.logger.level = level;
3379
+ }
3380
+ }
2447
3381
  }
2448
3382
  /**
2449
3383
  * Accumulate one semantic progress line for the channel bubble and schedule
@@ -2548,16 +3482,7 @@ export class Daemon extends EventEmitter {
2548
3482
  const fromInstance = meta.from_instance;
2549
3483
  let formatted;
2550
3484
  if (fromInstance) {
2551
- // #77: show the sender's display name for readability, keeping the machine
2552
- // instance name in parens so the recipient's send_to_instance target is valid.
2553
- const fromLabel = meta.from_display ? `${meta.from_display} (${fromInstance})` : fromInstance;
2554
- formatted = `[from:${fromLabel}] ${content}`;
2555
- formatted += renderHandoffMetadata(meta);
2556
- // A delegated task that requires a reply must not read like a chatty FYI —
2557
- // the "you may stay silent" line is for the latter only.
2558
- formatted += meta.requires_reply === "true"
2559
- ? "\n(A reply IS required: use report_result with the correlation_id above — or send_to_instance. Not direct text.)"
2560
- : "\n(If you need to reply, use send_to_instance tool, NOT direct text. If there is nothing to add, you may stay silent.)";
3485
+ formatted = formatCrossInstanceInboundMessage(content, meta);
2561
3486
  }
2562
3487
  else {
2563
3488
  const via = meta.source ? ` via ${meta.source}` : "";
@@ -2577,11 +3502,24 @@ export class Daemon extends EventEmitter {
2577
3502
  }
2578
3503
  return formatted;
2579
3504
  }
3505
+ /** Tell the fleet when an already-accepted cross-instance pane write failed. */
3506
+ reportCrossInstanceDeliveryFailure(meta, error) {
3507
+ if (!meta.from_instance)
3508
+ return;
3509
+ this.ipcServer?.broadcast({
3510
+ type: "cross_instance_delivery_failed",
3511
+ senderSession: meta.from_instance,
3512
+ targetInstance: this.name,
3513
+ correlationId: meta.correlation_id ?? "unknown",
3514
+ error: error ?? this.tmux?.getLastPasteError?.() ?? "target pane rejected the delivery",
3515
+ });
3516
+ }
2580
3517
  /**
2581
3518
  * /steer: interject into the CURRENT turn instead of queueing for idle.
2582
3519
  *
2583
3520
  * Differences from pushChannelMessage, and nothing else:
2584
- * - serialized on steerLock, not pasteLock — it must overtake, not queue
3521
+ * - serialized on steerLock, not pasteLock — it may overtake current work,
3522
+ * but never an ordinary delivery already queued ahead of it
2585
3523
  * - deliverMessage runs with { steer: true }, which takes the busy branch
2586
3524
  * that pastes immediately (the codex native-queue transaction) instead of
2587
3525
  * waiting for idle. Verification and silent-loss fallback are the ones
@@ -2592,11 +3530,21 @@ export class Daemon extends EventEmitter {
2592
3530
  * The steered text goes through formatInboundMessage so the agent sees a
2593
3531
  * normal inbound message, with a steering notice prepended for context.
2594
3532
  */
2595
- steerMessage(content, meta) {
3533
+ steerMessage(content, meta, deliveryEpoch = this.deliveryEpoch) {
3534
+ if (!this.isDeliveryEpochCurrent(deliveryEpoch))
3535
+ return;
3536
+ // The fleet facade prevents a steer from overtaking an item still in its
3537
+ // idle queue. Close the second race window here: an earlier item may already
3538
+ // have reached this daemon's paste queue by the time the steer IPC arrives.
3539
+ if (this.pasteQueueDepth > 0) {
3540
+ this.logger.info("Steer queued behind an earlier pane delivery to preserve message order");
3541
+ this.pushChannelMessage(content, meta, undefined, deliveryEpoch);
3542
+ return;
3543
+ }
2596
3544
  this.updateLastChat(meta.chat_id, meta.thread_id, meta.adapter_id);
2597
3545
  this.pendingWork.recordInbound();
2598
3546
  this.recordRecentUserMessage(content, meta);
2599
- const formatted = "[STEERING — mid-task course correction from the user. Fold this into the CURRENT work.]\n"
3547
+ const formatted = "[STEERING — mid-task course correction. Fold this into the CURRENT work if one is active.]\n"
2600
3548
  + this.formatInboundMessage(content, meta);
2601
3549
  const chatId = meta.chat_id;
2602
3550
  const messageId = meta.message_id;
@@ -2604,12 +3552,54 @@ export class Daemon extends EventEmitter {
2604
3552
  ? { chatId: meta.thread_id || chatId, messageId }
2605
3553
  : undefined;
2606
3554
  this.steerLock = this.steerLock.then(async () => {
3555
+ if (!this.isDeliveryEpochCurrent(deliveryEpoch))
3556
+ return;
2607
3557
  await this.wake();
2608
- if (await this.deliverMessage(formatted, status, { steer: true })) {
3558
+ if (!this.isDeliveryEpochCurrent(deliveryEpoch))
3559
+ return;
3560
+ if (await this.deliverMessage(formatted, status, { steer: true, deliveryEpoch })) {
2609
3561
  this.markTurnStarted(meta, formatted);
2610
3562
  }
3563
+ else if (this.isDeliveryEpochCurrent(deliveryEpoch)) {
3564
+ this.reportCrossInstanceDeliveryFailure(meta);
3565
+ }
2611
3566
  }).catch(err => {
2612
3567
  this.logger.warn({ err: err.message }, "steer delivery error");
3568
+ if (this.isDeliveryEpochCurrent(deliveryEpoch)) {
3569
+ this.reportCrossInstanceDeliveryFailure(meta, err.message);
3570
+ }
3571
+ });
3572
+ }
3573
+ /**
3574
+ * /btw: submit a separately-labelled side question immediately while asking
3575
+ * the agent not to change the active task. Claude's native `/btw` opens a
3576
+ * branchAndResume panel whose answer never reaches the channel or transcript,
3577
+ * so channel delivery intentionally uses a normal wrapped inbound instead.
3578
+ */
3579
+ btwMessage(content, meta, deliveryEpoch = this.deliveryEpoch) {
3580
+ if (!this.isDeliveryEpochCurrent(deliveryEpoch))
3581
+ return;
3582
+ this.updateLastChat(meta.chat_id, meta.thread_id, meta.adapter_id);
3583
+ this.pendingWork.recordInbound();
3584
+ this.recordRecentUserMessage(content, meta);
3585
+ const formatted = "[BTW — side question from the user. Answer it separately without changing or interrupting the CURRENT work.]\n"
3586
+ + this.formatInboundMessage(content, meta);
3587
+ const chatId = meta.chat_id;
3588
+ const messageId = meta.message_id;
3589
+ const status = (chatId && messageId)
3590
+ ? { chatId: meta.thread_id || chatId, messageId }
3591
+ : undefined;
3592
+ this.steerLock = this.steerLock.then(async () => {
3593
+ if (!this.isDeliveryEpochCurrent(deliveryEpoch))
3594
+ return;
3595
+ await this.wake();
3596
+ if (!this.isDeliveryEpochCurrent(deliveryEpoch))
3597
+ return;
3598
+ if (await this.deliverMessage(formatted, status, { steer: true, deliveryEpoch })) {
3599
+ this.markTurnStarted(meta, formatted);
3600
+ }
3601
+ }).catch(err => {
3602
+ this.logger.warn({ err: err.message }, "btw delivery error");
2613
3603
  });
2614
3604
  }
2615
3605
  /**
@@ -2617,7 +3607,11 @@ export class Daemon extends EventEmitter {
2617
3607
  * If targetSession is provided, only send to the matching socket.
2618
3608
  * Otherwise send to the instance's own session (this.name).
2619
3609
  */
2620
- pushChannelMessage(content, meta, _targetSession) {
3610
+ pushChannelMessage(content, meta, _targetSession, deliveryEpoch = this.deliveryEpoch) {
3611
+ if (!this.isDeliveryEpochCurrent(deliveryEpoch)) {
3612
+ this.logger.info("Pending channel delivery dropped by user cancel");
3613
+ return;
3614
+ }
2621
3615
  if (!this.tmux) {
2622
3616
  this.logger.warn("Cannot push channel message: tmux not running");
2623
3617
  return;
@@ -2651,7 +3645,9 @@ export class Daemon extends EventEmitter {
2651
3645
  const rawText = content.slice(5);
2652
3646
  this.logger.info({ user }, "Raw paste from topic mode user");
2653
3647
  this.pasteLock = this.pasteLock.then(async () => {
2654
- if (await this.deliverMessage(rawText))
3648
+ if (!this.isDeliveryEpochCurrent(deliveryEpoch))
3649
+ return;
3650
+ if (await this.deliverMessage(rawText, undefined, { deliveryEpoch }))
2655
3651
  this.markTurnStarted(meta, rawText);
2656
3652
  }).catch(err => {
2657
3653
  this.logger.warn({ err: err.message }, "pasteLock raw delivery error");
@@ -2661,8 +3657,7 @@ export class Daemon extends EventEmitter {
2661
3657
  const formatted = this.formatInboundMessage(content, meta);
2662
3658
  // Serialize deliveries: each message waits for the previous to complete,
2663
3659
  // and each waits for the CLI to be idle before pasting. Messages are never
2664
- // dropped for age — a long-busy CLI just queues them until it frees up
2665
- // (the user can press Cancel to interrupt and let the queue drain sooner).
3660
+ // dropped for age — only an explicit user Cancel invalidates queued work.
2666
3661
  const chatId = meta.chat_id;
2667
3662
  const messageId = meta.message_id;
2668
3663
  const wasQueued = this.pasteQueueDepth > 0;
@@ -2675,12 +3670,20 @@ export class Daemon extends EventEmitter {
2675
3670
  }
2676
3671
  this.pasteLock = this.pasteLock.then(async () => {
2677
3672
  try {
3673
+ if (!this.isDeliveryEpochCurrent(deliveryEpoch)) {
3674
+ this.logger.info("Pending channel delivery dropped by user cancel");
3675
+ return;
3676
+ }
2678
3677
  if (this.config.pre_task_command) {
2679
- await this.deliverMessage(this.config.pre_task_command);
3678
+ await this.deliverMessage(this.config.pre_task_command, undefined, { deliveryEpoch });
3679
+ if (!this.isDeliveryEpochCurrent(deliveryEpoch))
3680
+ return;
2680
3681
  }
2681
3682
  if (this.pendingInstructionsNotice) {
2682
3683
  this.pendingInstructionsNotice = false;
2683
- await this.deliverMessage(buildInstructionReloadNotice(this.backend?.binaryName ?? "unknown", this.name, this.instanceDir));
3684
+ await this.deliverMessage(buildInstructionReloadNotice(this.backend?.binaryName ?? "unknown", this.name, this.instanceDir), undefined, { deliveryEpoch });
3685
+ if (!this.isDeliveryEpochCurrent(deliveryEpoch))
3686
+ return;
2684
3687
  }
2685
3688
  const status = (chatId && messageId)
2686
3689
  ? { chatId: meta.thread_id || chatId, messageId }
@@ -2688,14 +3691,21 @@ export class Daemon extends EventEmitter {
2688
3691
  // A fresh delivery begins a fresh turn — its bubble must not inherit
2689
3692
  // the previous turn's tool list.
2690
3693
  this.resetToolProgress();
2691
- if (await this.deliverMessage(formatted, status))
3694
+ if (await this.deliverMessage(formatted, status, { deliveryEpoch })) {
2692
3695
  this.markTurnStarted(meta, formatted);
3696
+ }
3697
+ else if (meta.from_instance && this.isDeliveryEpochCurrent(deliveryEpoch)) {
3698
+ this.reportCrossInstanceDeliveryFailure(meta);
3699
+ }
2693
3700
  }
2694
3701
  finally {
2695
3702
  this.pasteQueueDepth--;
2696
3703
  }
2697
3704
  }).catch(err => {
2698
3705
  this.logger.warn({ err: err.message }, "pasteLock delivery error — chain continues");
3706
+ if (this.isDeliveryEpochCurrent(deliveryEpoch)) {
3707
+ this.reportCrossInstanceDeliveryFailure(meta, err.message);
3708
+ }
2699
3709
  });
2700
3710
  this.logger.debug({ user: meta.user, text: content.slice(0, 100) }, "Queued channel message for delivery");
2701
3711
  }
@@ -2716,24 +3726,53 @@ export class Daemon extends EventEmitter {
2716
3726
  * serial in both cases so separate PTY writes can never overlap.
2717
3727
  */
2718
3728
  async deliverMessage(formatted, status, opts) {
3729
+ const cancelled = () => opts?.deliveryEpoch !== undefined
3730
+ && !this.isDeliveryEpochCurrent(opts.deliveryEpoch);
3731
+ if (cancelled() || this.stormWindow?.isStopped())
3732
+ return false;
2719
3733
  // Sanitize unclosed code fences — they cause CLI to wait for closure on Enter
2720
3734
  const fenceCount = (formatted.match(/```/g) || []).length;
2721
3735
  if (fenceCount % 2 !== 0) {
2722
3736
  // Odd number of fences = unclosed. Remove all code fences from the message.
2723
3737
  formatted = formatted.replace(/```/g, "");
2724
3738
  }
3739
+ // Direct paths such as /steer, /btw, /raw and schedules do not pass the
3740
+ // FleetManager idle gate. Hold them at the final common pane-write path so
3741
+ // a tmux storm cannot force any text into a half-booted CLI.
3742
+ if (this.stormWindow?.isDeliveryHeld(this.name)) {
3743
+ this.logger.warn("Pane delivery held during tmux server recovery");
3744
+ await this.stormWindow.waitForDeliveryAllowed(this.name);
3745
+ if (cancelled() || this.stormWindow.isStopped())
3746
+ return false;
3747
+ }
2725
3748
  // Before anything reads the window id: a spawn in progress is about to change it.
2726
3749
  await this.waitForSpawnToSettle();
3750
+ if (cancelled())
3751
+ return false;
3752
+ if (this.refuseFatalStartupDelivery(status))
3753
+ return false;
2727
3754
  let windowId = this.getWindowId();
2728
3755
  const supportsQueuedInput = this.backend?.supportsQueuedInput?.() === true;
2729
3756
  let handingOffToNativeQueue = false;
2730
3757
  // If the CLI is busy, either hand the complete submission to its native input
2731
3758
  // queue or wait for idle. Native queue support is an explicit backend capability:
2732
3759
  // normal Enter input has steering/interrupt semantics in several other CLIs.
2733
- if (windowId && this.controlClient && !this.controlClient.isIdle(windowId)) {
3760
+ let readiness = "ready";
3761
+ const gateWindowId = windowId;
3762
+ if (gateWindowId && this.controlClient)
3763
+ readiness = await this.paneReadinessForDelivery(gateWindowId);
3764
+ if (readiness !== "ready") {
2734
3765
  if (status)
2735
3766
  this.emit("message_queued", status);
2736
- if (supportsQueuedInput || opts?.steer) {
3767
+ // Only a pane that is busy AND verifiably free of a blocking dialog may
3768
+ // be handed to the native input queue or steered — those paths paste and
3769
+ // press Enter at once. A dialog painted a moment ago is still "busy" to
3770
+ // the silence gate, so the probe is mandatory here; "dialog"/"unknown"
3771
+ // fall through to the wait.
3772
+ const canHandOff = (supportsQueuedInput || opts?.steer)
3773
+ && readiness === "busy"
3774
+ && (await this.probeBlockingDialog()).state === "clear";
3775
+ if (canHandOff) {
2737
3776
  // Native queue (codex), or an explicit /steer: hand the complete
2738
3777
  // paste+Enter transaction to the busy CLI now. For steer this is the
2739
3778
  // point — the user asked to interject, and the transaction's
@@ -2744,7 +3783,9 @@ export class Daemon extends EventEmitter {
2744
3783
  }
2745
3784
  else {
2746
3785
  this.logger.debug("CLI busy — queuing message until idle");
2747
- const becameIdle = await this.controlClient.waitUntilIdle(windowId);
3786
+ const becameIdle = await this.waitForPaneReadyForDelivery(gateWindowId);
3787
+ if (cancelled())
3788
+ return false;
2748
3789
  if (!becameIdle) {
2749
3790
  // The pane never freed up. Report the failure instead of pasting into a
2750
3791
  // wedged CLI (where the text would sit unsubmitted and the next message
@@ -2756,11 +3797,459 @@ export class Daemon extends EventEmitter {
2756
3797
  }
2757
3798
  }
2758
3799
  }
3800
+ // F3 (#kiro-enter-drop): an earlier delivery's Enter may have been dropped,
3801
+ // leaving that message in the input row. Pasting on top would submit both as
3802
+ // one. Submit the stranded text first (a bare Enter — a no-op on an empty
3803
+ // prompt, verified live), then wait for the prompt to come back before our
3804
+ // own paste. The wait happens OUTSIDE the pane lock like every other wait.
3805
+ if (windowId && this.controlClient && !handingOffToNativeQueue) {
3806
+ // Only a positively verified clear input row ("idle") lets the paste
3807
+ // proceed. After "submitted" (the stranded text's turn is running) or
3808
+ // "busy" (the pane changed under us, or could not be read) the prompt
3809
+ // must come back AND the check runs again — the stranded text may still
3810
+ // be there. "failed" (the Enter could not be sent) fails the delivery:
3811
+ // pasting now would submit both messages as one.
3812
+ for (let round = 0;; round++) {
3813
+ // Budget check BEFORE the next mutating call: each round may send one
3814
+ // recovery Enter, so exactly STRANDED_INPUT_MAX_ROUNDS of them go out.
3815
+ if (round >= STRANDED_INPUT_MAX_ROUNDS) {
3816
+ this.logger.error({ round }, "Input row still not clear after the stranded-text recovery budget — reporting delivery failure");
3817
+ if (status)
3818
+ this.emit("message_failed", status); // ❌
3819
+ return false;
3820
+ }
3821
+ const stranded = await this.submitStrandedInputIfAny(windowId);
3822
+ if (cancelled())
3823
+ return false;
3824
+ if (stranded === "idle")
3825
+ break;
3826
+ if (stranded === "failed") {
3827
+ this.logger.error({ round }, "Could not submit the stranded text — reporting delivery failure");
3828
+ if (status)
3829
+ this.emit("message_failed", status); // ❌
3830
+ return false;
3831
+ }
3832
+ const readyAgain = await this.waitForPaneReadyForDelivery(windowId);
3833
+ if (cancelled())
3834
+ return false;
3835
+ if (!readyAgain) {
3836
+ this.logger.error("Pane never returned to its prompt after submitting stranded input — reporting delivery failure");
3837
+ if (status)
3838
+ this.emit("message_failed", status); // ❌
3839
+ return false;
3840
+ }
3841
+ }
3842
+ }
2759
3843
  // Everything above is *waiting*; everything below *writes*. Only the write is
2760
3844
  // held under the pane lock — holding it across the idle wait (up to 30 min)
2761
3845
  // would starve the runtime-dialog dismisser, which is often the very thing
2762
3846
  // that would let the pane go idle again.
2763
- return this.paneWriteLock.run(() => this.writeMessageToPane(formatted, windowId, handingOffToNativeQueue, status));
3847
+ for (let round = 0;; round++) {
3848
+ const outcome = await this.paneWriteLock.run(async () => {
3849
+ if (cancelled())
3850
+ return false;
3851
+ if (this.refuseFatalStartupDelivery(status))
3852
+ return false;
3853
+ // TOCTOU: the probes above ran outside this lock, and the CLI repaints
3854
+ // whenever it likes — a resume prompt can be painted between "clear" and
3855
+ // this critical section. Re-probe here, right before the write; a
3856
+ // dialog or an unreadable pane means "not now", never "paste anyway".
3857
+ if (this.deliveryBlockingDialogs().length > 0) {
3858
+ const probe = await this.probeBlockingDialog();
3859
+ if (probe.state !== "clear")
3860
+ return "dialog";
3861
+ }
3862
+ return this.writeMessageToPane(formatted, windowId, handingOffToNativeQueue, status);
3863
+ });
3864
+ if (outcome !== "dialog")
3865
+ return outcome;
3866
+ // Wait OUTSIDE the lock (holding it would starve the runtime dismisser),
3867
+ // then try the write again — bounded, so a dialog nobody answers ends in
3868
+ // an honest failure rather than a message that never lands.
3869
+ if (round + 1 >= LATE_DIALOG_WRITE_ROUNDS) {
3870
+ this.logger.error({ rounds: round + 1 }, "A dialog kept appearing before the pane write — reporting delivery failure");
3871
+ if (status)
3872
+ this.emit("message_failed", status); // ❌
3873
+ return false;
3874
+ }
3875
+ this.logger.info("Dialog appeared before the pane write — waiting for it to clear");
3876
+ const clear = gateWindowId ? await this.waitForPaneReadyForDelivery(gateWindowId) : false;
3877
+ if (cancelled())
3878
+ return false;
3879
+ if (!clear) {
3880
+ if (status)
3881
+ this.emit("message_failed", status); // ❌
3882
+ return false;
3883
+ }
3884
+ }
3885
+ }
3886
+ /**
3887
+ * The runtime dialogs that hold delivery while on screen (`blocksDelivery`
3888
+ * or `holdOnly`). One canonical list and order, shared by the delivery gate
3889
+ * and the runtime scanner, so both observers agree on which dialog they see.
3890
+ * Startup-only patterns are deliberately excluded: several of them are loose
3891
+ * on purpose ("I trust", "Yes, continue") and would match ordinary prose.
3892
+ */
3893
+ deliveryBlockingDialogs() {
3894
+ return (this.backend?.getRuntimeDialogs?.() ?? []).filter(d => d.blocksDelivery || d.holdOnly);
3895
+ }
3896
+ static dialogKey(dialog) {
3897
+ return `${dialog.pattern.source}/${dialog.pattern.flags}`;
3898
+ }
3899
+ /** A dialog counts only when it is the CURRENT interactive region, if the backend can tell; else by pattern. */
3900
+ static dialogMatches(dialog, pane) {
3901
+ return dialog.isActive ? dialog.isActive(pane) : dialog.pattern.test(pane);
3902
+ }
3903
+ /**
3904
+ * Tri-state liveness. tmux's own isWindowAlive folds every query failure
3905
+ * into `false`, so a transient tmux hiccup would read as "the CLI died" —
3906
+ * and callers that clear the session on death must not act on that.
3907
+ * getPaneStatus returns null on a failed/ambiguous query and a definite
3908
+ * `{alive:false}` for a dead pane; only the latter is evidence.
3909
+ */
3910
+ async paneLiveness() {
3911
+ if (!this.tmux)
3912
+ return "unknown";
3913
+ const tmux = this.tmux;
3914
+ if (typeof tmux.getPaneStatus === "function") {
3915
+ try {
3916
+ const status = await tmux.getPaneStatus();
3917
+ if (status === null)
3918
+ return "unknown";
3919
+ return status.alive ? "alive" : "dead";
3920
+ }
3921
+ catch {
3922
+ return "unknown";
3923
+ }
3924
+ }
3925
+ try {
3926
+ return (await tmux.isWindowAlive()) ? "alive" : "dead";
3927
+ }
3928
+ catch {
3929
+ return "unknown";
3930
+ }
3931
+ }
3932
+ /**
3933
+ * Capture the pane and classify it. "unknown" (capture failed) is a distinct
3934
+ * outcome on purpose: the resume dialog is silent, so silence cannot vouch
3935
+ * for the pane, and treating an unreadable pane as clear is exactly the
3936
+ * fail-open that answers a dialog with its default. Callers retry unknown
3937
+ * within a bound and then fail loudly.
3938
+ */
3939
+ async probeBlockingDialog() {
3940
+ if (!this.tmux || !this.backend)
3941
+ return { state: "clear" };
3942
+ const dialogs = this.deliveryBlockingDialogs();
3943
+ if (dialogs.length === 0)
3944
+ return { state: "clear" };
3945
+ let pane;
3946
+ try {
3947
+ pane = await this.tmux.capturePane();
3948
+ }
3949
+ catch (err) {
3950
+ this.logger.debug({ err }, "capture-pane failed during the dialog probe — pane state unknown");
3951
+ return { state: "unknown" };
3952
+ }
3953
+ for (const dialog of dialogs) {
3954
+ if (Daemon.dialogMatches(dialog, pane)) {
3955
+ this.trackDialogParked(dialog);
3956
+ return { state: "dialog", dialog };
3957
+ }
3958
+ }
3959
+ this.trackDialogParked(null);
3960
+ return { state: "clear" };
3961
+ }
3962
+ /**
3963
+ * Remember how long a blocking dialog has been on screen. Once it has
3964
+ * outlived DIALOG_PARKED_NOTIFY_MS — the auto-dismiss did not take, or the
3965
+ * dialog is a hold-only variant — report it ONCE so a human answers it.
3966
+ * Never presses a key: the whole point is that we do not know which option
3967
+ * is safe. Identity is the pattern, so the delivery gate and the runtime
3968
+ * scanner (same canonical list) never reset each other's clock.
3969
+ */
3970
+ trackDialogParked(dialog) {
3971
+ if (!dialog) {
3972
+ if (this.dialogParkedSince !== 0)
3973
+ this.logger.info({ dialog: this.dialogParkedKey }, "Blocking dialog is gone from the pane");
3974
+ this.dialogParkedSince = 0;
3975
+ this.dialogParkedKey = null;
3976
+ this.dialogParkedReported = false;
3977
+ return;
3978
+ }
3979
+ const key = Daemon.dialogKey(dialog);
3980
+ if (this.dialogParkedSince === 0 || this.dialogParkedKey !== key) {
3981
+ this.dialogParkedSince = Date.now();
3982
+ this.dialogParkedKey = key;
3983
+ this.dialogParkedReported = false;
3984
+ return;
3985
+ }
3986
+ if (!this.dialogParkedReported && Date.now() - this.dialogParkedSince >= DIALOG_PARKED_NOTIFY_MS) {
3987
+ this.dialogParkedReported = true;
3988
+ this.logger.warn({ dialog: dialog.description, parkedForMs: Date.now() - this.dialogParkedSince }, "CLI dialog is still on screen — not auto-answering, reporting for a human");
3989
+ this.emit("dialog_parked", { name: this.name, description: dialog.description, holdOnly: dialog.holdOnly === true });
3990
+ }
3991
+ }
3992
+ /**
3993
+ * Bottom-anchored readiness for TUIs that drop Enter while busy (see
3994
+ * CliBackend.dropsEnterWhileBusy). Output silence is necessary but not
3995
+ * sufficient for them: Kiro paints nothing for the whole duration of a shell
3996
+ * or MCP tool call (and for each 10s backend retry), yet an Enter arriving
3997
+ * then is discarded. Reproduced deterministically: `sleep 9` tool, paste at
3998
+ * t+6s → text stranded in the prompt row, next message submitted both.
3999
+ * Other backends keep the silence-based gate byte-for-byte.
4000
+ */
4001
+ async isPaneReadyForDelivery(windowId) {
4002
+ return (await this.paneReadinessForDelivery(windowId)) === "ready";
4003
+ }
4004
+ /**
4005
+ * Why a pane is not deliverable matters. "busy" may be handed to a native
4006
+ * input queue or steered (after its own dialog probe); "dialog" and "unknown"
4007
+ * must never be — a paste + Enter into a dialog answers it with its default,
4008
+ * and an unreadable pane proves nothing. The silence gate comes first and
4009
+ * costs no capture, so a CLI that is simply working never pays for the probe.
4010
+ */
4011
+ async paneReadinessForDelivery(windowId) {
4012
+ if (!this.isPaneIdleForDelivery(windowId))
4013
+ return "busy";
4014
+ const probe = await this.probeBlockingDialog();
4015
+ if (probe.state !== "clear")
4016
+ return probe.state;
4017
+ if (this.backend?.dropsEnterWhileBusy?.() !== true)
4018
+ return "ready";
4019
+ const prompt = this.backend.getBottomReadyPattern?.();
4020
+ if (!prompt || !this.tmux)
4021
+ return "ready";
4022
+ try {
4023
+ const pane = await this.tmux.capturePane();
4024
+ if (this.backend.getBusyPattern?.()?.test(pane))
4025
+ return "busy";
4026
+ if (bottomRowIsReady(pane, prompt))
4027
+ return "ready";
4028
+ // FAIL CLOSED on any non-blank screen without the prompt at the bottom.
4029
+ // The obvious "unknown layout → fall back to silence" shortcut re-opened
4030
+ // the bug: a long tool output scrolls the turn's own `N% !>` row out of
4031
+ // the visible pane (capture-pane -p is the viewport only), the bottom row
4032
+ // is tool output, the pane is quiet — and a paste there strands the text
4033
+ // exactly as before. The prompt row is the only evidence Kiro accepts an
4034
+ // Enter; a modal or panel is handled by the dialog dismisser, and the
4035
+ // bounded wait reports a failure rather than pasting into a screen we do
4036
+ // not understand. That includes a completely BLANK capture: a clear /
4037
+ // redraw / alternate-screen transition paints nothing for a frame, tmux
4038
+ // still accepts a paste, and "nothing visible" is not evidence the TUI is
4039
+ // idle. The next 250ms poll sees the prompt if it is coming.
4040
+ return "busy";
4041
+ }
4042
+ catch (err) {
4043
+ // Same rule for a capture that fails: no positive proof, no paste. The
4044
+ // poll retries; a persistently unreadable pane ends in the bounded
4045
+ // timeout's ❌ rather than in a silently stranded message.
4046
+ this.logger.debug({ err }, "capture-pane failed during the readiness check — treating the pane as not ready");
4047
+ return "busy";
4048
+ }
4049
+ }
4050
+ /**
4051
+ * Like waitForPaneIdleForDelivery, plus the bottom-anchored prompt check for
4052
+ * Enter-dropping TUIs. Polls at a coarse cadence (each poll is a capture-pane)
4053
+ * only after the pane is already quiet, so the common fast-idle delivery still
4054
+ * costs a single capture.
4055
+ */
4056
+ async waitForPaneReadyForDelivery(windowId, timeoutMs = 30 * 60_000) {
4057
+ // Every backend polls: readiness now also means "no blocking dialog on
4058
+ // screen", and a dialog disappears without any output edge the silence
4059
+ // gate could see.
4060
+ const deadline = Date.now() + timeoutMs;
4061
+ const bottomGated = this.backend?.dropsEnterWhileBusy?.() === true;
4062
+ let unknownStreak = 0;
4063
+ for (;;) {
4064
+ const remaining = deadline - Date.now();
4065
+ if (remaining <= 0)
4066
+ return false;
4067
+ const idle = await this.waitForPaneIdleForDelivery(windowId, remaining);
4068
+ if (!idle)
4069
+ return false;
4070
+ // The idle wait just resolved, so re-reading the silence gate would only
4071
+ // add a stale-cache race; ask the pane the remaining questions directly.
4072
+ if (bottomGated) {
4073
+ if (await this.isPaneReadyForDelivery(windowId))
4074
+ return true;
4075
+ }
4076
+ else {
4077
+ const probe = await this.probeBlockingDialog();
4078
+ if (probe.state === "clear")
4079
+ return true;
4080
+ if (probe.state === "unknown") {
4081
+ if (++unknownStreak >= DIALOG_PROBE_UNKNOWN_MAX) {
4082
+ this.logger.error({ probes: unknownStreak }, "Pane stayed unreadable during the dialog probe — refusing to deliver blind");
4083
+ return false;
4084
+ }
4085
+ }
4086
+ else {
4087
+ unknownStreak = 0;
4088
+ }
4089
+ }
4090
+ await new Promise(r => setTimeout(r, BOTTOM_READY_POLL_MS));
4091
+ }
4092
+ }
4093
+ /**
4094
+ * F3: if an AgEnD message is sitting unsubmitted in the input row (a previous
4095
+ * delivery's Enter was dropped), submit it with one bare Enter under the pane
4096
+ * lock. Only acts on a positive signature — the `[user:`/`[from:` marker every
4097
+ * pasted message carries — never on Kiro's own placeholder hints.
4098
+ *
4099
+ * Outcomes: "idle" = positively verified nothing stranded (the caller may
4100
+ * paste); "submitted" = an Enter went out for the stranded text; "busy" = the
4101
+ * pane is not at its prompt or could not be read (the caller must wait for
4102
+ * the prompt and run this check AGAIN before pasting — never assume someone
4103
+ * else submitted it); "failed" = the Enter could not be sent (the delivery
4104
+ * must fail rather than paste on top of the stranded text).
4105
+ */
4106
+ async submitStrandedInputIfAny(windowId) {
4107
+ if (this.backend?.dropsEnterWhileBusy?.() !== true || !this.tmux)
4108
+ return "idle";
4109
+ const prompt = this.backend.getBottomReadyPattern?.();
4110
+ if (!prompt)
4111
+ return "idle";
4112
+ // Pre-check outside the lock so the common case (verified clear) never
4113
+ // contends with the dialog dismisser. Only a POSITIVE "clear" skips the
4114
+ // lock; an unreadable pane is re-examined under it, not waved through.
4115
+ const pre = await this.strandedInputState(prompt);
4116
+ if (pre === "clear")
4117
+ return "idle";
4118
+ if (pre === "busy")
4119
+ return "busy";
4120
+ if (pre === "stranded") {
4121
+ this.logger.warn({ windowId }, "Unsubmitted message found in the input row — submitting it before this delivery");
4122
+ }
4123
+ return this.paneWriteLock.run(async () => {
4124
+ // Re-check UNDER the lock right before the key goes out: between the
4125
+ // pre-check and acquiring the lock the CLI may have started generating
4126
+ // (the stranded text got submitted by someone else, or the dismisser
4127
+ // acted), or a fatal startup screen may have come up. A bare Enter is a
4128
+ // no-op at an empty prompt, but into a modal it is a confirmation.
4129
+ if (this.fatalStartupBlocked)
4130
+ return "busy";
4131
+ const state = await this.strandedInputState(prompt);
4132
+ if (state === "clear")
4133
+ return "idle";
4134
+ if (state !== "stranded")
4135
+ return "busy"; // busy, or unreadable: no evidence the prompt is free
4136
+ const sent = await this.sendDeliveryEnter("stranded-input-submit");
4137
+ return sent ? "submitted" : "failed";
4138
+ });
4139
+ }
4140
+ /**
4141
+ * "stranded": an AgEnD message sits in the input row; "clear": the prompt is
4142
+ * at the bottom with no stranded text; "busy": generating, or the prompt is
4143
+ * not at the bottom (incl. a blank frame); "unknown": the pane could not be
4144
+ * read. Only "clear" is positive evidence.
4145
+ */
4146
+ async strandedInputState(prompt) {
4147
+ let pane;
4148
+ try {
4149
+ pane = await this.tmux.capturePane();
4150
+ }
4151
+ catch {
4152
+ return "unknown";
4153
+ }
4154
+ if (this.backend?.getBusyPattern?.()?.test(pane))
4155
+ return "busy";
4156
+ if (strandedAgendMessageInInput(pane, prompt))
4157
+ return "stranded";
4158
+ return bottomRowIsReady(pane, prompt) ? "clear" : "busy";
4159
+ }
4160
+ /**
4161
+ * F2: for Enter-dropping TUIs, "some output after Enter" proves nothing — the
4162
+ * turn that made the pane look idle keeps producing output. Confirm the
4163
+ * SUBMISSION instead: the pasted text is gone from the input row ("absent"),
4164
+ * or the TUI's own busy marker is up. "unknown" = the pane could not be read;
4165
+ * it never counts as confirmed.
4166
+ */
4167
+ async pasteResidue(formatted) {
4168
+ const prompt = this.backend?.getBottomReadyPattern?.();
4169
+ if (!prompt || !this.tmux)
4170
+ return "absent";
4171
+ let pane;
4172
+ try {
4173
+ pane = await this.tmux.capturePane();
4174
+ }
4175
+ catch {
4176
+ return "unknown";
4177
+ }
4178
+ if (this.backend?.getBusyPattern?.()?.test(pane))
4179
+ return "absent"; // generating = submitted
4180
+ return pasteLeftInInput(pane, prompt, formatted) ? "present" : "absent";
4181
+ }
4182
+ /**
4183
+ * Submission = the legacy idle→busy output signal AND the pasted text has
4184
+ * verifiably left the input row. The first alone is what produced false ✅s:
4185
+ * the turn that made the pane look idle keeps emitting output after our
4186
+ * (dropped) Enter — and an unreadable pane is not a verified absence either.
4187
+ */
4188
+ async confirmSubmittedAfterEnter(windowId, enterAt, formatted) {
4189
+ const sawOutput = await this.confirmBusyAfterEnter(windowId, enterAt);
4190
+ const residue = await this.pasteResidue(formatted);
4191
+ if (residue === "unknown")
4192
+ this.logger.warn("Could not read the pane after Enter — not confirming the submission");
4193
+ return sawOutput && residue === "absent";
4194
+ }
4195
+ /**
4196
+ * While the CLI is parked on a fatal startup screen (see fatalStartupBlocked),
4197
+ * no pane write may happen: paste+Enter into the corrupt-config modal would
4198
+ * confirm "Exit and fix manually" and exit the CLI. The message is failed (❌)
4199
+ * rather than held — the accompanying config_error notification tells the user
4200
+ * what to fix, and holding would only build a queue that floods the pane the
4201
+ * moment the flag clears.
4202
+ */
4203
+ refuseFatalStartupDelivery(status) {
4204
+ if (!this.fatalStartupBlocked)
4205
+ return false;
4206
+ this.logger.error("Delivery refused — CLI is parked on a fatal startup screen");
4207
+ if (status)
4208
+ this.emit("message_failed", status); // ❌
4209
+ return true;
4210
+ }
4211
+ /**
4212
+ * A raw `%output` silence check is authoritative for ordinary TUIs. It is not
4213
+ * for a backend such as agy whose status-line hook periodically redraws an
4214
+ * identical screen. In that case trust an idle pane-state snapshot only when
4215
+ * it was captured after both the latest output and the latest control-client
4216
+ * observation reset.
4217
+ */
4218
+ isPaneIdleForDelivery(windowId) {
4219
+ if (!this.controlClient)
4220
+ return true;
4221
+ if (this.controlClient.isIdle(windowId))
4222
+ return true;
4223
+ if (this.backend?.hasPeriodicPaneRedraw?.() !== true || this.instanceState !== "idle")
4224
+ return false;
4225
+ const lastOutputAt = this.controlClient.getLastOutputAt(windowId);
4226
+ if (lastOutputAt === undefined || !this.instanceStateMachine)
4227
+ return false;
4228
+ const observedAt = this.instanceStateMachine.snapshot().observedAt;
4229
+ return observedAt >= lastOutputAt
4230
+ && observedAt >= this.controlClient.getObservationResetAt();
4231
+ }
4232
+ waitForPaneIdleForDelivery(windowId, timeoutMs = 30 * 60_000) {
4233
+ if (!this.controlClient)
4234
+ return Promise.resolve(true);
4235
+ if (this.backend?.hasPeriodicPaneRedraw?.() !== true) {
4236
+ return this.controlClient.waitUntilIdle(windowId, timeoutMs);
4237
+ }
4238
+ if (this.isPaneIdleForDelivery(windowId))
4239
+ return Promise.resolve(true);
4240
+ return new Promise(resolve => {
4241
+ const check = setInterval(() => {
4242
+ if (!this.isPaneIdleForDelivery(windowId))
4243
+ return;
4244
+ clearInterval(check);
4245
+ clearTimeout(timer);
4246
+ resolve(true);
4247
+ }, 50);
4248
+ const timer = setTimeout(() => {
4249
+ clearInterval(check);
4250
+ resolve(false);
4251
+ }, timeoutMs);
4252
+ });
2764
4253
  }
2765
4254
  /**
2766
4255
  * The write half of a delivery: paste → Enter → confirm, with retries.
@@ -2788,7 +4277,16 @@ export class Daemon extends EventEmitter {
2788
4277
  const pasteStartedAt = Date.now();
2789
4278
  const pasted = await this.tmux.pasteBuffer(formatted);
2790
4279
  if (!pasted) {
2791
- this.logger.warn({ attempt }, "pasteBuffer failed — recovering window and backing off");
4280
+ const tmuxError = this.tmux.getLastPasteError?.() ?? "unknown tmux paste failure";
4281
+ const recoverable = this.tmux.isLastPasteFailureRecoverable?.() ?? true;
4282
+ this.logger[recoverable ? "warn" : "error"]({ attempt, tmuxError, recoverable }, recoverable
4283
+ ? "pasteBuffer failed — recovering window and backing off"
4284
+ : "pasteBuffer failed — non-retryable tmux error");
4285
+ if (!recoverable) {
4286
+ if (status)
4287
+ this.emit("message_failed", status);
4288
+ return false;
4289
+ }
2792
4290
  windowId = (await this.recoverWindow()) ?? windowId;
2793
4291
  if (attempt < maxAttempts)
2794
4292
  await new Promise(r => setTimeout(r, 2000));
@@ -2870,7 +4368,9 @@ export class Daemon extends EventEmitter {
2870
4368
  }
2871
4369
  const repasted = await this.tmux.pasteBuffer(formatted);
2872
4370
  if (!repasted) {
2873
- this.logger.error("Idle-gated redelivery paste failed after native-queue silent loss");
4371
+ this.logger.error({
4372
+ tmuxError: this.tmux.getLastPasteError?.() ?? "unknown tmux paste failure",
4373
+ }, "Idle-gated redelivery paste failed after native-queue silent loss");
2874
4374
  if (status)
2875
4375
  this.emit("message_failed", status); // ❌
2876
4376
  return false;
@@ -2910,7 +4410,36 @@ export class Daemon extends EventEmitter {
2910
4410
  this.emit("message_failed", status); // ❌
2911
4411
  return false;
2912
4412
  }
2913
- if (windowId && this.controlClient) {
4413
+ if (windowId && this.controlClient && this.backend?.dropsEnterWhileBusy?.() === true) {
4414
+ // F2: output after Enter is necessary but not sufficient — the paste
4415
+ // must also have LEFT the input row.
4416
+ let submitted = await this.confirmSubmittedAfterEnter(windowId, enterAt, formatted);
4417
+ if (!submitted) {
4418
+ // The Enter landed while the TUI was still busy (it kept the text but
4419
+ // dropped the key). A retry is only useful once the prompt is back —
4420
+ // bounded, since we hold the pane lock here.
4421
+ this.logger.warn("Message not submitted after Enter — waiting for the prompt, then re-sending Enter once");
4422
+ const promptBack = await this.waitForPaneReadyForDelivery(windowId, STRANDED_RETRY_READY_WAIT_MS);
4423
+ const retryAt = Date.now();
4424
+ if (promptBack && !(await this.sendDeliveryEnter("stranded-text-retry"))) {
4425
+ if (status)
4426
+ this.emit("message_failed", status); // ❌
4427
+ return false;
4428
+ }
4429
+ submitted = promptBack && await this.confirmSubmittedAfterEnter(windowId, retryAt, formatted);
4430
+ }
4431
+ if (submitted) {
4432
+ if (status)
4433
+ this.emit("message_confirmed", status); // ✅
4434
+ }
4435
+ else {
4436
+ this.logger.error("Message pasted but never submitted (text still in the input row after Enter retry)");
4437
+ if (status)
4438
+ this.emit("message_failed", status); // ❌
4439
+ return false;
4440
+ }
4441
+ }
4442
+ else if (windowId && this.controlClient) {
2914
4443
  let becameBusy = await this.confirmBusyAfterEnter(windowId, enterAt);
2915
4444
  if (!becameBusy) {
2916
4445
  this.logger.warn("No idle→busy transition after Enter — re-sending Enter once");
@@ -3064,14 +4593,24 @@ export class Daemon extends EventEmitter {
3064
4593
  const args = (msg.args ?? {});
3065
4594
  const requestId = msg.requestId;
3066
4595
  this.logger.debug({ tool, requestId }, "Tool call from MCP server");
4596
+ // A tool call can only come from a live MCP server of ours — proof of life
4597
+ // that outranks whatever the pid slot says.
4598
+ if (this.socketSessionNames.get(socket) === this.name) {
4599
+ this.noteMcpProofOfLife("tool_call", this.socketPids.get(socket));
4600
+ }
3067
4601
  // For now, log and respond. Full adapter routing will be wired in fleet manager.
3068
4602
  const respond = (result, error) => {
3069
4603
  // A message that verifiably went out stands down the dead-MCP proxy reply
3070
4604
  // for this turn: the agent proved it can still speak for itself.
3071
4605
  if (!error && result != null && TURN_REPLY_TOOLS.has(tool)) {
3072
4606
  this.turnOutboundDelivered = true;
4607
+ if (tool === "reply")
4608
+ this.turnReplyDelivered = true;
4609
+ }
4610
+ const sent = this.ipcServer?.send(socket, { requestId, result, error }) ?? false;
4611
+ if (!sent) {
4612
+ this.logger.warn({ tool, requestId }, "Tool result could not be returned — originating MCP socket is closed");
3073
4613
  }
3074
- this.ipcServer?.send(socket, { requestId, result, error });
3075
4614
  };
3076
4615
  // Repo checkout — handled locally in daemon (no fleet-manager)
3077
4616
  if (tool === "checkout_repo") {
@@ -3084,13 +4623,7 @@ export class Daemon extends EventEmitter {
3084
4623
  }
3085
4624
  if (tool === "set_display_name" || tool === "set_description") {
3086
4625
  const type = tool === "set_display_name" ? "fleet_set_display_name" : "fleet_set_description";
3087
- const fleetReqId = `${tool === "set_display_name" ? "dn" : "desc"}_${requestId}`;
3088
- this.ipcServer?.broadcast({
3089
- type,
3090
- payload: args,
3091
- meta: { instance_name: this.name },
3092
- fleetRequestId: fleetReqId,
3093
- });
4626
+ const fleetReqId = `${tool === "set_display_name" ? "dn" : "desc"}_${++this.fleetRequestSeq}_${requestId}`;
3094
4627
  const timeout = setTimeout(() => {
3095
4628
  this.pendingIpcRequests.delete(fleetReqId);
3096
4629
  respond(null, `${tool} timed out`);
@@ -3099,16 +4632,16 @@ export class Daemon extends EventEmitter {
3099
4632
  clearTimeout(timeout);
3100
4633
  respond(respMsg.result, respMsg.error);
3101
4634
  });
3102
- return;
3103
- }
3104
- if (tool === TASK_TOOL) {
3105
- const fleetReqId = `task_${requestId}`;
3106
4635
  this.ipcServer?.broadcast({
3107
- type: "fleet_task",
4636
+ type,
3108
4637
  payload: args,
3109
4638
  meta: { instance_name: this.name },
3110
4639
  fleetRequestId: fleetReqId,
3111
4640
  });
4641
+ return;
4642
+ }
4643
+ if (tool === TASK_TOOL) {
4644
+ const fleetReqId = `task_${++this.fleetRequestSeq}_${requestId}`;
3112
4645
  const timeout = setTimeout(() => {
3113
4646
  this.pendingIpcRequests.delete(fleetReqId);
3114
4647
  respond(null, `Task operation timed out after ${daemonBudgetMs(tool) / 1000}s`);
@@ -3117,6 +4650,12 @@ export class Daemon extends EventEmitter {
3117
4650
  clearTimeout(timeout);
3118
4651
  respond(respMsg.result, respMsg.error);
3119
4652
  });
4653
+ this.ipcServer?.broadcast({
4654
+ type: "fleet_task",
4655
+ payload: args,
4656
+ meta: { instance_name: this.name },
4657
+ fleetRequestId: fleetReqId,
4658
+ });
3120
4659
  return;
3121
4660
  }
3122
4661
  if (DECISION_TOOLS.has(tool)) {
@@ -3125,13 +4664,7 @@ export class Daemon extends EventEmitter {
3125
4664
  list_decisions: "fleet_decision_list",
3126
4665
  update_decision: "fleet_decision_update",
3127
4666
  };
3128
- const fleetReqId = `dec_${requestId}`;
3129
- this.ipcServer?.broadcast({
3130
- type: typeMap[tool],
3131
- payload: args,
3132
- meta: { instance_name: this.name, working_directory: this.config.working_directory },
3133
- fleetRequestId: fleetReqId,
3134
- });
4667
+ const fleetReqId = `dec_${++this.fleetRequestSeq}_${requestId}`;
3135
4668
  const timeout = setTimeout(() => {
3136
4669
  this.pendingIpcRequests.delete(fleetReqId);
3137
4670
  respond(null, `Decision operation timed out after ${daemonBudgetMs(tool) / 1000}s`);
@@ -3140,6 +4673,12 @@ export class Daemon extends EventEmitter {
3140
4673
  clearTimeout(timeout);
3141
4674
  respond(respMsg.result, respMsg.error);
3142
4675
  });
4676
+ this.ipcServer?.broadcast({
4677
+ type: typeMap[tool],
4678
+ payload: args,
4679
+ meta: { instance_name: this.name, working_directory: this.config.working_directory },
4680
+ fleetRequestId: fleetReqId,
4681
+ });
3143
4682
  return;
3144
4683
  }
3145
4684
  if (SCHEDULE_TOOLS.has(tool)) {
@@ -3151,13 +4690,7 @@ export class Daemon extends EventEmitter {
3151
4690
  };
3152
4691
  // Use fleetRequestId (not requestId) to avoid MCP server resolving the
3153
4692
  // pending tool call prematurely when it receives the broadcast.
3154
- const fleetReqId = `sched_${requestId}`;
3155
- this.ipcServer?.broadcast({
3156
- type: typeMap[tool],
3157
- payload: args,
3158
- meta: { chat_id: this.lastChatId, thread_id: this.lastThreadId, instance_name: this.name },
3159
- fleetRequestId: fleetReqId,
3160
- });
4693
+ const fleetReqId = `sched_${++this.fleetRequestSeq}_${requestId}`;
3161
4694
  // Wait for fleet_schedule_response via pending request map
3162
4695
  const timeout = setTimeout(() => {
3163
4696
  this.pendingIpcRequests.delete(fleetReqId);
@@ -3167,6 +4700,12 @@ export class Daemon extends EventEmitter {
3167
4700
  clearTimeout(timeout);
3168
4701
  respond(respMsg.result, respMsg.error);
3169
4702
  });
4703
+ this.ipcServer?.broadcast({
4704
+ type: typeMap[tool],
4705
+ payload: args,
4706
+ meta: { chat_id: this.lastChatId, thread_id: this.lastThreadId, instance_name: this.name },
4707
+ fleetRequestId: fleetReqId,
4708
+ });
3170
4709
  return;
3171
4710
  }
3172
4711
  if (CROSS_INSTANCE_TOOLS.has(tool)) {
@@ -3174,15 +4713,8 @@ export class Daemon extends EventEmitter {
3174
4713
  if (this.topicMode && this.ipcServer) {
3175
4714
  // Use fleetRequestId (not requestId) to avoid MCP server resolving the
3176
4715
  // pending tool call prematurely when it receives the broadcast.
3177
- const fleetReqId = `xmsg_${requestId}`;
4716
+ const fleetReqId = `xmsg_${++this.fleetRequestSeq}_${requestId}`;
3178
4717
  const senderSessionName = this.socketSessionNames.get(socket);
3179
- this.ipcServer.broadcast({
3180
- type: "fleet_outbound",
3181
- tool,
3182
- args,
3183
- fleetRequestId: fleetReqId,
3184
- senderSessionName,
3185
- });
3186
4718
  // Shared table, so delete_instance and the deployment tools get the same
3187
4719
  // budget here as the MCP client expects (the ad-hoc list omitted them).
3188
4720
  const crossTimeoutMs = daemonBudgetMs(tool);
@@ -3194,6 +4726,13 @@ export class Daemon extends EventEmitter {
3194
4726
  clearTimeout(timeout);
3195
4727
  respond(respMsg.result, respMsg.error);
3196
4728
  });
4729
+ this.ipcServer.broadcast({
4730
+ type: "fleet_outbound",
4731
+ tool,
4732
+ args,
4733
+ fleetRequestId: fleetReqId,
4734
+ senderSessionName,
4735
+ });
3197
4736
  }
3198
4737
  else {
3199
4738
  respond(null, "Cross-instance messaging requires topic mode");
@@ -3227,8 +4766,24 @@ export class Daemon extends EventEmitter {
3227
4766
  // The fleet manager's IPC client receives this and routes to shared adapter.
3228
4767
  // Use fleetRequestId (not requestId) to avoid other MCP sessions on this daemon
3229
4768
  // from prematurely resolving their pending requests when they receive the broadcast.
3230
- const fleetReqId = `tool_${requestId}`;
4769
+ // requestId is local to one MCP server process. Claude may keep multiple
4770
+ // MCP children attached (main session, resumed session, subagent); those
4771
+ // processes all emit requestId=1,2,... and previously overwrote this
4772
+ // shared pending map. The wrong caller could then receive another POST's
4773
+ // success + messageId while its own reply had never been confirmed.
4774
+ const fleetReqId = `tool_${++this.fleetRequestSeq}_${requestId}`;
3231
4775
  const outboundKey = fleetReqId;
4776
+ const timeout = setTimeout(() => {
4777
+ this.pendingIpcRequests.delete(outboundKey);
4778
+ respond(null, `Fleet outbound timed out after ${daemonBudgetMs(tool) / 1000}s`);
4779
+ }, daemonBudgetMs(tool));
4780
+ this.pendingIpcRequests.set(outboundKey, (respMsg) => {
4781
+ clearTimeout(timeout);
4782
+ respond(respMsg.result, respMsg.error);
4783
+ });
4784
+ // Register the waiter before broadcasting. A fast/mock adapter can settle
4785
+ // immediately; broadcasting first left a response-before-registration
4786
+ // window that degraded into a timeout despite a completed POST.
3232
4787
  this.ipcServer?.broadcast({
3233
4788
  type: "fleet_outbound",
3234
4789
  tool,
@@ -3239,14 +4794,6 @@ export class Daemon extends EventEmitter {
3239
4794
  // from a secondary world, and message ids are not portable across bots.
3240
4795
  adapterId: this.lastAdapterId,
3241
4796
  });
3242
- const timeout = setTimeout(() => {
3243
- this.pendingIpcRequests.delete(outboundKey);
3244
- respond(null, `Fleet outbound timed out after ${daemonBudgetMs(tool) / 1000}s`);
3245
- }, daemonBudgetMs(tool));
3246
- this.pendingIpcRequests.set(outboundKey, (respMsg) => {
3247
- clearTimeout(timeout);
3248
- respond(respMsg.result, respMsg.error);
3249
- });
3250
4797
  return;
3251
4798
  }
3252
4799
  const adapter = adapters[0];
@@ -3256,7 +4803,10 @@ export class Daemon extends EventEmitter {
3256
4803
  }
3257
4804
  /** Build config object for the CLI backend */
3258
4805
  buildBackendConfig() {
3259
- const isCliMode = this.config.agent_mode === "cli" || (this.config.agent_mode == null && this.config.backend === "antigravity");
4806
+ // Antigravity 1.1.17 has no per-workspace MCP config, but its global MCP
4807
+ // children inherit the CLI process environment. The backend installs one
4808
+ // environment-selecting launcher, so MCP is now the safe default here too.
4809
+ const isCliMode = this.config.agent_mode === "cli";
3260
4810
  const sockPath = join(this.instanceDir, "channel.sock");
3261
4811
  let serverJs = join(__dirname, "channel", "mcp-server.js");
3262
4812
  if (!existsSync(serverJs)) {
@@ -3515,9 +5065,35 @@ export class Daemon extends EventEmitter {
3515
5065
  throw new Error("No backend configured — cannot spawn CLI window");
3516
5066
  }
3517
5067
  const attemptedResume = !this.skipResume;
3518
- const alive = await this.trySpawn();
5068
+ // A resume launch may get a longer budget than a fresh one (kiro: the
5069
+ // conversation must come back from the backend before anything paints).
5070
+ const resumeBudget = attemptedResume ? this.startupBudgetFor(true) : undefined;
5071
+ let alive = await this.trySpawn(false, resumeBudget);
5072
+ if (!alive && attemptedResume) {
5073
+ // Resume failed. Before abandoning the session:
5074
+ // 1. If the backend is unreachable (fleet-level memory, or this very pane
5075
+ // shows the outage text), clearing the session would trade a
5076
+ // transient outage for permanent conversation loss — and the fresh
5077
+ // start would hang on the same backend. Fail the startup and keep the
5078
+ // session; the fleet's delayed retry comes back once it is up.
5079
+ // 2. Otherwise, for backends that ask for it, retry resume ONCE — the
5080
+ // first miss is usually slowness, not a broken session.
5081
+ await this.noteStartupPaneForBackendOutage();
5082
+ await this.failStartupIfBackendUnreachable();
5083
+ if (this.backend.retriesResumeOnStartupFailure?.() === true) {
5084
+ this.logger.warn("Resume startup failed — retrying resume once before abandoning the session");
5085
+ await this.killProcessTree();
5086
+ await this.tmux.killWindow();
5087
+ alive = await this.trySpawn(false, resumeBudget);
5088
+ if (!alive) {
5089
+ await this.noteStartupPaneForBackendOutage();
5090
+ await this.failStartupIfBackendUnreachable();
5091
+ }
5092
+ }
5093
+ }
3519
5094
  if (!alive) {
3520
- // First attempt failed (stale --resume, crash, rate limit, etc.)
5095
+ // Resume (or a fresh start) failed for a reason we do not recognise as an
5096
+ // outage (stale --resume, crash, rate limit, etc.).
3521
5097
  // Clean slate: clear session-id, skip resume, and retry once.
3522
5098
  this.logger.warn("CLI startup failed — clearing session-id and retrying without resume");
3523
5099
  const sidFile = join(this.instanceDir, "session-id");
@@ -3528,7 +5104,7 @@ export class Daemon extends EventEmitter {
3528
5104
  this.skipResume = true;
3529
5105
  await this.killProcessTree();
3530
5106
  await this.tmux.killWindow();
3531
- const retryAlive = await this.trySpawn();
5107
+ const retryAlive = await this.trySpawn(false, this.startupBudgetFor(false));
3532
5108
  if (!retryAlive) {
3533
5109
  await this.killProcessTree();
3534
5110
  await this.tmux.killWindow();
@@ -3537,6 +5113,9 @@ export class Daemon extends EventEmitter {
3537
5113
  }
3538
5114
  else if (attemptedResume) {
3539
5115
  resumedSuccessfully = true;
5116
+ // A resume needs the backend: its success is positive proof the backend
5117
+ // is reachable again (a fresh prompt is local and proves nothing).
5118
+ this.backendOutage?.clear(this.backendKey());
3540
5119
  }
3541
5120
  this.lastSpawnAt = Date.now();
3542
5121
  this.skipResume = false; // CLI started successfully — reset for next spawn
@@ -3547,6 +5126,122 @@ export class Daemon extends EventEmitter {
3547
5126
  }
3548
5127
  return resumedSuccessfully;
3549
5128
  }
5129
+ /**
5130
+ * Startup budget for this launch: the backend's override (resume-aware), never
5131
+ * below a user-configured startup_timeout_ms; undefined = trySpawn's default.
5132
+ */
5133
+ startupBudgetFor(resume) {
5134
+ const override = this.backend?.getStartupBudgetMs?.({ resume });
5135
+ if (override == null)
5136
+ return undefined;
5137
+ const configured = this.config.startup_timeout_ms;
5138
+ return configured != null ? Math.max(configured, override) : override;
5139
+ }
5140
+ /** Effective wake budget = max(caller timeout, backend resume override, configured startup_timeout_ms). */
5141
+ wakeBudgetMs(timeoutMs) {
5142
+ return Math.max(timeoutMs, this.startupBudgetFor(!this.skipResume) ?? 0, this.config.startup_timeout_ms ?? 0);
5143
+ }
5144
+ /** The fleet-level backend key this instance runs on (matches the lifecycle's backendOf). */
5145
+ backendKey() {
5146
+ return this.runtimeIdentity?.backend ?? this.config.backend ?? this.backend?.binaryName ?? "unknown";
5147
+ }
5148
+ /**
5149
+ * After a failed launch, read the pane for the backend's fleet-wide outage
5150
+ * text (e.g. kiro's `dispatch failure (timeout) … kiro.dev`) and record it.
5151
+ * The lifecycle only learns about outages from RUNNING instances; during a
5152
+ * post-update herd nothing is running yet, so the startup path must report
5153
+ * what it sees or the first instance in would clear its session for nothing.
5154
+ */
5155
+ async noteStartupPaneForBackendOutage() {
5156
+ if (!this.backendOutage || !this.backend || !this.tmux)
5157
+ return;
5158
+ const outagePattern = this.backend.getErrorPatterns?.().find(p => p.fleetWide && p.type === "network");
5159
+ if (!outagePattern)
5160
+ return;
5161
+ try {
5162
+ const pane = await this.tmux.capturePane();
5163
+ if (!outagePattern.pattern.test(pane))
5164
+ return;
5165
+ const detail = this.resolveErrorMessage(pane, outagePattern);
5166
+ this.backendOutage.record(this.backendKey(), this.name, detail);
5167
+ this.logger.warn({ detail }, "Startup pane shows the backend outage text");
5168
+ }
5169
+ catch { /* capture failed — nothing to learn */ }
5170
+ }
5171
+ /**
5172
+ * Fail the startup WITHOUT touching the session while the backend is known to
5173
+ * be down. The failed window is torn down first: the alternative — a live but
5174
+ * never-ready CLI left behind while the daemon reports `crashed` — would keep
5175
+ * the health monitor from ever seeing a dead pane and retrying.
5176
+ */
5177
+ async failStartupIfBackendUnreachable() {
5178
+ if (!this.backendOutage?.isActive(this.backendKey()))
5179
+ return;
5180
+ this.logger.warn("Backend unreachable — keeping the session and failing startup for a delayed retry");
5181
+ await this.killProcessTree();
5182
+ await this.tmux.killWindow();
5183
+ throw new BackendUnreachableStartupError(this.backendKey());
5184
+ }
5185
+ /**
5186
+ * Crash-respawn ran into the backend outage: the ordinary path would leave
5187
+ * `crashed` + a paused monitor + nothing scheduled (and count it as a crash).
5188
+ * Hand the instance to the fleet instead — the lifecycle stops this daemon
5189
+ * and the fleet's delayed startup retry re-creates it later with the session
5190
+ * intact. Returns true when someone took the hand-off.
5191
+ */
5192
+ handOffBackendUnreachableRespawn(err) {
5193
+ if (!(err instanceof BackendUnreachableStartupError))
5194
+ return false;
5195
+ if (this.listenerCount("startup_backend_unreachable") === 0)
5196
+ return false;
5197
+ this.logger.warn({ backend: err.backend }, "Respawn blocked by backend outage — handing off to the fleet's delayed startup retry");
5198
+ this.healthCheckPaused = true;
5199
+ this.emit("startup_backend_unreachable", { name: this.name, backend: err.backend });
5200
+ return true;
5201
+ }
5202
+ /**
5203
+ * Dispose a Daemon whose start() rejected. The lifecycle registers a daemon
5204
+ * only after start() resolves, so nothing else owns this object — and by the
5205
+ * time the spawn fails it has already bound its IPC server (channel.sock),
5206
+ * written daemon.pid and possibly created a window. Left alone, each failed
5207
+ * start leaks one live-but-unreachable server handle (the next start unlinks
5208
+ * the socket path and binds a new one), multiplied by the automatic startup
5209
+ * retries. Idempotent, and fast: no graceful CLI quit, the CLI never reached
5210
+ * its prompt.
5211
+ */
5212
+ async abortStartup() {
5213
+ if (this.startupAborted)
5214
+ return;
5215
+ this.startupAborted = true;
5216
+ this.freezeRuntimeMonitors();
5217
+ this.pendingIpcRequests.clear();
5218
+ try {
5219
+ await this.killProcessTree();
5220
+ }
5221
+ catch { /* nothing running */ }
5222
+ if (this.tmux) {
5223
+ const windowId = this.tmux.getWindowId();
5224
+ try {
5225
+ await this.tmux.killWindow();
5226
+ }
5227
+ catch { /* window may not exist */ }
5228
+ if (windowId)
5229
+ this.controlClient?.unregisterWindow(windowId);
5230
+ }
5231
+ try {
5232
+ await this.ipcServer?.close();
5233
+ }
5234
+ catch (err) {
5235
+ this.logger.debug({ err }, "IPC server close failed during startup abort");
5236
+ }
5237
+ this.ipcServer = null;
5238
+ for (const file of ["daemon.pid", "window-id"]) {
5239
+ try {
5240
+ unlinkSync(join(this.instanceDir, file));
5241
+ }
5242
+ catch { /* absent */ }
5243
+ }
5244
+ }
3550
5245
  /** Kill the entire process tree of the current tmux pane (CLI + MCP server). */
3551
5246
  async killProcessTree(signal = "SIGTERM") {
3552
5247
  if (!this.tmux)
@@ -3567,6 +5262,15 @@ export class Daemon extends EventEmitter {
3567
5262
  * Returns true if CLI is ready, false if it failed or got stuck.
3568
5263
  */
3569
5264
  async trySpawn(reuseWindow = false, startupTimeoutMs) {
5265
+ if (!this.spawnGate)
5266
+ return this.trySpawnInsideGate(reuseWindow, startupTimeoutMs);
5267
+ return this.spawnGate.run({
5268
+ instanceName: this.name,
5269
+ workingDirectory: this.config.working_directory,
5270
+ reason: reuseWindow ? "wake" : this.lastSpawnAt > 0 ? "recovery" : "startup",
5271
+ }, () => this.trySpawnInsideGate(reuseWindow, startupTimeoutMs));
5272
+ }
5273
+ async trySpawnInsideGate(reuseWindow = false, startupTimeoutMs) {
3570
5274
  const backendConfig = this.buildBackendConfig();
3571
5275
  // Compare freshly-built instructions against the last value the agent was
3572
5276
  // told about. Computed for ALL backends (not gated by
@@ -3631,6 +5335,9 @@ export class Daemon extends EventEmitter {
3631
5335
  const cmd = `${envPrefix} ` + this.backend.buildCommand(backendConfig);
3632
5336
  // Ensure tmux session exists (may have been destroyed if all windows died)
3633
5337
  await TmuxManager.ensureSession(this.tmuxSessionName);
5338
+ if (this.stormWindow?.observeServerAlive(await TmuxManager.getServerPid(this.tmuxSessionName))) {
5339
+ this.emit("tmux_server_crash", this.name);
5340
+ }
3634
5341
  let windowId;
3635
5342
  if (reuseWindow) {
3636
5343
  this.controlClient?.unregisterWindow(this.tmux.getWindowId());
@@ -3682,18 +5389,47 @@ export class Daemon extends EventEmitter {
3682
5389
  // Dismiss confirmation dialogs and verify CLI reached prompt.
3683
5390
  // With remain-on-exit, isWindowAlive() returns true even for dead panes,
3684
5391
  // but a startup crash would already be caught by waitForOutput/waitForIdle above.
3685
- if (!await this.tmux.isWindowAlive())
5392
+ return this.finishStartupScan(STARTUP_DIALOG_BUDGET_MS);
5393
+ }
5394
+ /**
5395
+ * The last leg of a spawn: confirm the pane is not positively dead, then run
5396
+ * the dialog scan. The old pre-check used tmux's isWindowAlive(), which folds
5397
+ * every query failure into `false` — so one transient tmux error right here
5398
+ * read as "the CLI died", the caller abandoned --resume and cleared the
5399
+ * session, before the tri-state scanner ever got a look. Only a POSITIVE
5400
+ * dead pane fails; an unknown liveness is the scanner's problem (bounded,
5401
+ * session-preserving).
5402
+ */
5403
+ async finishStartupScan(budgetMs) {
5404
+ if ((await this.paneLiveness()) === "dead")
3686
5405
  return false;
3687
- const ready = await this.dismissDialogsUntilReady(3);
3688
- if (ready)
5406
+ const ready = await this.dismissDialogsUntilReady(budgetMs);
5407
+ // A fatal startup screen returns true (the spawn is over — respawning
5408
+ // would only re-show the same screen) but is NOT ready-for-delivery:
5409
+ // fatalStartupBlocked gates every pane write in deliverMessage.
5410
+ if (ready && !this.fatalStartupBlocked)
3689
5411
  this.firstDeliveryDelay.recordReady();
3690
5412
  return ready;
3691
5413
  }
3692
5414
  /**
3693
- * Repeatedly check pane content, dismiss any confirmation dialogs,
3694
- * and return true once CLI reaches a ready prompt.
5415
+ * Poll the pane within a wall-clock budget, dismiss the confirmation dialogs
5416
+ * the backend knows, and return true once the CLI shows a ready prompt with
5417
+ * no dialog on screen for two consecutive polls.
5418
+ *
5419
+ * Polls are spaced by `pollMs` and every key sequence / render wait counts
5420
+ * against the budget: Claude paints its session-resume prompt only after
5421
+ * loading the session, i.e. AFTER the first quiet moment, and three
5422
+ * back-to-back captures used to miss it — the scan then "assumed ok", the
5423
+ * `❯` inside the dialog satisfied the ready pattern, and the instance sat on
5424
+ * the dialog with deliveries free to press Enter into it.
5425
+ *
5426
+ * Only hard evidence returns false: a dead window or "command not found".
5427
+ * A transient capture failure is logged and retried; exhausting the budget is
5428
+ * logged at WARN and returns true — false makes the caller abandon --resume
5429
+ * and clear the session, which is the loss this code exists to prevent. The
5430
+ * delivery gate keeps refusing pastes while a blocking dialog is visible.
3695
5431
  */
3696
- async dismissDialogsUntilReady(maxAttempts) {
5432
+ async dismissDialogsUntilReady(budgetMs, pollMs = STARTUP_DIALOG_POLL_MS) {
3697
5433
  // Backend-specific startup dialogs, with hardcoded fallback for backward compat
3698
5434
  const startupDialogs = this.backend?.getStartupDialogs?.() ?? [
3699
5435
  { pattern: /[❯›]\s*\d+\.\s*No/m, keys: ["Down", "Enter"], description: "Confirmation dialog — navigate past No" },
@@ -3701,14 +5437,76 @@ export class Daemon extends EventEmitter {
3701
5437
  { pattern: /No, exit|No, quit|Don't trust|I accept|I trust|Yes, continue|Trust folder/i, keys: ["Enter"], description: "Generic confirmation dialog" },
3702
5438
  { pattern: /Resume Session/i, keys: ["Escape"], description: "Resume session picker — start fresh" },
3703
5439
  ];
3704
- for (let i = 0; i < maxAttempts; i++) {
5440
+ const deadline = Date.now() + budgetMs;
5441
+ const remaining = () => deadline - Date.now();
5442
+ const sleep = async () => { if (remaining() > 0)
5443
+ await new Promise(r => setTimeout(r, Math.min(pollMs, Math.max(remaining(), 0)))); };
5444
+ let cleanReadyPolls = 0;
5445
+ let lastDialog = null;
5446
+ let captureFailures = 0;
5447
+ let attempts = 0;
5448
+ do {
5449
+ attempts++;
5450
+ let pane;
5451
+ try {
5452
+ pane = await this.tmux.capturePane();
5453
+ }
5454
+ catch (err) {
5455
+ // Transient tmux trouble is not evidence about the CLI. Returning false
5456
+ // here would clear the session; retry within the budget instead.
5457
+ captureFailures++;
5458
+ this.logger.warn({ err, captureFailures }, "capture-pane failed during the startup dialog scan — retrying");
5459
+ // Only a POSITIVE dead pane ends the scan; an unanswerable liveness
5460
+ // query is the same transient trouble and must not clear the session.
5461
+ if ((await this.paneLiveness()) === "dead")
5462
+ return false;
5463
+ await sleep();
5464
+ continue;
5465
+ }
3705
5466
  try {
3706
- const pane = await this.tmux.capturePane();
3707
5467
  // Try each startup dialog pattern before checking ready state
3708
5468
  let matched = false;
5469
+ lastDialog = null;
3709
5470
  for (const dialog of startupDialogs) {
3710
- if (dialog.pattern.test(pane)) {
3711
- this.logger.debug(`Dismissing startup dialog: ${dialog.description}`);
5471
+ if (Daemon.dialogMatches(dialog, pane)) {
5472
+ lastDialog = dialog;
5473
+ cleanReadyPolls = 0;
5474
+ // Start the parked clock for every delivery-blocking dialog, exact
5475
+ // ones included: if the auto-dismiss keeps failing here, the human
5476
+ // report should count from first sight, not from the runtime scan.
5477
+ if (!dialog.fatal && this.deliveryBlockingDialogs().some(d => Daemon.dialogKey(d) === Daemon.dialogKey(dialog))) {
5478
+ this.trackDialogParked(dialog);
5479
+ }
5480
+ // A fatal startup screen cannot be dismissed by keypresses (e.g. a
5481
+ // corrupt claude.json modal offering only "exit" / "reset config").
5482
+ // It must be caught HERE and not fall through to the ready check:
5483
+ // the modal contains the `❯` selector the ready pattern matches, so
5484
+ // it would otherwise pass as ready and queued messages would be
5485
+ // typed into the modal. Report the error (pause stops delivery) and
5486
+ // stop the retry loop — like the sign-in screen case below, only a
5487
+ // human fixing the config ends this state.
5488
+ if (dialog.fatal) {
5489
+ // Block deliveries BEFORE the emit: the pty_error pause handler
5490
+ // is async, and a queued message would otherwise win the race
5491
+ // and be pasted into the modal.
5492
+ this.fatalStartupBlocked = true;
5493
+ if (!this.fatalStartupReported) {
5494
+ this.fatalStartupReported = true;
5495
+ this.logger.warn({ description: dialog.description }, "Fatal startup screen detected — reporting instead of dismissing");
5496
+ this.emit("pty_error", { name: this.name, ...dialog.fatal });
5497
+ }
5498
+ return true;
5499
+ }
5500
+ if (dialog.holdOnly) {
5501
+ // Recognised but deliberately not answered (see RuntimeDialog.holdOnly).
5502
+ if (this.dialogParkedKey !== Daemon.dialogKey(dialog)) {
5503
+ this.logger.warn({ description: dialog.description }, "Startup dialog held — not auto-answering, deliveries stay blocked");
5504
+ }
5505
+ this.trackDialogParked(dialog);
5506
+ matched = true;
5507
+ break;
5508
+ }
5509
+ this.logger.info(`Dismissing startup dialog: ${dialog.description}`);
3712
5510
  // Restart is exactly when inbound messages pile up, and nothing gates
3713
5511
  // delivery on `spawning`. Take the pane lock for the key sequence so a
3714
5512
  // queued message cannot be pasted into a half-dismissed trust dialog.
@@ -3723,35 +5521,82 @@ export class Daemon extends EventEmitter {
3723
5521
  await new Promise(r => setTimeout(r, 200));
3724
5522
  }
3725
5523
  });
3726
- // Wait for next screen to render
5524
+ // Wait for next screen to render — bounded by what is left of the budget.
5525
+ const renderWait = Math.max(0, Math.min(10_000, remaining()));
3727
5526
  if (this.controlClient) {
3728
5527
  const wid = readFileSync(join(this.instanceDir, "window-id"), "utf-8").trim();
3729
- await this.controlClient.waitForIdle(wid, 10_000);
5528
+ await this.controlClient.waitForIdle(wid, renderWait);
3730
5529
  }
3731
5530
  else {
3732
- await new Promise(r => setTimeout(r, 3_000));
5531
+ await new Promise(r => setTimeout(r, Math.min(3_000, renderWait)));
3733
5532
  }
3734
- if (!await this.tmux.isWindowAlive())
5533
+ if ((await this.paneLiveness()) === "dead")
3735
5534
  return false;
3736
5535
  matched = true;
3737
5536
  break;
3738
5537
  }
3739
5538
  }
3740
- if (matched)
5539
+ if (matched) {
5540
+ if (lastDialog?.holdOnly)
5541
+ await sleep();
5542
+ continue;
5543
+ }
5544
+ if (this.dialogParkedSince !== 0 && !this.deliveryBlockingDialogs().some(d => Daemon.dialogMatches(d, pane)))
5545
+ this.trackDialogParked(null);
5546
+ // CLI is ready (pattern defined by each backend). Require it on two
5547
+ // consecutive polls: the first ready frame is also the moment a late
5548
+ // dialog is about to be painted over it.
5549
+ if (this.backend.getReadyPattern().test(pane)) {
5550
+ cleanReadyPolls++;
5551
+ if (cleanReadyPolls >= 2 || remaining() <= 0) {
5552
+ // A real ready prompt (with no fatal dialog on screen — those are
5553
+ // matched above, before this check) means the fatal screen is gone,
5554
+ // e.g. the user fixed the config and the instance respawned.
5555
+ this.fatalStartupBlocked = false;
5556
+ return true;
5557
+ }
5558
+ await sleep();
3741
5559
  continue;
3742
- // CLI is ready (pattern defined by each backend)
3743
- if (this.backend.getReadyPattern().test(pane))
5560
+ }
5561
+ cleanReadyPolls = 0;
5562
+ // A CLI parked at its own sign-in screen is an AUTH incident: no dialog
5563
+ // key can dismiss it and no ready pattern will ever appear. Report it as
5564
+ // an auth error (the lifecycle double-checks with the token-free probe)
5565
+ // instead of letting it decay into crash/hang/MCP-died noise, and stop
5566
+ // the retry loop — only re-login (see /login) ends this state.
5567
+ const loginScreen = LOGIN_FLOWS[this.config.backend ?? "claude-code"]?.loginScreenPattern;
5568
+ if (loginScreen?.test(pane) && !this.loginScreenReported) {
5569
+ this.loginScreenReported = true;
5570
+ this.authFailureUnresolved = true;
5571
+ this.logger.warn({ backend: this.config.backend }, "CLI is waiting at its sign-in screen — reporting auth error");
5572
+ this.emit("pty_error", {
5573
+ name: this.name,
5574
+ type: "auth_error",
5575
+ action: "pause",
5576
+ message: "CLI is waiting at its sign-in screen — credentials are missing or expired",
5577
+ });
3744
5578
  return true;
5579
+ }
3745
5580
  // Fatal: command not found (must match full phrase to avoid false positives
3746
5581
  // like Kiro's "agent X not found, using default")
3747
5582
  if (/command not found|: not found$/m.test(pane))
3748
5583
  return false;
3749
5584
  }
3750
- catch {
3751
- return false;
5585
+ catch (err) {
5586
+ // Key sends / isWindowAlive failing: same rule — log, retry within the budget.
5587
+ this.logger.warn({ err }, "startup dialog scan step failed — retrying");
5588
+ if ((await this.paneLiveness()) === "dead")
5589
+ return false;
3752
5590
  }
5591
+ await sleep();
5592
+ } while (remaining() > 0);
5593
+ // Budget exhausted. Never silently: say which way it went.
5594
+ if (lastDialog) {
5595
+ this.logger.warn({ description: lastDialog.description, attempts, budgetMs }, "Startup scan exhausted with a dialog still on screen — reporting the CLI alive; deliveries stay blocked until it is answered");
5596
+ }
5597
+ else {
5598
+ this.logger.warn({ attempts, budgetMs, captureFailures }, "Startup scan exhausted without a ready prompt or a known dialog — assuming ready (unknown CLI screen)");
3753
5599
  }
3754
- // Exhausted attempts — assume ok for unknown CLI prompts
3755
5600
  return true;
3756
5601
  }
3757
5602
  /**