@songsid/agend 2.1.1-beta.17 → 2.1.1-beta.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/dist/backend/claude-code.d.ts +37 -0
  2. package/dist/backend/claude-code.js +40 -1
  3. package/dist/backend/claude-code.js.map +1 -1
  4. package/dist/backend/kiro.d.ts +20 -0
  5. package/dist/backend/kiro.js +30 -0
  6. package/dist/backend/kiro.js.map +1 -1
  7. package/dist/backend/types.d.ts +27 -0
  8. package/dist/backend/types.js.map +1 -1
  9. package/dist/channel/adapters/discord.d.ts +15 -0
  10. package/dist/channel/adapters/discord.js +81 -1
  11. package/dist/channel/adapters/discord.js.map +1 -1
  12. package/dist/channel/adapters/telegram.d.ts +21 -0
  13. package/dist/channel/adapters/telegram.js +82 -0
  14. package/dist/channel/adapters/telegram.js.map +1 -1
  15. package/dist/channel/ipc-bridge.d.ts +9 -1
  16. package/dist/channel/ipc-bridge.js +12 -3
  17. package/dist/channel/ipc-bridge.js.map +1 -1
  18. package/dist/channel/types.d.ts +29 -0
  19. package/dist/daemon.d.ts +101 -5
  20. package/dist/daemon.js +304 -50
  21. package/dist/daemon.js.map +1 -1
  22. package/dist/fleet-manager.d.ts +92 -1
  23. package/dist/fleet-manager.js +330 -19
  24. package/dist/fleet-manager.js.map +1 -1
  25. package/dist/hang-detector.d.ts +19 -13
  26. package/dist/hang-detector.js +19 -49
  27. package/dist/hang-detector.js.map +1 -1
  28. package/dist/pane-write-lock.d.ts +48 -0
  29. package/dist/pane-write-lock.js +73 -0
  30. package/dist/pane-write-lock.js.map +1 -0
  31. package/dist/tmux-control.d.ts +48 -1
  32. package/dist/tmux-control.js +79 -6
  33. package/dist/tmux-control.js.map +1 -1
  34. package/package.json +1 -1
  35. package/dist/channel/tool-tracker.d.ts +0 -13
  36. package/dist/channel/tool-tracker.js +0 -58
  37. package/dist/channel/tool-tracker.js.map +0 -1
  38. package/dist/daemon-entry.d.ts +0 -1
  39. package/dist/daemon-entry.js +0 -30
  40. package/dist/daemon-entry.js.map +0 -1
package/dist/daemon.js CHANGED
@@ -16,6 +16,7 @@ import { shellQuote } from "./backend/types.js";
16
16
  import { getTmuxSession } from "./config.js";
17
17
  import { routeToolCall } from "./channel/tool-router.js";
18
18
  import { HangDetector } from "./hang-detector.js";
19
+ import { PaneWriteLock } from "./pane-write-lock.js";
19
20
  import { buildFleetInstructions } from "./instructions.js";
20
21
  const __filename = fileURLToPath(import.meta.url);
21
22
  const __dirname = dirname(__filename);
@@ -40,8 +41,6 @@ export function buildInstructionReloadNotice(binaryName, instanceName, instanceD
40
41
  export const DEFAULT_STUCK_TIMEOUT_MS = 10 * 60_000;
41
42
  export const DEFAULT_STATE_IDLE_DEBOUNCE_MS = 2_000;
42
43
  export const DEFAULT_STATE_SAFETY_SWEEP_MS = 60_000;
43
- /** @deprecated State detection is event-driven; this now aliases the safety sweep. */
44
- export const DEFAULT_STATE_POLL_INTERVAL_MS = DEFAULT_STATE_SAFETY_SWEEP_MS;
45
44
  const LAST_INBOUND_FILE = "last-inbound-at";
46
45
  /** Minimum gap between "health check is failing" notifications for one instance. */
47
46
  const HEALTH_ERROR_NOTIFY_INTERVAL_MS = 10 * 60_000;
@@ -186,24 +185,47 @@ export class AutoPauseController {
186
185
  * Pane motion wins over a ready match because several backends keep their ready
187
186
  * marker in a persistent header/footer while generating. A stable ready pane is
188
187
  * idle; changing content is working; stable non-ready content eventually sticks.
188
+ *
189
+ * ## Why an optional busy pattern exists
190
+ *
191
+ * "Motion wins" degrades badly when a backend's ready marker is *always* on
192
+ * screen. `stuck` requires `!ready`, so for such a backend the state can only ever
193
+ * be idle or working — the stuck edge, and with it `handleStuckTransition` and the
194
+ * hang notification, is unreachable. A frozen CLI is then reported as idle, which
195
+ * also clears pending work: the message the user sent is quietly booked as done.
196
+ *
197
+ * claude-code was exactly this case (see ClaudeCodeBackend.getReadyPattern). A
198
+ * backend that can point at a marker meaning "generating right now" supplies
199
+ * `getBusyPattern()`, and that match vetoes ready regardless of what the ready
200
+ * pattern says.
189
201
  */
190
202
  export class PaneStateMachine {
191
203
  stuckTimeoutMs;
192
204
  readyPattern;
205
+ busyPattern;
193
206
  lastPaneHash = null;
194
207
  lastPaneChangeAt;
195
208
  lastObservedAt;
196
209
  stateChangedAt;
197
210
  currentState = "idle";
198
- constructor(readyPattern, stuckTimeoutMs = DEFAULT_STUCK_TIMEOUT_MS, now = Date.now()) {
211
+ constructor(readyPattern, stuckTimeoutMs = DEFAULT_STUCK_TIMEOUT_MS, now = Date.now(), busyPattern) {
199
212
  this.stuckTimeoutMs = stuckTimeoutMs;
200
213
  // Stateful g/y regexes mutate lastIndex and can alternate true/false across
201
214
  // polls. State detection must be deterministic for identical pane content.
202
215
  this.readyPattern = new RegExp(readyPattern.source, readyPattern.flags.replace(/[gy]/g, ""));
216
+ this.busyPattern = busyPattern
217
+ ? new RegExp(busyPattern.source, busyPattern.flags.replace(/[gy]/g, ""))
218
+ : null;
203
219
  this.lastPaneChangeAt = now;
204
220
  this.lastObservedAt = now;
205
221
  this.stateChangedAt = now;
206
222
  }
223
+ /** Ready, unless the backend can positively see that it is still generating. */
224
+ isReady(pane) {
225
+ if (this.busyPattern?.test(pane))
226
+ return false;
227
+ return this.readyPattern.test(pane);
228
+ }
207
229
  observe(pane, now = Date.now()) {
208
230
  const paneHash = createHash("sha256").update(pane).digest("hex");
209
231
  const firstObservation = this.lastPaneHash === null;
@@ -213,7 +235,7 @@ export class PaneStateMachine {
213
235
  this.lastPaneChangeAt = now;
214
236
  }
215
237
  this.lastObservedAt = now;
216
- const ready = this.readyPattern.test(pane);
238
+ const ready = this.isReady(pane);
217
239
  const nextState = firstObservation
218
240
  ? ready ? "idle" : "working"
219
241
  : paneChanged
@@ -279,6 +301,21 @@ const FIRST_ENTER_SETTLE_MS = 1_750;
279
301
  const FIRST_DELIVERY_WINDOW_MS = 5_000;
280
302
  /** After busy native-queue paste+Enter, wait before checking the pane for silent loss. */
281
303
  const NATIVE_QUEUE_PASTE_VERIFY_MS = 2_000;
304
+ /** Enter-confirmation poll: 10 × 200ms ≈ 2s of observed silence before giving up. */
305
+ const CONFIRM_BUSY_POLLS = 10;
306
+ const CONFIRM_BUSY_POLL_MS = 200;
307
+ /**
308
+ * Ceiling on the whole confirmation, including time re-earned after a control-mode
309
+ * reconnect. Without it a reconnect loop could restart the poll indefinitely and
310
+ * hold the pane write lock along with it.
311
+ */
312
+ const CONFIRM_BUSY_MAX_WAIT_MS = 10_000;
313
+ /**
314
+ * How long a delivery waits for an in-flight spawn. Comfortably past the default
315
+ * 25s startup timeout plus dialog dismissal; past it we fall back to the old
316
+ * behaviour rather than holding a message indefinitely.
317
+ */
318
+ const SPAWN_SETTLE_MAX_WAIT_MS = 60_000;
282
319
  /**
283
320
  * One-shot timing gate for the first paste after a CLI reaches its ready
284
321
  * prompt. Recent tmux versions expose a short interval where the prompt is
@@ -337,7 +374,6 @@ export class Daemon extends EventEmitter {
337
374
  ipcServer = null;
338
375
  messageBus;
339
376
  transcriptMonitor = null;
340
- toolTracker = null;
341
377
  guardian = null;
342
378
  adapter = null;
343
379
  pendingIpcRequests = new Map();
@@ -348,6 +384,8 @@ export class Daemon extends EventEmitter {
348
384
  // Pending ack: react 🫡 on first transcript activity after receiving a message
349
385
  pendingAckMessage = null;
350
386
  // Tool status tracking for channel adapter
387
+ /** Last activity published to the fleet manager; null when nothing is running. */
388
+ currentActivity = null;
351
389
  toolStatusMessageId = null;
352
390
  toolStatusLines = [];
353
391
  toolStatusDebounce = null;
@@ -367,6 +405,15 @@ export class Daemon extends EventEmitter {
367
405
  /** CLI pane availability, independent from the daemon process and tri-state. */
368
406
  processStatus = "running";
369
407
  spawning = false;
408
+ /**
409
+ * Resolves when the in-flight spawn finishes; null when none is running.
410
+ *
411
+ * `spawning` alone can only be polled. Delivery needs to *wait*, and a boolean
412
+ * poll would either busy-loop or race the flag being cleared.
413
+ */
414
+ spawnSettled = null;
415
+ resolveSpawnSettled = null;
416
+ spawnDepth = 0;
370
417
  skipResume = false;
371
418
  backgroundSessionRecoveryAttempted = false;
372
419
  /** Whether the last spawn started a fresh session (not resumed). */
@@ -389,6 +436,7 @@ export class Daemon extends EventEmitter {
389
436
  instanceStateIdleDebounceMs = DEFAULT_STATE_IDLE_DEBOUNCE_MS;
390
437
  instanceStateStuckTimeoutMs = DEFAULT_STUCK_TIMEOUT_MS;
391
438
  instanceStateReadyPattern = null;
439
+ instanceStateBusyPattern = null;
392
440
  instanceStateMonitorActive = false;
393
441
  statePollInFlight = false;
394
442
  autoPauseController;
@@ -409,7 +457,13 @@ export class Daemon extends EventEmitter {
409
457
  recentEvents = [];
410
458
  recentToolActivity = [];
411
459
  snapshotConsumed = false;
460
+ /** Orders inbound channel messages against each other (queue-depth accounting
461
+ * and ⏳/👀/✅ reactions live here). It does NOT cover the other writers that
462
+ * reach the pane — {@link paneWriteLock} does. */
412
463
  pasteLock = Promise.resolve();
464
+ /** Mutual exclusion for *every* write into the pane, whichever subsystem it
465
+ * comes from. See PaneWriteLock for why interleaving is destructive. */
466
+ paneWriteLock = new PaneWriteLock();
413
467
  pendingInstructionsUpdate;
414
468
  pendingInstructionsNotice = false;
415
469
  // Whether the warmup steering-reload notice should be injected after spawn.
@@ -552,9 +606,6 @@ export class Daemon extends EventEmitter {
552
606
  if ((type === "fleet_schedule_response" || type === "fleet_outbound_response" || type === "fleet_decision_response" || type === "fleet_task_response" || type === "fleet_display_name_response" || type === "fleet_description_response") && msg.fleetRequestId) {
553
607
  key = String(msg.fleetRequestId);
554
608
  }
555
- else if (type === "fleet_outbound_response" && msg.requestId != null) {
556
- key = `fleet_out_${msg.requestId}`;
557
- }
558
609
  if (key && this.pendingIpcRequests.has(key)) {
559
610
  const handler = this.pendingIpcRequests.get(key);
560
611
  this.pendingIpcRequests.delete(key);
@@ -603,6 +654,14 @@ export class Daemon extends EventEmitter {
603
654
  // Fleet manager routed a message to us (topic mode)
604
655
  const meta = msg.meta;
605
656
  const targetSession = msg.targetSession;
657
+ // `no_wake` marks low-value signals (a non-approval reaction, #408): worth
658
+ // showing an instance that is already awake, never worth resuming a paused
659
+ // one and spending a turn on. Dropped rather than queued, so a paused
660
+ // instance does not surface a pile of stale emoji on its next wake.
661
+ if (meta?.no_wake === "true" && this.isPaused) {
662
+ this.logger.debug({ preview: String(msg.content).slice(0, 60) }, "Dropping no_wake message for paused instance");
663
+ return;
664
+ }
606
665
  void this.wake().then(() => {
607
666
  this.pushChannelMessage(msg.content, meta, targetSession);
608
667
  }).catch(err => {
@@ -705,7 +764,12 @@ export class Daemon extends EventEmitter {
705
764
  else {
706
765
  await new Promise(r => setTimeout(r, 5000));
707
766
  }
708
- await this.tmux?.pasteText(buildInstructionReloadNotice(this.backend?.binaryName ?? "unknown", this.name, this.instanceDir));
767
+ // This path only runs when pasteQueueDepth > 0 — i.e. exactly when a real
768
+ // delivery is already in flight or queued. Without the lock the notice and
769
+ // that delivery race into the same pane.
770
+ await this.paneWriteLock.run(async () => {
771
+ await this.tmux?.pasteText(buildInstructionReloadNotice(this.backend?.binaryName ?? "unknown", this.name, this.instanceDir));
772
+ });
709
773
  // Record the value the agent has now been told about so the next
710
774
  // unchanged restart skips the reload.
711
775
  try {
@@ -738,27 +802,31 @@ export class Daemon extends EventEmitter {
738
802
  this.transcriptMonitor.on("tool_use", (name, input) => {
739
803
  this.logger.debug({ tool: name }, "Tool use");
740
804
  ackIfPending();
741
- this.hangDetector?.recordActivity();
742
805
  this.recordRecentEvent({ type: "tool_use", name, preview: this.summarizeTool(name, input) });
743
806
  this.recordRecentToolActivity(this.summarizeTool(name, input));
807
+ this.publishActivity(this.summarizeTool(name, input));
744
808
  });
745
809
  this.transcriptMonitor.on("tool_result", (name, _output) => {
746
- this.hangDetector?.recordActivity();
747
810
  this.recordRecentEvent({ type: "tool_result", name });
811
+ // The tool finished; whatever comes next has not started yet. Better to
812
+ // show only elapsed time than to leave a stale "Bash: npm test" on screen.
813
+ this.publishActivity(null);
748
814
  });
749
815
  this.transcriptMonitor.on("assistant_text", (text) => {
750
816
  this.logger.debug({ text: text.slice(0, 200) }, "Claude response");
751
817
  ackIfPending();
752
- this.hangDetector?.recordActivity();
753
818
  this.recordRecentEvent({ type: "assistant_text", preview: text.slice(0, 100) });
754
819
  });
755
820
  this.transcriptMonitor.startPolling();
756
- // HangDetector remains the fleet-manager notification event bridge. Its
757
- // legacy silence timer is intentionally not started: pane state transitions
758
- // below are now the sole source of hang events.
821
+ // HangDetector is the notification bridge to the fleet manager; pane state
822
+ // transitions below are the sole source of hang events (see hang-detector.ts).
823
+ // `hang_detector.enabled: false` opts out of the notification entirely.
824
+ // NOTE: `hang_detector.timeout_minutes` is NOT read here — the stuck timeout
825
+ // the pane monitor actually uses is resolved separately below. The two have
826
+ // long been separate; documented rather than silently ignored.
759
827
  const hangConfig = this.config.hang_detector;
760
828
  if (hangConfig?.enabled !== false) {
761
- this.hangDetector = new HangDetector(hangConfig?.timeout_minutes ?? 10);
829
+ this.hangDetector = new HangDetector();
762
830
  }
763
831
  // 8. Context guardian
764
832
  const statusFile = join(this.instanceDir, "statusline.json");
@@ -766,7 +834,6 @@ export class Daemon extends EventEmitter {
766
834
  this.guardian.startWatching();
767
835
  this.guardian.on("status_update", () => {
768
836
  this.saveSessionId();
769
- this.hangDetector?.recordStatuslineUpdate();
770
837
  });
771
838
  // Context rotation removed: all CLI backends have built-in auto-compact.
772
839
  // Crash recovery (health check + respawn with snapshot) is retained below.
@@ -1185,6 +1252,7 @@ export class Daemon extends EventEmitter {
1185
1252
  if (!this.backend)
1186
1253
  return; // lightweight mode has no backend
1187
1254
  const readyPattern = this.backend.getReadyPattern();
1255
+ const busyPattern = this.backend.getBusyPattern?.() ?? null;
1188
1256
  this.errorMonitorTimer = setInterval(async () => {
1189
1257
  if (!this.tmux || this.spawning)
1190
1258
  return;
@@ -1197,20 +1265,30 @@ export class Daemon extends EventEmitter {
1197
1265
  for (const dialog of dialogs) {
1198
1266
  if (!dialog.pattern.test(pane))
1199
1267
  continue;
1200
- this.logger.info(`Auto-dismissing runtime dialog: ${dialog.description}`);
1201
- const SPECIAL_KEYS = new Set(["Up", "Down", "Enter", "Escape", "Right", "Left"]);
1202
- for (const key of dialog.keys) {
1203
- if (SPECIAL_KEYS.has(key)) {
1204
- await this.tmux.sendSpecialKey(key);
1205
- }
1206
- else {
1207
- await this.tmux.pasteText(key);
1268
+ // These keys go straight into the pane. Sent while a message delivery is
1269
+ // mid-transaction, an `Escape` wipes the pasted text and an `Enter`
1270
+ // submits it half-composed — the user sees a message that vanished. Skip
1271
+ // (not queue) when the pane is busy: this poller runs every 5s, and the
1272
+ // dialog will still be on screen next tick.
1273
+ const dismissed = await this.paneWriteLock.tryRun(async () => {
1274
+ this.logger.info(`Auto-dismissing runtime dialog: ${dialog.description}`);
1275
+ const SPECIAL_KEYS = new Set(["Up", "Down", "Enter", "Escape", "Right", "Left"]);
1276
+ for (const key of dialog.keys) {
1277
+ if (SPECIAL_KEYS.has(key)) {
1278
+ await this.tmux.sendSpecialKey(key);
1279
+ }
1280
+ else {
1281
+ await this.tmux.pasteText(key);
1282
+ }
1283
+ await new Promise(r => setTimeout(r, 200));
1208
1284
  }
1209
- await new Promise(r => setTimeout(r, 200));
1285
+ });
1286
+ if (!dismissed) {
1287
+ this.logger.debug({ dialog: dialog.description }, "Dialog dismissal deferred — pane write in flight");
1210
1288
  }
1211
- return; // Dialog dismissed, skip error checks this cycle
1289
+ return; // Dialog handled (or deliberately deferred): skip error checks this cycle
1212
1290
  }
1213
- this.evaluateErrorPatterns(pane, patterns, readyPattern);
1291
+ this.evaluateErrorPatterns(pane, patterns, readyPattern, Date.now(), busyPattern);
1214
1292
  }
1215
1293
  catch {
1216
1294
  // capturePane can fail if window is transitioning — ignore
@@ -1218,7 +1296,7 @@ export class Daemon extends EventEmitter {
1218
1296
  }, 5_000); // Check every 5 seconds (runtime dialogs need fast response)
1219
1297
  }
1220
1298
  /** Evaluate one pane snapshot. Kept synchronous so state-machine edges are unit-testable. */
1221
- evaluateErrorPatterns(pane, patterns, readyPattern, now = Date.now()) {
1299
+ evaluateErrorPatterns(pane, patterns, readyPattern, now = Date.now(), busyPattern = null) {
1222
1300
  // Count occurrences across the WHOLE pane (not just text after the last
1223
1301
  // ready prompt). Clone with `g` so stateful backend regexes cannot leak
1224
1302
  // lastIndex between monitor cycles.
@@ -1226,10 +1304,15 @@ export class Daemon extends EventEmitter {
1226
1304
  const flags = pattern.flags.includes("g") ? pattern.flags : pattern.flags + "g";
1227
1305
  return (pane.match(new RegExp(pattern.source, flags)) || []).length;
1228
1306
  };
1307
+ // Same veto as the state machine: a backend whose ready marker is permanently
1308
+ // on screen would otherwise "recover" on the very first tick after the error,
1309
+ // rebaselining the occurrence count while the error is still displayed — one
1310
+ // notification, a false recovery log, then silence.
1311
+ const looksReady = () => !busyPattern?.test(pane) && readyPattern.test(pane);
1229
1312
  // State: waiting for recovery. A missing/outdated ready pattern must not
1230
1313
  // suppress every future error forever, so the gate has a hard deadline.
1231
1314
  if (this.errorWaitingForRecovery) {
1232
- if (readyPattern.test(pane)) {
1315
+ if (looksReady()) {
1233
1316
  const downtime = Math.round((now - this.errorDetectedAt) / 1000);
1234
1317
  const active = patterns.find(ep => Daemon.errorPatternKey(ep) === this.activeErrorPatternKey);
1235
1318
  if (active) {
@@ -1328,6 +1411,12 @@ export class Daemon extends EventEmitter {
1328
1411
  * Interrupt the CLI's current generation (cancel button / `/cancel`).
1329
1412
  * Direct tmux key event (not a paste) so it registers as the interrupt key.
1330
1413
  * kiro-cli interrupts on Ctrl+C; the others (claude-code, codex, …) on Escape.
1414
+ *
1415
+ * Deliberately NOT taken through `paneWriteLock`. Cancel is the user's way out
1416
+ * of a pane that is busy or wedged — queueing it behind the very delivery chain
1417
+ * it exists to unblock would make the button useless exactly when it is needed.
1418
+ * The cost is accepted: an Escape landing between a paste and its Enter discards
1419
+ * that message, which is what the user asked for anyway.
1331
1420
  */
1332
1421
  async sendEscape() {
1333
1422
  const cancelKey = this.backend?.getCancelKey() ?? "Escape";
@@ -1365,7 +1454,6 @@ export class Daemon extends EventEmitter {
1365
1454
  this.toolStatusDebounce = null;
1366
1455
  }
1367
1456
  this.pendingIpcRequests.clear();
1368
- this.hangDetector?.stop();
1369
1457
  if (this.adapter)
1370
1458
  await this.adapter.stop();
1371
1459
  // Notify MCP servers of graceful shutdown (prevents reconnect attempts)
@@ -1392,7 +1480,12 @@ export class Daemon extends EventEmitter {
1392
1480
  if (!killed)
1393
1481
  this.logger.warn("CLI did not exit gracefully within 3s, force killing window");
1394
1482
  // Always kill window — remain-on-exit keeps dead panes around after CLI exits
1483
+ const stoppedWindowId = this.tmux.getWindowId();
1395
1484
  await this.tmux.killWindow();
1485
+ // The control client is shared across the fleet and outlives this daemon, so
1486
+ // a registration left behind here is re-resolved on every reconnect forever.
1487
+ if (stoppedWindowId)
1488
+ this.controlClient?.unregisterWindow(stoppedWindowId);
1396
1489
  const windowIdFile = join(this.instanceDir, "window-id");
1397
1490
  try {
1398
1491
  unlinkSync(windowIdFile);
@@ -1528,7 +1621,7 @@ export class Daemon extends EventEmitter {
1528
1621
  return;
1529
1622
  }
1530
1623
  this.pauseWakeState = "waking";
1531
- this.spawning = true;
1624
+ this.beginSpawn();
1532
1625
  const transition = this.autoPauseController.wakeOnDeliver(async () => {
1533
1626
  let timeout;
1534
1627
  try {
@@ -1569,7 +1662,7 @@ export class Daemon extends EventEmitter {
1569
1662
  throw err;
1570
1663
  }
1571
1664
  finally {
1572
- this.spawning = false;
1665
+ this.endSpawn();
1573
1666
  if (this.pauseWakeTransition === transition)
1574
1667
  this.pauseWakeTransition = null;
1575
1668
  }
@@ -1581,6 +1674,10 @@ export class Daemon extends EventEmitter {
1581
1674
  // observations between enqueue and paste must not clear a newer inbound.
1582
1675
  if (snapshot.state === "idle" && previous !== "idle") {
1583
1676
  this.pendingWork.recordIdle(snapshot.observedAt);
1677
+ // The turn is over. A transcript can end on a tool_use with no matching
1678
+ // tool_result (interrupted, crashed, cancelled), which would otherwise leave
1679
+ // the last tool pinned to the progress line for the rest of the session.
1680
+ this.publishActivity(null);
1584
1681
  }
1585
1682
  if (snapshot.state !== previous) {
1586
1683
  this.logger.info({
@@ -1671,6 +1768,13 @@ export class Daemon extends EventEmitter {
1671
1768
  this.instanceStateMachine.observe(pane, observedChangeAt);
1672
1769
  const snapshot = this.instanceStateMachine.observe(pane, Date.now());
1673
1770
  this.applyInstanceStateSnapshot(snapshot, pane);
1771
+ // Backends without a transcript feed can still say what they are doing, if
1772
+ // their TUI names it. Free-riding on a capture that already happened: no
1773
+ // extra tmux call. Cadence is the capture cadence (idle debounce + the 60s
1774
+ // safety sweep), which matches the progress ticker's own 60s interval.
1775
+ if (this.backend?.getPaneActivity) {
1776
+ this.publishActivity(snapshot.state === "idle" ? null : this.backend.getPaneActivity(pane));
1777
+ }
1674
1778
  if (snapshot.state === "idle") {
1675
1779
  this.clearInstanceStateStuckTimer();
1676
1780
  }
@@ -1725,7 +1829,8 @@ export class Daemon extends EventEmitter {
1725
1829
  ? rawConfig.idle_debounce_ms
1726
1830
  : DEFAULT_STATE_IDLE_DEBOUNCE_MS;
1727
1831
  this.instanceStateReadyPattern = this.backend.getReadyPattern();
1728
- this.instanceStateMachine = new PaneStateMachine(this.instanceStateReadyPattern, this.instanceStateStuckTimeoutMs);
1832
+ this.instanceStateBusyPattern = this.backend.getBusyPattern?.() ?? null;
1833
+ this.instanceStateMachine = new PaneStateMachine(this.instanceStateReadyPattern, this.instanceStateStuckTimeoutMs, Date.now(), this.instanceStateBusyPattern);
1729
1834
  this.instanceStateLastOutputAt = 0;
1730
1835
  this.instanceStateMonitorActive = true;
1731
1836
  if (this.controlClient) {
@@ -1766,6 +1871,12 @@ export class Daemon extends EventEmitter {
1766
1871
  paneTail: sanitizePaneTail(pane),
1767
1872
  readyPattern: readyPattern.toString(),
1768
1873
  readyMatched: deterministicReadyPattern.test(pane),
1874
+ // A pane that is stuck *while* the ready marker matches means the busy
1875
+ // pattern is what held it back — worth seeing when tuning either regex.
1876
+ busyPattern: this.instanceStateBusyPattern?.toString(),
1877
+ busyMatched: this.instanceStateBusyPattern
1878
+ ? new RegExp(this.instanceStateBusyPattern.source, this.instanceStateBusyPattern.flags.replace(/[gy]/g, "")).test(pane)
1879
+ : undefined,
1769
1880
  unchangedForMs: snapshot.unchangedForMs,
1770
1881
  pendingWork: this.pendingWork.hasPendingWork(),
1771
1882
  };
@@ -1808,6 +1919,28 @@ export class Daemon extends EventEmitter {
1808
1919
  return this.messageBus;
1809
1920
  }
1810
1921
  // ── Tool status tracking ──────────────────────────────────────
1922
+ /**
1923
+ * Tell the fleet manager what this instance is doing right now, for the live
1924
+ * progress line on the cancel button.
1925
+ *
1926
+ * Purely cosmetic — nothing decides anything from it, so it is fine that only
1927
+ * backends with a transcript feed report at all (claude-code today). For the
1928
+ * rest the progress line keeps showing just elapsed time.
1929
+ *
1930
+ * Repeats are dropped: the ticker only edits the channel message when the text
1931
+ * changes, and a stream of identical broadcasts would defeat that.
1932
+ */
1933
+ publishActivity(activity) {
1934
+ const next = activity && activity.trim() ? activity.trim() : null;
1935
+ if (next === this.currentActivity)
1936
+ return;
1937
+ this.currentActivity = next;
1938
+ this.ipcServer?.broadcast({
1939
+ type: "instance_activity",
1940
+ instanceName: this.name,
1941
+ activity: next,
1942
+ });
1943
+ }
1811
1944
  summarizeTool(name, input) {
1812
1945
  const inp = input;
1813
1946
  if (!inp)
@@ -1897,7 +2030,6 @@ export class Daemon extends EventEmitter {
1897
2030
  writeFileSync(join(this.instanceDir, "prev-instructions"), this.pendingInstructionsUpdate);
1898
2031
  this.pendingInstructionsUpdate = undefined;
1899
2032
  }
1900
- this.hangDetector?.recordInbound();
1901
2033
  this.pendingWork.recordInbound();
1902
2034
  // v3: record user messages for rotation snapshot
1903
2035
  this.recordRecentUserMessage(content, meta);
@@ -1997,6 +2129,8 @@ export class Daemon extends EventEmitter {
1997
2129
  // Odd number of fences = unclosed. Remove all code fences from the message.
1998
2130
  formatted = formatted.replace(/```/g, "");
1999
2131
  }
2132
+ // Before anything reads the window id: a spawn in progress is about to change it.
2133
+ await this.waitForSpawnToSettle();
2000
2134
  let windowId = this.getWindowId();
2001
2135
  const supportsQueuedInput = this.backend?.supportsQueuedInput?.() === true;
2002
2136
  let handingOffToNativeQueue = false;
@@ -2024,6 +2158,21 @@ export class Daemon extends EventEmitter {
2024
2158
  }
2025
2159
  }
2026
2160
  }
2161
+ // Everything above is *waiting*; everything below *writes*. Only the write is
2162
+ // held under the pane lock — holding it across the idle wait (up to 30 min)
2163
+ // would starve the runtime-dialog dismisser, which is often the very thing
2164
+ // that would let the pane go idle again.
2165
+ return this.paneWriteLock.run(() => this.writeMessageToPane(formatted, windowId, handingOffToNativeQueue, status));
2166
+ }
2167
+ /**
2168
+ * The write half of a delivery: paste → Enter → confirm, with retries.
2169
+ *
2170
+ * Always called under {@link paneWriteLock}. Split out from `deliverMessage`
2171
+ * precisely so the lock's scope is visible at the call site rather than being
2172
+ * an invariant maintained by comments.
2173
+ */
2174
+ async writeMessageToPane(formatted, initialWindowId, handingOffToNativeQueue, status) {
2175
+ let windowId = initialWindowId;
2027
2176
  // Bug A: paste with backoff. Transient failures are usually a stale window id
2028
2177
  // after a crash/respawn — recover by name and retry (max 3 attempts, 2s apart).
2029
2178
  const maxAttempts = 3;
@@ -2164,6 +2313,7 @@ export class Daemon extends EventEmitter {
2164
2313
  }
2165
2314
  /** Re-resolve this instance's tmux window by name (stale id after crash/respawn). */
2166
2315
  async recoverWindow() {
2316
+ const previousWindowId = this.tmux?.getWindowId();
2167
2317
  try {
2168
2318
  const windows = await TmuxManager.listWindows(this.tmuxSessionName);
2169
2319
  const match = windows.find(w => w.name === this.name);
@@ -2171,6 +2321,12 @@ export class Daemon extends EventEmitter {
2171
2321
  return undefined;
2172
2322
  this.tmux = new TmuxManager(this.tmuxSessionName, match.id, resolveTmuxLogicalSize(this.config.terminal));
2173
2323
  writeFileSync(join(this.instanceDir, "window-id"), match.id);
2324
+ // The window we were talking to is gone; leaving it registered means the
2325
+ // control client re-resolves a dead id — one tmux subprocess — on every
2326
+ // reconnect, for the life of the fleet process.
2327
+ if (previousWindowId && previousWindowId !== match.id) {
2328
+ this.controlClient?.unregisterWindow(previousWindowId);
2329
+ }
2174
2330
  await this.controlClient?.registerWindow(match.id);
2175
2331
  this.bindInstanceStateOutputListener(match.id);
2176
2332
  this.logger.info({ windowId: match.id }, "Recovered window ID for message delivery");
@@ -2181,12 +2337,38 @@ export class Daemon extends EventEmitter {
2181
2337
  return undefined;
2182
2338
  }
2183
2339
  }
2184
- /** Poll up to ~2s (200ms × 10) for the pane to emit output after `since`. */
2340
+ /**
2341
+ * Poll up to ~2s (200ms × 10) for the pane to emit output after `since`.
2342
+ *
2343
+ * A `false` here is not "no output" — it is read as **"the message was pasted but
2344
+ * never submitted"**, which re-sends Enter and then reports ❌ to the user. So a
2345
+ * false negative costs a possible double submit and a failure notice for a
2346
+ * message that actually arrived.
2347
+ *
2348
+ * That is exactly what a control-mode reconnect used to produce. `connect()`
2349
+ * drops every output timestamp, so `hasOutputSince` answers `false` for a pane
2350
+ * that did react — the evidence was thrown away, not absent. Blind time is
2351
+ * therefore not counted against the budget: after a reset the poll restarts from
2352
+ * the moment observation resumed. A hard wall-clock cap keeps a reconnect loop
2353
+ * from extending this forever.
2354
+ */
2185
2355
  async confirmBusyAfterEnter(windowId, since) {
2186
- for (let i = 0; i < 10; i++) {
2187
- await new Promise(r => setTimeout(r, 200));
2188
- if (this.controlClient.hasOutputSince(windowId, since))
2356
+ const client = this.controlClient;
2357
+ const hardDeadline = Date.now() + CONFIRM_BUSY_MAX_WAIT_MS;
2358
+ let observedFrom = since;
2359
+ let polls = 0;
2360
+ while (polls < CONFIRM_BUSY_POLLS) {
2361
+ await new Promise(r => setTimeout(r, CONFIRM_BUSY_POLL_MS));
2362
+ if (client.hasOutputSince(windowId, observedFrom))
2189
2363
  return true;
2364
+ const resetAt = client.getObservationResetAt();
2365
+ if (resetAt > observedFrom && Date.now() < hardDeadline) {
2366
+ this.logger.debug({ windowId, resetAt }, "Control mode reconnected mid-confirmation — restarting the Enter check from when we could see again");
2367
+ observedFrom = resetAt;
2368
+ polls = 0;
2369
+ continue;
2370
+ }
2371
+ polls++;
2190
2372
  }
2191
2373
  return false;
2192
2374
  }
@@ -2564,7 +2746,9 @@ export class Daemon extends EventEmitter {
2564
2746
  // Small delay to let the CLI fully render its ready prompt
2565
2747
  await new Promise(r => setTimeout(r, 1_000));
2566
2748
  try {
2567
- await this.tmux.pasteText(`[system:session-snapshot]\n${snapshot}\n\nThis is a background context restore — do NOT reply to or acknowledge this message. Simply resume normal operation when the next user or instance message arrives.`);
2749
+ // Messages can arrive during a restart and be queued on pasteLock before the
2750
+ // snapshot lands; both write to the pane, so both go through the same lock.
2751
+ await this.paneWriteLock.run(() => this.tmux.pasteText(`[system:session-snapshot]\n${snapshot}\n\nThis is a background context restore — do NOT reply to or acknowledge this message. Simply resume normal operation when the next user or instance message arrives.`));
2568
2752
  this.logger.info("Injected session snapshot as first message");
2569
2753
  this.emit("snapshot_injected", this.name);
2570
2754
  }
@@ -2573,9 +2757,69 @@ export class Daemon extends EventEmitter {
2573
2757
  this.emit("snapshot_failed", this.name);
2574
2758
  }
2575
2759
  }
2760
+ /**
2761
+ * Depth, not a flag: the wake path marks a spawn and then calls trySpawn, which
2762
+ * marks another. With a plain boolean the inner one's completion would clear the
2763
+ * latch while the outer spawn was still dismissing dialogs, releasing delivery
2764
+ * into exactly the window this exists to close.
2765
+ */
2766
+ beginSpawn() {
2767
+ this.spawnDepth++;
2768
+ this.spawning = true;
2769
+ if (!this.spawnSettled) {
2770
+ this.spawnSettled = new Promise(resolve => { this.resolveSpawnSettled = resolve; });
2771
+ }
2772
+ }
2773
+ endSpawn() {
2774
+ this.spawnDepth = Math.max(0, this.spawnDepth - 1);
2775
+ if (this.spawnDepth > 0)
2776
+ return; // an enclosing spawn is still running
2777
+ this.spawning = false;
2778
+ const resolve = this.resolveSpawnSettled;
2779
+ this.spawnSettled = null;
2780
+ this.resolveSpawnSettled = null;
2781
+ resolve?.();
2782
+ }
2783
+ /**
2784
+ * Hold until the CLI has finished starting up.
2785
+ *
2786
+ * Startup is not a quiet period: `dismissDialogsUntilReady` is clicking through
2787
+ * trust prompts and session pickers. A pane showing a modal dialog produces no
2788
+ * output, so `waitUntilIdle` reports it idle and a queued message gets pasted
2789
+ * *into the dialog* — where the text is discarded and the Enter picks a menu
2790
+ * item. The paste and the Enter both "succeed", so this loses the message
2791
+ * without even a ❌.
2792
+ *
2793
+ * The pane write lock (#414) does not cover this: it serialises each key
2794
+ * sequence, but it is released between one dialog and the next.
2795
+ *
2796
+ * Bounded, and called BEFORE the pane lock is taken — waiting on the spawn while
2797
+ * holding the lock the spawn itself needs would deadlock.
2798
+ */
2799
+ async waitForSpawnToSettle() {
2800
+ const settled = this.spawnSettled;
2801
+ if (!settled)
2802
+ return;
2803
+ this.logger.debug("Holding delivery until the CLI has finished starting up");
2804
+ let timer;
2805
+ const cap = new Promise(resolve => {
2806
+ timer = setTimeout(resolve, SPAWN_SETTLE_MAX_WAIT_MS);
2807
+ timer.unref?.();
2808
+ });
2809
+ try {
2810
+ await Promise.race([settled, cap]);
2811
+ }
2812
+ finally {
2813
+ if (timer)
2814
+ clearTimeout(timer);
2815
+ }
2816
+ if (this.spawning) {
2817
+ this.logger.warn("CLI still starting after the delivery hold — delivering anyway");
2818
+ }
2819
+ }
2576
2820
  /** Spawn a CLI window. Returns true if --resume was used successfully. */
2577
2821
  async spawnClaudeWindow() {
2578
- this.spawning = true;
2822
+ this.beginSpawn();
2579
2823
  let resumedSuccessfully = false;
2580
2824
  try {
2581
2825
  this.toolStatusLines = [];
@@ -2612,7 +2856,7 @@ export class Daemon extends EventEmitter {
2612
2856
  this.backgroundSessionRecoveryAttempted = false;
2613
2857
  }
2614
2858
  finally {
2615
- this.spawning = false;
2859
+ this.endSpawn();
2616
2860
  }
2617
2861
  return resumedSuccessfully;
2618
2862
  }
@@ -2705,7 +2949,12 @@ export class Daemon extends EventEmitter {
2705
2949
  windowId = this.tmux.getWindowId();
2706
2950
  }
2707
2951
  else {
2952
+ // A crash respawn makes a brand new window. The dead one stays registered
2953
+ // unless we say so — the respawn branch above already does.
2954
+ const retired = this.tmux.getWindowId();
2708
2955
  windowId = await this.tmux.createWindow(cmd, resolvedCwd, this.name);
2956
+ if (retired && retired !== windowId)
2957
+ this.controlClient?.unregisterWindow(retired);
2709
2958
  }
2710
2959
  writeFileSync(join(this.instanceDir, "window-id"), windowId);
2711
2960
  // Enable remain-on-exit to capture exit codes on crash
@@ -2771,15 +3020,20 @@ export class Daemon extends EventEmitter {
2771
3020
  for (const dialog of startupDialogs) {
2772
3021
  if (dialog.pattern.test(pane)) {
2773
3022
  this.logger.debug(`Dismissing startup dialog: ${dialog.description}`);
2774
- for (const key of dialog.keys) {
2775
- if (key === "Up" || key === "Down" || key === "Enter" || key === "Escape") {
2776
- await this.tmux.sendSpecialKey(key);
2777
- }
2778
- else {
2779
- await this.tmux.sendKeys(key);
3023
+ // Restart is exactly when inbound messages pile up, and nothing gates
3024
+ // delivery on `spawning`. Take the pane lock for the key sequence so a
3025
+ // queued message cannot be pasted into a half-dismissed trust dialog.
3026
+ await this.paneWriteLock.run(async () => {
3027
+ for (const key of dialog.keys) {
3028
+ if (key === "Up" || key === "Down" || key === "Enter" || key === "Escape") {
3029
+ await this.tmux.sendSpecialKey(key);
3030
+ }
3031
+ else {
3032
+ await this.tmux.sendKeys(key);
3033
+ }
3034
+ await new Promise(r => setTimeout(r, 200));
2780
3035
  }
2781
- await new Promise(r => setTimeout(r, 200));
2782
- }
3036
+ });
2783
3037
  // Wait for next screen to render
2784
3038
  if (this.controlClient) {
2785
3039
  const wid = readFileSync(join(this.instanceDir, "window-id"), "utf-8").trim();