@songsid/agend 2.1.1-beta.13 → 2.1.1-beta.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/daemon.js CHANGED
@@ -42,6 +42,8 @@ export const DEFAULT_STATE_SAFETY_SWEEP_MS = 60_000;
42
42
  /** @deprecated State detection is event-driven; this now aliases the safety sweep. */
43
43
  export const DEFAULT_STATE_POLL_INTERVAL_MS = DEFAULT_STATE_SAFETY_SWEEP_MS;
44
44
  const LAST_INBOUND_FILE = "last-inbound-at";
45
+ /** Minimum gap between "health check is failing" notifications for one instance. */
46
+ const HEALTH_ERROR_NOTIFY_INTERVAL_MS = 10 * 60_000;
45
47
  /**
46
48
  * Whether two working directories belong to the same project, so a fleet-scoped
47
49
  * decision recorded in one reaches the other. Covers the worktree/checkout
@@ -105,6 +107,40 @@ export function writeLastInboundAt(instanceDir, timestamp) {
105
107
  writeFileSync(temp, String(timestamp));
106
108
  renameSync(temp, target);
107
109
  }
110
+ /**
111
+ * Render the handoff/routing metadata that rides along with an inbound message
112
+ * into the block actually pasted into the agent's pane.
113
+ *
114
+ * These fields were all populated by the sender (outbound-handlers builds them
115
+ * into `ipcMeta`) and delivered over IPC, but never rendered — so the receiving
116
+ * agent could not see them, and several documented flows could not work:
117
+ *
118
+ * - `report_result` requires a `correlation_id` the agent had no way to know, so
119
+ * the "correlation_id not recognized" warning fired on essentially every call.
120
+ * - Delegation cancel buttons are retired by correlation id, so they never retired.
121
+ * - `react`, `edit_message` and `reply.reply_to` need `message_id`, which the tool
122
+ * descriptions tell the agent to take "from the inbound block".
123
+ * - `download_attachment` needs `attachment_file_id`.
124
+ * - `requires_reply` was invisible, so a delegated task looked like an FYI.
125
+ *
126
+ * Only non-empty fields are emitted, so a plain user message gains at most a
127
+ * message_id line.
128
+ */
129
+ export function renderHandoffMetadata(meta) {
130
+ const rows = [];
131
+ const add = (label, value) => {
132
+ if (value && value.trim())
133
+ rows.push(`${label}: ${value.trim()}`);
134
+ };
135
+ add("message_id", meta.message_id);
136
+ add("correlation_id", meta.correlation_id);
137
+ add("request_kind", meta.request_kind);
138
+ add("task_summary", meta.task_summary);
139
+ add("working_directory", meta.working_directory);
140
+ add("branch", meta.branch);
141
+ add("attachment_file_id", meta.attachment_file_id);
142
+ return rows.length ? `\n(${rows.join(" | ")})` : "";
143
+ }
108
144
  /** Headless inactivity timer used by the daemon and unit tests. */
109
145
  export class AutoPauseController {
110
146
  thresholdMs;
@@ -326,6 +362,7 @@ export class Daemon extends EventEmitter {
326
362
  lastSpawnAt = 0;
327
363
  crashTimestamps = [];
328
364
  healthCheckPaused = false;
365
+ lastHealthErrorNotifyAt = 0;
329
366
  /** CLI pane availability, independent from the daemon process and tri-state. */
330
367
  processStatus = "running";
331
368
  spawning = false;
@@ -797,256 +834,280 @@ export class Daemon extends EventEmitter {
797
834
  this.healthCheckTimer = null;
798
835
  if (this.runtimeMonitorsFrozen)
799
836
  return;
800
- // Instance directory removed externally (e.g. `rm -rf ~/.agend/instances/<name>`).
801
- // Stop the loop permanently otherwise every tick triggers a respawn, whose
802
- // writeRotationSnapshot fails with ENOENT and gets caught as "Failed to respawn",
803
- // spamming errors every ~30s forever.
804
- if (!existsSync(this.instanceDir)) {
805
- this.logger.warn({ instanceDir: this.instanceDir }, "Instance directory missing — stopping health check");
806
- this.healthCheckPaused = true;
807
- this.healthCheckTimer = null;
808
- return;
809
- }
810
- if (!this.tmux || this.spawning || this.healthCheckPaused || Daemon.tmuxServerPaused) {
811
- scheduleNext();
812
- return;
813
- }
814
- // The CLI owns the MCP server process, so the daemon can only observe it.
815
- this.checkMcpServerAlive();
816
- // Human-readable backend label for logs (e.g. "claude", "kiro-cli")
817
- const cliLabel = this.backend?.binaryName ?? "CLI";
818
- let paneStatus = await this.tmux.getPaneStatus();
819
- // Auto-pause intentionally exits the pane process. A health tick that
820
- // began just before pause must not classify that exit as a crash.
821
- if (this.isPaused || this.pauseWakeState === "waking") {
822
- scheduleNext();
823
- return;
824
- }
825
- if (paneStatus?.alive) {
826
- // Instance output.log is fed by tmux pipe-pane and was previously never
827
- // rotated (only fleet.log / daemon.log were). Cap growth every tick.
828
- if (!this.config.lightweight) {
829
- rotateLogIfNeeded(join(this.instanceDir, "output.log"));
837
+ // The whole tick is guarded: an unguarded throw in here (a tmux hiccup,
838
+ // ENOSPC on the crash-history write) became an unhandled rejection, which
839
+ // the CLI turned into stopAll() + exit(1) for the ENTIRE fleet, and it also
840
+ // ended this instance's health loop because scheduleNext() sat after the
841
+ // throwing code. Catch, log, and keep checking.
842
+ try {
843
+ // Instance directory removed externally (e.g. `rm -rf ~/.agend/instances/<name>`).
844
+ // Stop the loop permanently — otherwise every tick triggers a respawn, whose
845
+ // writeRotationSnapshot fails with ENOENT and gets caught as "Failed to respawn",
846
+ // spamming errors every ~30s forever.
847
+ if (!existsSync(this.instanceDir)) {
848
+ this.logger.warn({ instanceDir: this.instanceDir }, "Instance directory missing — stopping health check");
849
+ this.healthCheckPaused = true;
850
+ this.healthCheckTimer = null;
851
+ return;
852
+ }
853
+ if (!this.tmux || this.spawning || this.healthCheckPaused || Daemon.tmuxServerPaused) {
854
+ scheduleNext();
855
+ return;
856
+ }
857
+ // The CLI owns the MCP server process, so the daemon can only observe it.
858
+ this.checkMcpServerAlive();
859
+ // Human-readable backend label for logs (e.g. "claude", "kiro-cli")
860
+ const cliLabel = this.backend?.binaryName ?? "CLI";
861
+ let paneStatus = await this.tmux.getPaneStatus();
862
+ // Auto-pause intentionally exits the pane process. A health tick that
863
+ // began just before pause must not classify that exit as a crash.
864
+ if (this.isPaused || this.pauseWakeState === "waking") {
865
+ scheduleNext();
866
+ return;
830
867
  }
831
- scheduleNext();
832
- return;
833
- }
834
- // A null status is ambiguous: it can be a transient `tmux list-panes`
835
- // failure (e.g. tmux busy during a fleet-restart storm) rather than a
836
- // real exit. Re-confirm once after a short delay before treating it as
837
- // a crash. A non-null {alive:false} is a definite dead pane (real exit)
838
- // and needs no recheck.
839
- if (paneStatus === null) {
840
- await new Promise(r => setTimeout(r, 1500));
841
- paneStatus = await this.tmux.getPaneStatus();
842
868
  if (paneStatus?.alive) {
843
- this.logger.debug(`[health] ${cliLabel} pane reported gone then alive on recheck transient query failure, ignoring`);
869
+ // Instance output.log is fed by tmux pipe-pane and was previously never
870
+ // rotated (only fleet.log / daemon.log were). Cap growth every tick.
871
+ if (!this.config.lightweight) {
872
+ rotateLogIfNeeded(join(this.instanceDir, "output.log"));
873
+ }
844
874
  scheduleNext();
845
875
  return;
846
876
  }
847
- }
848
- // paneStatus === null → window gone entirely (e.g. tmux server crash)
849
- // paneStatus.alive === false pane dead, exit code available
850
- const exitCode = paneStatus?.exitCode;
851
- this.logger.debug({ exitCode }, `[health] pane exited with code: ${exitCode}`);
852
- // Normal exit (e.g. user Ctrl+C or /exit) — no crash, no respawn
853
- if (paneStatus && exitCode === 0) {
854
- this.setProcessStatus("stopped");
855
- this.logger.info("CLI exited normally (code 0) — pausing health check");
856
- await this.tmux.killWindow();
857
- this.healthCheckPaused = true;
858
- return;
859
- }
860
- this.setProcessStatus("crashed");
861
- // Distinguish tmux server crash from single window crash.
862
- // nullReason records *why* getPaneStatus returned null (for diagnosing
863
- // whether this was a real window loss or a transient query failure).
864
- let crashType = "window";
865
- let nullReason;
866
- if (!paneStatus) {
867
- const serverAlive = await TmuxManager.sessionExists(this.tmuxSessionName);
868
- if (!serverAlive) {
869
- crashType = "server";
870
- nullReason = "server_gone";
871
- this.logger.error(`tmux server died — all ${cliLabel} windows lost`);
872
- // Fleet-level circuit breaker: pause all instances on repeated tmux server crashes
873
- Daemon.tmuxServerCrashTimestamps.push(Date.now());
874
- const cutoff = Date.now() - 5 * 60_000;
875
- Daemon.tmuxServerCrashTimestamps = Daemon.tmuxServerCrashTimestamps.filter(t => t > cutoff);
876
- if (Daemon.tmuxServerCrashTimestamps.length >= 2 && !Daemon.tmuxServerPaused) {
877
- Daemon.tmuxServerPaused = true;
878
- this.logger.error("Fleet-level tmux server circuit breaker triggered — pausing all respawns for 30s");
879
- this.emit("tmux_server_crash", this.name);
880
- if (!Daemon.tmuxServerRecoveryTimer) {
881
- Daemon.tmuxServerRecoveryTimer = setTimeout(() => {
882
- Daemon.tmuxServerRecoveryTimer = null;
883
- Daemon.tmuxServerPaused = false;
884
- }, 30_000);
885
- }
877
+ // A null status is ambiguous: it can be a transient `tmux list-panes`
878
+ // failure (e.g. tmux busy during a fleet-restart storm) rather than a
879
+ // real exit. Re-confirm once after a short delay before treating it as
880
+ // a crash. A non-null {alive:false} is a definite dead pane (real exit)
881
+ // and needs no recheck.
882
+ if (paneStatus === null) {
883
+ await new Promise(r => setTimeout(r, 1500));
884
+ paneStatus = await this.tmux.getPaneStatus();
885
+ if (paneStatus?.alive) {
886
+ this.logger.debug(`[health] ${cliLabel} pane reported gone then alive on recheck — transient query failure, ignoring`);
886
887
  scheduleNext();
887
888
  return;
888
889
  }
889
- await new Promise(r => setTimeout(r, 2_000)); // let session stabilize
890
890
  }
891
- else {
892
- // null but server alive: window-level disappearance. Probe whether
893
- // the window truly no longer exists vs a transient query glitch.
894
- nullReason = "no_window";
895
- try {
896
- const windows = await TmuxManager.listWindows(this.tmuxSessionName);
897
- if (windows.some(w => w.name === this.name))
898
- nullReason = "window_present_query_glitch";
891
+ // paneStatus === null → window gone entirely (e.g. tmux server crash)
892
+ // paneStatus.alive === false pane dead, exit code available
893
+ const exitCode = paneStatus?.exitCode;
894
+ this.logger.debug({ exitCode }, `[health] pane exited with code: ${exitCode}`);
895
+ // Normal exit (e.g. user Ctrl+C or /exit) — no crash, no respawn
896
+ if (paneStatus && exitCode === 0) {
897
+ this.setProcessStatus("stopped");
898
+ this.logger.info("CLI exited normally (code 0) — pausing health check");
899
+ await this.tmux.killWindow();
900
+ this.healthCheckPaused = true;
901
+ return;
902
+ }
903
+ this.setProcessStatus("crashed");
904
+ // Distinguish tmux server crash from single window crash.
905
+ // nullReason records *why* getPaneStatus returned null (for diagnosing
906
+ // whether this was a real window loss or a transient query failure).
907
+ let crashType = "window";
908
+ let nullReason;
909
+ if (!paneStatus) {
910
+ const serverAlive = await TmuxManager.sessionExists(this.tmuxSessionName);
911
+ if (!serverAlive) {
912
+ crashType = "server";
913
+ nullReason = "server_gone";
914
+ this.logger.error(`tmux server died — all ${cliLabel} windows lost`);
915
+ // Fleet-level circuit breaker: pause all instances on repeated tmux server crashes
916
+ Daemon.tmuxServerCrashTimestamps.push(Date.now());
917
+ const cutoff = Date.now() - 5 * 60_000;
918
+ Daemon.tmuxServerCrashTimestamps = Daemon.tmuxServerCrashTimestamps.filter(t => t > cutoff);
919
+ if (Daemon.tmuxServerCrashTimestamps.length >= 2 && !Daemon.tmuxServerPaused) {
920
+ Daemon.tmuxServerPaused = true;
921
+ this.logger.error("Fleet-level tmux server circuit breaker triggered — pausing all respawns for 30s");
922
+ this.emit("tmux_server_crash", this.name);
923
+ if (!Daemon.tmuxServerRecoveryTimer) {
924
+ Daemon.tmuxServerRecoveryTimer = setTimeout(() => {
925
+ Daemon.tmuxServerRecoveryTimer = null;
926
+ Daemon.tmuxServerPaused = false;
927
+ }, 30_000);
928
+ }
929
+ scheduleNext();
930
+ return;
931
+ }
932
+ await new Promise(r => setTimeout(r, 2_000)); // let session stabilize
899
933
  }
900
- catch {
901
- nullReason = "query_error";
934
+ else {
935
+ // null but server alive: window-level disappearance. Probe whether
936
+ // the window truly no longer exists vs a transient query glitch.
937
+ nullReason = "no_window";
938
+ try {
939
+ const windows = await TmuxManager.listWindows(this.tmuxSessionName);
940
+ if (windows.some(w => w.name === this.name))
941
+ nullReason = "window_present_query_glitch";
942
+ }
943
+ catch {
944
+ nullReason = "query_error";
945
+ }
946
+ this.logger.warn({ exitCode, nullReason }, `${cliLabel} window not found (tmux server alive)`);
902
947
  }
903
- this.logger.warn({ exitCode, nullReason }, `${cliLabel} window not found (tmux server alive)`);
904
948
  }
905
- }
906
- else {
907
- this.logger.warn({ exitCode }, `${cliLabel} process exited`);
908
- }
909
- // Capture last output before killing. Best-effort even when the pane is
910
- // gone (paneStatus null) gives the crash record something to diagnose
911
- // from instead of an empty lastOutput.
912
- let lastOutput;
913
- try {
914
- const raw = await this.tmux.capturePaneWithHistory(50);
915
- // Strip ANSI escape codes for readability
916
- const cleaned = raw.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
917
- lastOutput = cleaned.trimEnd() || undefined;
918
- }
919
- catch { /* best effort pane may already be gone */ }
920
- // Kill the dead window (remain-on-exit keeps it around) before respawn
921
- if (paneStatus) {
922
- await this.tmux.killWindow();
923
- }
924
- // Detect claude-code background session conflict recover without counting as crash
925
- if (lastOutput && (lastOutput.includes("background agent") || lastOutput.includes("Session is currently running"))) {
926
- if (!this.backgroundSessionRecoveryAttempted) {
927
- this.backgroundSessionRecoveryAttempted = true;
928
- this.logger.warn("Detected lingering background agent session — starting fresh (no resume)");
929
- const sidFile = join(this.instanceDir, "session-id");
949
+ else {
950
+ this.logger.warn({ exitCode }, `${cliLabel} process exited`);
951
+ }
952
+ // Capture last output before killing. Best-effort even when the pane is
953
+ // gone (paneStatus null) gives the crash record something to diagnose
954
+ // from instead of an empty lastOutput.
955
+ let lastOutput;
956
+ try {
957
+ const raw = await this.tmux.capturePaneWithHistory(50);
958
+ // Strip ANSI escape codes for readability
959
+ const cleaned = raw.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
960
+ lastOutput = cleaned.trimEnd() || undefined;
961
+ }
962
+ catch { /* best effort — pane may already be gone */ }
963
+ // Kill the dead window (remain-on-exit keeps it around) before respawn
964
+ if (paneStatus) {
965
+ await this.tmux.killWindow();
966
+ }
967
+ // Detect claude-code background session conflict — recover without counting as crash
968
+ if (lastOutput && (lastOutput.includes("background agent") || lastOutput.includes("Session is currently running"))) {
969
+ if (!this.backgroundSessionRecoveryAttempted) {
970
+ this.backgroundSessionRecoveryAttempted = true;
971
+ this.logger.warn("Detected lingering background agent session — starting fresh (no resume)");
972
+ const sidFile = join(this.instanceDir, "session-id");
973
+ try {
974
+ unlinkSync(sidFile);
975
+ }
976
+ catch { }
977
+ this.skipResume = true;
978
+ await new Promise(r => setTimeout(r, 2_000));
979
+ try {
980
+ await this.spawnClaudeWindow();
981
+ this.setProcessStatus("running");
982
+ this.logger.info("Recovered from background session conflict");
983
+ this.emit("crash_respawn", this.name);
984
+ }
985
+ catch (err) {
986
+ this.logger.error({ err: err.message }, "Recovery from background session conflict failed");
987
+ }
988
+ return; // Don't count as crash
989
+ }
990
+ // Already attempted recovery — fall through to normal crash handling
991
+ }
992
+ // Detect a --continue/--resume failure (no conversation to resume). The
993
+ // session-id file persists across the crash, so a blind respawn would add
994
+ // --continue again and crash in the same way → loop. Clear the session id
995
+ // and skip resume so the next spawn starts fresh. (skipResume also stops
996
+ // saveSessionId below from resurrecting the id from statusline.json.)
997
+ if (lastOutput && /no conversation found|no conversation to (continue|resume)|no previous (session|conversation)|--continue/i.test(lastOutput)) {
998
+ this.logger.warn("Detected --continue/resume failure — clearing session-id; next spawn starts fresh");
930
999
  try {
931
- unlinkSync(sidFile);
1000
+ unlinkSync(join(this.instanceDir, "session-id"));
932
1001
  }
933
- catch { }
1002
+ catch { /* may not exist */ }
934
1003
  this.skipResume = true;
935
- await new Promise(r => setTimeout(r, 2_000));
1004
+ }
1005
+ // Append to crash history
1006
+ this.appendCrashHistory({ exitCode, lastOutput, crashType, reason: nullReason });
1007
+ if (max_retries <= 0) {
1008
+ this.healthCheckPaused = true;
1009
+ this.logger.warn(`${cliLabel} window died — automatic restart is disabled`);
1010
+ return;
1011
+ }
1012
+ // Detect rapid crash: sliding window — 3+ crashes in 5 minutes
1013
+ this.crashTimestamps.push(Date.now());
1014
+ const crashWindowMs = 5 * 60_000;
1015
+ this.crashTimestamps = this.crashTimestamps.filter(t => t > Date.now() - crashWindowMs);
1016
+ if (this.crashTimestamps.length >= 3) {
1017
+ this.healthCheckPaused = true;
1018
+ this.logger.error({ crashesInWindow: this.crashTimestamps.length }, "3+ crashes in 5 minutes — pausing respawn");
1019
+ // P1: Persist crash state so next process restart skips resume
936
1020
  try {
937
- await this.spawnClaudeWindow();
938
- this.setProcessStatus("running");
939
- this.logger.info("Recovered from background session conflict");
940
- this.emit("crash_respawn", this.name);
941
- }
942
- catch (err) {
943
- this.logger.error({ err: err.message }, "Recovery from background session conflict failed");
1021
+ writeFileSync(join(this.instanceDir, "crash-state.json"), JSON.stringify({
1022
+ crashesInWindow: this.crashTimestamps.length,
1023
+ lastCrashAt: Date.now(),
1024
+ resumeDisabled: true,
1025
+ }));
944
1026
  }
945
- return; // Don't count as crash
946
- }
947
- // Already attempted recoveryfall through to normal crash handling
948
- }
949
- // Detect a --continue/--resume failure (no conversation to resume). The
950
- // session-id file persists across the crash, so a blind respawn would add
951
- // --continue again and crash in the same way → loop. Clear the session id
952
- // and skip resume so the next spawn starts fresh. (skipResume also stops
953
- // saveSessionId below from resurrecting the id from statusline.json.)
954
- if (lastOutput && /no conversation found|no conversation to (continue|resume)|no previous (session|conversation)|--continue/i.test(lastOutput)) {
955
- this.logger.warn("Detected --continue/resume failure — clearing session-id; next spawn starts fresh");
956
- try {
957
- unlinkSync(join(this.instanceDir, "session-id"));
1027
+ catch { /* best effort */ }
1028
+ this.emit("crash_loop", this.name);
1029
+ return; // don't schedule nextpaused
958
1030
  }
959
- catch { /* may not exist */ }
960
- this.skipResume = true;
961
- }
962
- // Append to crash history
963
- this.appendCrashHistory({ exitCode, lastOutput, crashType, reason: nullReason });
964
- if (max_retries <= 0) {
965
- this.healthCheckPaused = true;
966
- this.logger.warn(`${cliLabel} window died — automatic restart is disabled`);
967
- return;
968
- }
969
- // Detect rapid crash: sliding window — 3+ crashes in 5 minutes
970
- this.crashTimestamps.push(Date.now());
971
- const crashWindowMs = 5 * 60_000;
972
- this.crashTimestamps = this.crashTimestamps.filter(t => t > Date.now() - crashWindowMs);
973
- if (this.crashTimestamps.length >= 3) {
974
- this.healthCheckPaused = true;
975
- this.logger.error({ crashesInWindow: this.crashTimestamps.length }, "3+ crashes in 5 minutes — pausing respawn");
976
- // P1: Persist crash state so next process restart skips resume
977
- try {
978
- writeFileSync(join(this.instanceDir, "crash-state.json"), JSON.stringify({
979
- crashesInWindow: this.crashTimestamps.length,
980
- lastCrashAt: Date.now(),
981
- resumeDisabled: true,
982
- }));
1031
+ // Reset crash count if enough time has passed
1032
+ if (reset_after > 0 && Date.now() - this.lastCrashAt > reset_after) {
1033
+ this.crashCount = 0;
983
1034
  }
984
- catch { /* best effort */ }
985
- this.emit("crash_loop", this.name);
986
- return; // don't schedule next — paused
987
- }
988
- // Reset crash count if enough time has passed
989
- if (reset_after > 0 && Date.now() - this.lastCrashAt > reset_after) {
990
- this.crashCount = 0;
991
- }
992
- this.crashCount++;
993
- this.lastCrashAt = Date.now();
994
- if (this.crashCount > max_retries) {
995
- this.logger.error({ crashCount: this.crashCount, maxRetries: max_retries }, "Max crash retries exceeded — not respawning");
996
- return; // don't schedule next — given up
997
- }
998
- // Calculate backoff delay
999
- const delay = backoff === "exponential"
1000
- ? Math.min(1000 * Math.pow(2, this.crashCount - 1), 60_000)
1001
- : 1000 * this.crashCount;
1002
- this.logger.warn({ crashCount: this.crashCount, delay }, `${cliLabel} window died — respawning after backoff`);
1003
- await new Promise(r => setTimeout(r, delay));
1004
- try {
1005
- this.saveSessionId();
1006
- this.transcriptMonitor?.resetOffset();
1007
- // Kill orphan MCP server from the crashed CLI session.
1008
- // MCP server writes its PID to channel.mcp.pid on startup.
1009
- try {
1010
- const pidFile = join(this.instanceDir, "channel.mcp.pid");
1011
- const pid = parseInt(readFileSync(pidFile, "utf-8").trim(), 10);
1012
- process.kill(pid, "SIGTERM");
1013
- this.logger.info({ pid }, "Killed orphan MCP server");
1035
+ this.crashCount++;
1036
+ this.lastCrashAt = Date.now();
1037
+ if (this.crashCount > max_retries) {
1038
+ this.logger.error({ crashCount: this.crashCount, maxRetries: max_retries }, "Max crash retries exceeded — not respawning");
1039
+ return; // don't schedule next given up
1014
1040
  }
1015
- catch { /* no pid file or process already dead */ }
1016
- // Kill any same-name windows before respawn to prevent orphans.
1017
- // Wrapped in try-catch: if tmux server is dead, listWindows throws —
1018
- // must not block spawnClaudeWindow (which calls ensureSession).
1041
+ // Calculate backoff delay
1042
+ const delay = backoff === "exponential"
1043
+ ? Math.min(1000 * Math.pow(2, this.crashCount - 1), 60_000)
1044
+ : 1000 * this.crashCount;
1045
+ this.logger.warn({ crashCount: this.crashCount, delay }, `${cliLabel} window died — respawning after backoff`);
1046
+ await new Promise(r => setTimeout(r, delay));
1019
1047
  try {
1020
- const windows = await TmuxManager.listWindows(this.tmuxSessionName);
1021
- for (const w of windows) {
1022
- if (w.name === this.name) {
1023
- const tm = new TmuxManager(this.tmuxSessionName, w.id);
1024
- await tm.killWindow();
1025
- }
1048
+ this.saveSessionId();
1049
+ this.transcriptMonitor?.resetOffset();
1050
+ // Kill orphan MCP server from the crashed CLI session.
1051
+ // MCP server writes its PID to channel.mcp.pid on startup.
1052
+ try {
1053
+ const pidFile = join(this.instanceDir, "channel.mcp.pid");
1054
+ const pid = parseInt(readFileSync(pidFile, "utf-8").trim(), 10);
1055
+ process.kill(pid, "SIGTERM");
1056
+ this.logger.info({ pid }, "Killed orphan MCP server");
1026
1057
  }
1027
- }
1028
- catch { /* tmux server may be dead ensureSession in trySpawn will recover */ }
1029
- // Write snapshot before spawn consumed only if resume fails
1030
- this.writeRotationSnapshot("crash");
1031
- // Try --resume first; spawnClaudeWindow falls back to fresh session if resume fails
1032
- const resumed = await this.spawnClaudeWindow();
1033
- if (!resumed) {
1034
- // Resume failed → fresh session → inject snapshot for context
1035
- await this.injectSnapshotMessage();
1036
- }
1037
- else {
1038
- // Clean up stale snapshot — resume restored full context
1058
+ catch { /* no pid file or process already dead */ }
1059
+ // Kill any same-name windows before respawn to prevent orphans.
1060
+ // Wrapped in try-catch: if tmux server is dead, listWindows throws —
1061
+ // must not block spawnClaudeWindow (which calls ensureSession).
1039
1062
  try {
1040
- unlinkSync(join(this.instanceDir, "rotation-state.json"));
1063
+ const windows = await TmuxManager.listWindows(this.tmuxSessionName);
1064
+ for (const w of windows) {
1065
+ if (w.name === this.name) {
1066
+ const tm = new TmuxManager(this.tmuxSessionName, w.id);
1067
+ await tm.killWindow();
1068
+ }
1069
+ }
1041
1070
  }
1042
- catch { /* may not exist */ }
1071
+ catch { /* tmux server may be dead — ensureSession in trySpawn will recover */ }
1072
+ // Write snapshot before spawn — consumed only if resume fails
1073
+ this.writeRotationSnapshot("crash");
1074
+ // Try --resume first; spawnClaudeWindow falls back to fresh session if resume fails
1075
+ const resumed = await this.spawnClaudeWindow();
1076
+ if (!resumed) {
1077
+ // Resume failed → fresh session → inject snapshot for context
1078
+ await this.injectSnapshotMessage();
1079
+ }
1080
+ else {
1081
+ // Clean up stale snapshot — resume restored full context
1082
+ try {
1083
+ unlinkSync(join(this.instanceDir, "rotation-state.json"));
1084
+ }
1085
+ catch { /* may not exist */ }
1086
+ }
1087
+ this.setProcessStatus("running");
1088
+ this.logger.info({ resumed }, `Respawned ${cliLabel} window after crash`);
1089
+ this.emit("crash_respawn", this.name);
1090
+ }
1091
+ catch (err) {
1092
+ this.logger.error({ err }, `Failed to respawn ${cliLabel} window`);
1043
1093
  }
1044
- this.setProcessStatus("running");
1045
- this.logger.info({ resumed }, `Respawned ${cliLabel} window after crash`);
1046
- this.emit("crash_respawn", this.name);
1047
1094
  }
1048
1095
  catch (err) {
1049
- this.logger.error({ err }, `Failed to respawn ${cliLabel} window`);
1096
+ this.logger.error({ err }, "Health check tick failed — continuing");
1097
+ // Surface it to the operator, not just the log — a health check that
1098
+ // keeps throwing means this instance is no longer being supervised.
1099
+ // Throttled: the tick repeats every ~30s, so an unnotified persistent
1100
+ // fault would otherwise post twice a minute forever.
1101
+ const now = Date.now();
1102
+ if (now - this.lastHealthErrorNotifyAt > HEALTH_ERROR_NOTIFY_INTERVAL_MS) {
1103
+ this.lastHealthErrorNotifyAt = now;
1104
+ this.emit("health_check_error", {
1105
+ name: this.name,
1106
+ message: err instanceof Error ? err.message : String(err),
1107
+ });
1108
+ }
1109
+ scheduleNext();
1110
+ return;
1050
1111
  }
1051
1112
  scheduleNext();
1052
1113
  }, healthCheckIntervalMs);
@@ -1838,12 +1899,20 @@ export class Daemon extends EventEmitter {
1838
1899
  // #77: show the sender's display name for readability, keeping the machine
1839
1900
  // instance name in parens so the recipient's send_to_instance target is valid.
1840
1901
  const fromLabel = meta.from_display ? `${meta.from_display} (${fromInstance})` : fromInstance;
1841
- formatted = `[from:${fromLabel}] ${content}\n(If you need to reply, use send_to_instance tool, NOT direct text. If there is nothing to add, you may stay silent.)`;
1902
+ formatted = `[from:${fromLabel}] ${content}`;
1903
+ formatted += renderHandoffMetadata(meta);
1904
+ // A delegated task that requires a reply must not read like a chatty FYI —
1905
+ // the "you may stay silent" line is for the latter only.
1906
+ formatted += meta.requires_reply === "true"
1907
+ ? "\n(A reply IS required: use report_result with the correlation_id above — or send_to_instance. Not direct text.)"
1908
+ : "\n(If you need to reply, use send_to_instance tool, NOT direct text. If there is nothing to add, you may stay silent.)";
1842
1909
  }
1843
1910
  else {
1844
1911
  const via = meta.source ? ` via ${meta.source}` : "";
1845
1912
  const idTag = meta.user_id ? `, id:${meta.user_id}` : "";
1846
- formatted = `[user:${user}${via}${idTag}] ${content}\n(Reply using the reply tool — do NOT respond with direct text)`;
1913
+ formatted = `[user:${user}${via}${idTag}] ${content}`;
1914
+ formatted += renderHandoffMetadata(meta);
1915
+ formatted += "\n(Reply using the reply tool — do NOT respond with direct text)";
1847
1916
  }
1848
1917
  if (meta.reply_to_text) {
1849
1918
  formatted += `\n(reply_to: "${meta.reply_to_text}")`;
@@ -1922,7 +1991,16 @@ export class Daemon extends EventEmitter {
1922
1991
  }
1923
1992
  else {
1924
1993
  this.logger.debug("CLI busy — queuing message until idle");
1925
- await this.controlClient.waitUntilIdle(windowId);
1994
+ const becameIdle = await this.controlClient.waitUntilIdle(windowId);
1995
+ if (!becameIdle) {
1996
+ // The pane never freed up. Report the failure instead of pasting into a
1997
+ // wedged CLI (where the text would sit unsubmitted and the next message
1998
+ // would land on top of it) — and instead of holding the queue silently.
1999
+ this.logger.error("Pane still busy after the idle wait — reporting delivery failure");
2000
+ if (status)
2001
+ this.emit("message_failed", status); // ❌
2002
+ return false;
2003
+ }
1926
2004
  }
1927
2005
  }
1928
2006
  // Bug A: paste with backoff. Transient failures are usually a stale window id
@@ -2007,8 +2085,20 @@ export class Daemon extends EventEmitter {
2007
2085
  await this.tmux.sendSpecialKey("Enter");
2008
2086
  becameBusy = await this.confirmBusyAfterEnter(windowId, retryAt);
2009
2087
  }
2010
- if (becameBusy && status)
2011
- this.emit("message_confirmed", status); // ✅
2088
+ if (becameBusy) {
2089
+ if (status)
2090
+ this.emit("message_confirmed", status); // ✅
2091
+ }
2092
+ else {
2093
+ // Both Enters were swallowed: the text is sitting UNSUBMITTED in the
2094
+ // CLI's input box. This used to return true, so the reaction stayed at 👀
2095
+ // forever and the next delivery pasted on top — submitting two messages
2096
+ // as one. Say so instead.
2097
+ this.logger.error("Message pasted but never submitted (no idle→busy after two Enters)");
2098
+ if (status)
2099
+ this.emit("message_failed", status); // ❌
2100
+ return false;
2101
+ }
2012
2102
  }
2013
2103
  else {
2014
2104
  // No control client to observe output: fall back to the legacy double-Enter.