@songsid/agend 2.1.1-beta.13 → 2.1.1-beta.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-path.js +15 -6
- package/dist/access-path.js.map +1 -1
- package/dist/classic-channel-manager.js +1 -5
- package/dist/classic-channel-manager.js.map +1 -1
- package/dist/cli.js +53 -24
- package/dist/cli.js.map +1 -1
- package/dist/cost-guard.d.ts +3 -1
- package/dist/cost-guard.js +3 -1
- package/dist/cost-guard.js.map +1 -1
- package/dist/daemon.d.ts +21 -0
- package/dist/daemon.js +318 -228
- package/dist/daemon.js.map +1 -1
- package/dist/event-log.js +4 -0
- package/dist/event-log.js.map +1 -1
- package/dist/fleet-manager.d.ts +56 -0
- package/dist/fleet-manager.js +206 -34
- package/dist/fleet-manager.js.map +1 -1
- package/dist/instance-lifecycle.js +5 -0
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/outbound-schemas.d.ts +2 -0
- package/dist/outbound-schemas.js +9 -1
- package/dist/outbound-schemas.js.map +1 -1
- package/dist/scheduler/db.js +3 -0
- package/dist/scheduler/db.js.map +1 -1
- package/dist/tmux-control.d.ts +10 -4
- package/dist/tmux-control.js +19 -6
- package/dist/tmux-control.js.map +1 -1
- package/dist/topic-commands.js +16 -7
- package/dist/topic-commands.js.map +1 -1
- package/dist/ui/view.html +10 -1
- package/dist/usage/providers.d.ts +24 -0
- package/dist/usage/providers.js +43 -9
- package/dist/usage/providers.js.map +1 -1
- package/dist/view-api.js +1 -0
- package/dist/view-api.js.map +1 -1
- package/dist/web-api.js +5 -2
- package/dist/web-api.js.map +1 -1
- package/package.json +4 -1
package/dist/daemon.js
CHANGED
|
@@ -42,6 +42,8 @@ export const DEFAULT_STATE_SAFETY_SWEEP_MS = 60_000;
|
|
|
42
42
|
/** @deprecated State detection is event-driven; this now aliases the safety sweep. */
|
|
43
43
|
export const DEFAULT_STATE_POLL_INTERVAL_MS = DEFAULT_STATE_SAFETY_SWEEP_MS;
|
|
44
44
|
const LAST_INBOUND_FILE = "last-inbound-at";
|
|
45
|
+
/** Minimum gap between "health check is failing" notifications for one instance. */
|
|
46
|
+
const HEALTH_ERROR_NOTIFY_INTERVAL_MS = 10 * 60_000;
|
|
45
47
|
/**
|
|
46
48
|
* Whether two working directories belong to the same project, so a fleet-scoped
|
|
47
49
|
* decision recorded in one reaches the other. Covers the worktree/checkout
|
|
@@ -105,6 +107,40 @@ export function writeLastInboundAt(instanceDir, timestamp) {
|
|
|
105
107
|
writeFileSync(temp, String(timestamp));
|
|
106
108
|
renameSync(temp, target);
|
|
107
109
|
}
|
|
110
|
+
/**
|
|
111
|
+
* Render the handoff/routing metadata that rides along with an inbound message
|
|
112
|
+
* into the block actually pasted into the agent's pane.
|
|
113
|
+
*
|
|
114
|
+
* These fields were all populated by the sender (outbound-handlers builds them
|
|
115
|
+
* into `ipcMeta`) and delivered over IPC, but never rendered — so the receiving
|
|
116
|
+
* agent could not see them, and several documented flows could not work:
|
|
117
|
+
*
|
|
118
|
+
* - `report_result` requires a `correlation_id` the agent had no way to know, so
|
|
119
|
+
* the "correlation_id not recognized" warning fired on essentially every call.
|
|
120
|
+
* - Delegation cancel buttons are retired by correlation id, so they never retired.
|
|
121
|
+
* - `react`, `edit_message` and `reply.reply_to` need `message_id`, which the tool
|
|
122
|
+
* descriptions tell the agent to take "from the inbound block".
|
|
123
|
+
* - `download_attachment` needs `attachment_file_id`.
|
|
124
|
+
* - `requires_reply` was invisible, so a delegated task looked like an FYI.
|
|
125
|
+
*
|
|
126
|
+
* Only non-empty fields are emitted, so a plain user message gains at most a
|
|
127
|
+
* message_id line.
|
|
128
|
+
*/
|
|
129
|
+
export function renderHandoffMetadata(meta) {
|
|
130
|
+
const rows = [];
|
|
131
|
+
const add = (label, value) => {
|
|
132
|
+
if (value && value.trim())
|
|
133
|
+
rows.push(`${label}: ${value.trim()}`);
|
|
134
|
+
};
|
|
135
|
+
add("message_id", meta.message_id);
|
|
136
|
+
add("correlation_id", meta.correlation_id);
|
|
137
|
+
add("request_kind", meta.request_kind);
|
|
138
|
+
add("task_summary", meta.task_summary);
|
|
139
|
+
add("working_directory", meta.working_directory);
|
|
140
|
+
add("branch", meta.branch);
|
|
141
|
+
add("attachment_file_id", meta.attachment_file_id);
|
|
142
|
+
return rows.length ? `\n(${rows.join(" | ")})` : "";
|
|
143
|
+
}
|
|
108
144
|
/** Headless inactivity timer used by the daemon and unit tests. */
|
|
109
145
|
export class AutoPauseController {
|
|
110
146
|
thresholdMs;
|
|
@@ -326,6 +362,7 @@ export class Daemon extends EventEmitter {
|
|
|
326
362
|
lastSpawnAt = 0;
|
|
327
363
|
crashTimestamps = [];
|
|
328
364
|
healthCheckPaused = false;
|
|
365
|
+
lastHealthErrorNotifyAt = 0;
|
|
329
366
|
/** CLI pane availability, independent from the daemon process and tri-state. */
|
|
330
367
|
processStatus = "running";
|
|
331
368
|
spawning = false;
|
|
@@ -797,256 +834,280 @@ export class Daemon extends EventEmitter {
|
|
|
797
834
|
this.healthCheckTimer = null;
|
|
798
835
|
if (this.runtimeMonitorsFrozen)
|
|
799
836
|
return;
|
|
800
|
-
//
|
|
801
|
-
//
|
|
802
|
-
//
|
|
803
|
-
//
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
//
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
837
|
+
// The whole tick is guarded: an unguarded throw in here (a tmux hiccup,
|
|
838
|
+
// ENOSPC on the crash-history write) became an unhandled rejection, which
|
|
839
|
+
// the CLI turned into stopAll() + exit(1) for the ENTIRE fleet, and it also
|
|
840
|
+
// ended this instance's health loop because scheduleNext() sat after the
|
|
841
|
+
// throwing code. Catch, log, and keep checking.
|
|
842
|
+
try {
|
|
843
|
+
// Instance directory removed externally (e.g. `rm -rf ~/.agend/instances/<name>`).
|
|
844
|
+
// Stop the loop permanently — otherwise every tick triggers a respawn, whose
|
|
845
|
+
// writeRotationSnapshot fails with ENOENT and gets caught as "Failed to respawn",
|
|
846
|
+
// spamming errors every ~30s forever.
|
|
847
|
+
if (!existsSync(this.instanceDir)) {
|
|
848
|
+
this.logger.warn({ instanceDir: this.instanceDir }, "Instance directory missing — stopping health check");
|
|
849
|
+
this.healthCheckPaused = true;
|
|
850
|
+
this.healthCheckTimer = null;
|
|
851
|
+
return;
|
|
852
|
+
}
|
|
853
|
+
if (!this.tmux || this.spawning || this.healthCheckPaused || Daemon.tmuxServerPaused) {
|
|
854
|
+
scheduleNext();
|
|
855
|
+
return;
|
|
856
|
+
}
|
|
857
|
+
// The CLI owns the MCP server process, so the daemon can only observe it.
|
|
858
|
+
this.checkMcpServerAlive();
|
|
859
|
+
// Human-readable backend label for logs (e.g. "claude", "kiro-cli")
|
|
860
|
+
const cliLabel = this.backend?.binaryName ?? "CLI";
|
|
861
|
+
let paneStatus = await this.tmux.getPaneStatus();
|
|
862
|
+
// Auto-pause intentionally exits the pane process. A health tick that
|
|
863
|
+
// began just before pause must not classify that exit as a crash.
|
|
864
|
+
if (this.isPaused || this.pauseWakeState === "waking") {
|
|
865
|
+
scheduleNext();
|
|
866
|
+
return;
|
|
830
867
|
}
|
|
831
|
-
scheduleNext();
|
|
832
|
-
return;
|
|
833
|
-
}
|
|
834
|
-
// A null status is ambiguous: it can be a transient `tmux list-panes`
|
|
835
|
-
// failure (e.g. tmux busy during a fleet-restart storm) rather than a
|
|
836
|
-
// real exit. Re-confirm once after a short delay before treating it as
|
|
837
|
-
// a crash. A non-null {alive:false} is a definite dead pane (real exit)
|
|
838
|
-
// and needs no recheck.
|
|
839
|
-
if (paneStatus === null) {
|
|
840
|
-
await new Promise(r => setTimeout(r, 1500));
|
|
841
|
-
paneStatus = await this.tmux.getPaneStatus();
|
|
842
868
|
if (paneStatus?.alive) {
|
|
843
|
-
|
|
869
|
+
// Instance output.log is fed by tmux pipe-pane and was previously never
|
|
870
|
+
// rotated (only fleet.log / daemon.log were). Cap growth every tick.
|
|
871
|
+
if (!this.config.lightweight) {
|
|
872
|
+
rotateLogIfNeeded(join(this.instanceDir, "output.log"));
|
|
873
|
+
}
|
|
844
874
|
scheduleNext();
|
|
845
875
|
return;
|
|
846
876
|
}
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
this.healthCheckPaused = true;
|
|
858
|
-
return;
|
|
859
|
-
}
|
|
860
|
-
this.setProcessStatus("crashed");
|
|
861
|
-
// Distinguish tmux server crash from single window crash.
|
|
862
|
-
// nullReason records *why* getPaneStatus returned null (for diagnosing
|
|
863
|
-
// whether this was a real window loss or a transient query failure).
|
|
864
|
-
let crashType = "window";
|
|
865
|
-
let nullReason;
|
|
866
|
-
if (!paneStatus) {
|
|
867
|
-
const serverAlive = await TmuxManager.sessionExists(this.tmuxSessionName);
|
|
868
|
-
if (!serverAlive) {
|
|
869
|
-
crashType = "server";
|
|
870
|
-
nullReason = "server_gone";
|
|
871
|
-
this.logger.error(`tmux server died — all ${cliLabel} windows lost`);
|
|
872
|
-
// Fleet-level circuit breaker: pause all instances on repeated tmux server crashes
|
|
873
|
-
Daemon.tmuxServerCrashTimestamps.push(Date.now());
|
|
874
|
-
const cutoff = Date.now() - 5 * 60_000;
|
|
875
|
-
Daemon.tmuxServerCrashTimestamps = Daemon.tmuxServerCrashTimestamps.filter(t => t > cutoff);
|
|
876
|
-
if (Daemon.tmuxServerCrashTimestamps.length >= 2 && !Daemon.tmuxServerPaused) {
|
|
877
|
-
Daemon.tmuxServerPaused = true;
|
|
878
|
-
this.logger.error("Fleet-level tmux server circuit breaker triggered — pausing all respawns for 30s");
|
|
879
|
-
this.emit("tmux_server_crash", this.name);
|
|
880
|
-
if (!Daemon.tmuxServerRecoveryTimer) {
|
|
881
|
-
Daemon.tmuxServerRecoveryTimer = setTimeout(() => {
|
|
882
|
-
Daemon.tmuxServerRecoveryTimer = null;
|
|
883
|
-
Daemon.tmuxServerPaused = false;
|
|
884
|
-
}, 30_000);
|
|
885
|
-
}
|
|
877
|
+
// A null status is ambiguous: it can be a transient `tmux list-panes`
|
|
878
|
+
// failure (e.g. tmux busy during a fleet-restart storm) rather than a
|
|
879
|
+
// real exit. Re-confirm once after a short delay before treating it as
|
|
880
|
+
// a crash. A non-null {alive:false} is a definite dead pane (real exit)
|
|
881
|
+
// and needs no recheck.
|
|
882
|
+
if (paneStatus === null) {
|
|
883
|
+
await new Promise(r => setTimeout(r, 1500));
|
|
884
|
+
paneStatus = await this.tmux.getPaneStatus();
|
|
885
|
+
if (paneStatus?.alive) {
|
|
886
|
+
this.logger.debug(`[health] ${cliLabel} pane reported gone then alive on recheck — transient query failure, ignoring`);
|
|
886
887
|
scheduleNext();
|
|
887
888
|
return;
|
|
888
889
|
}
|
|
889
|
-
await new Promise(r => setTimeout(r, 2_000)); // let session stabilize
|
|
890
890
|
}
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
891
|
+
// paneStatus === null → window gone entirely (e.g. tmux server crash)
|
|
892
|
+
// paneStatus.alive === false → pane dead, exit code available
|
|
893
|
+
const exitCode = paneStatus?.exitCode;
|
|
894
|
+
this.logger.debug({ exitCode }, `[health] pane exited with code: ${exitCode}`);
|
|
895
|
+
// Normal exit (e.g. user Ctrl+C or /exit) — no crash, no respawn
|
|
896
|
+
if (paneStatus && exitCode === 0) {
|
|
897
|
+
this.setProcessStatus("stopped");
|
|
898
|
+
this.logger.info("CLI exited normally (code 0) — pausing health check");
|
|
899
|
+
await this.tmux.killWindow();
|
|
900
|
+
this.healthCheckPaused = true;
|
|
901
|
+
return;
|
|
902
|
+
}
|
|
903
|
+
this.setProcessStatus("crashed");
|
|
904
|
+
// Distinguish tmux server crash from single window crash.
|
|
905
|
+
// nullReason records *why* getPaneStatus returned null (for diagnosing
|
|
906
|
+
// whether this was a real window loss or a transient query failure).
|
|
907
|
+
let crashType = "window";
|
|
908
|
+
let nullReason;
|
|
909
|
+
if (!paneStatus) {
|
|
910
|
+
const serverAlive = await TmuxManager.sessionExists(this.tmuxSessionName);
|
|
911
|
+
if (!serverAlive) {
|
|
912
|
+
crashType = "server";
|
|
913
|
+
nullReason = "server_gone";
|
|
914
|
+
this.logger.error(`tmux server died — all ${cliLabel} windows lost`);
|
|
915
|
+
// Fleet-level circuit breaker: pause all instances on repeated tmux server crashes
|
|
916
|
+
Daemon.tmuxServerCrashTimestamps.push(Date.now());
|
|
917
|
+
const cutoff = Date.now() - 5 * 60_000;
|
|
918
|
+
Daemon.tmuxServerCrashTimestamps = Daemon.tmuxServerCrashTimestamps.filter(t => t > cutoff);
|
|
919
|
+
if (Daemon.tmuxServerCrashTimestamps.length >= 2 && !Daemon.tmuxServerPaused) {
|
|
920
|
+
Daemon.tmuxServerPaused = true;
|
|
921
|
+
this.logger.error("Fleet-level tmux server circuit breaker triggered — pausing all respawns for 30s");
|
|
922
|
+
this.emit("tmux_server_crash", this.name);
|
|
923
|
+
if (!Daemon.tmuxServerRecoveryTimer) {
|
|
924
|
+
Daemon.tmuxServerRecoveryTimer = setTimeout(() => {
|
|
925
|
+
Daemon.tmuxServerRecoveryTimer = null;
|
|
926
|
+
Daemon.tmuxServerPaused = false;
|
|
927
|
+
}, 30_000);
|
|
928
|
+
}
|
|
929
|
+
scheduleNext();
|
|
930
|
+
return;
|
|
931
|
+
}
|
|
932
|
+
await new Promise(r => setTimeout(r, 2_000)); // let session stabilize
|
|
899
933
|
}
|
|
900
|
-
|
|
901
|
-
|
|
934
|
+
else {
|
|
935
|
+
// null but server alive: window-level disappearance. Probe whether
|
|
936
|
+
// the window truly no longer exists vs a transient query glitch.
|
|
937
|
+
nullReason = "no_window";
|
|
938
|
+
try {
|
|
939
|
+
const windows = await TmuxManager.listWindows(this.tmuxSessionName);
|
|
940
|
+
if (windows.some(w => w.name === this.name))
|
|
941
|
+
nullReason = "window_present_query_glitch";
|
|
942
|
+
}
|
|
943
|
+
catch {
|
|
944
|
+
nullReason = "query_error";
|
|
945
|
+
}
|
|
946
|
+
this.logger.warn({ exitCode, nullReason }, `${cliLabel} window not found (tmux server alive)`);
|
|
902
947
|
}
|
|
903
|
-
this.logger.warn({ exitCode, nullReason }, `${cliLabel} window not found (tmux server alive)`);
|
|
904
948
|
}
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
949
|
+
else {
|
|
950
|
+
this.logger.warn({ exitCode }, `${cliLabel} process exited`);
|
|
951
|
+
}
|
|
952
|
+
// Capture last output before killing. Best-effort even when the pane is
|
|
953
|
+
// gone (paneStatus null) — gives the crash record something to diagnose
|
|
954
|
+
// from instead of an empty lastOutput.
|
|
955
|
+
let lastOutput;
|
|
956
|
+
try {
|
|
957
|
+
const raw = await this.tmux.capturePaneWithHistory(50);
|
|
958
|
+
// Strip ANSI escape codes for readability
|
|
959
|
+
const cleaned = raw.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
|
|
960
|
+
lastOutput = cleaned.trimEnd() || undefined;
|
|
961
|
+
}
|
|
962
|
+
catch { /* best effort — pane may already be gone */ }
|
|
963
|
+
// Kill the dead window (remain-on-exit keeps it around) before respawn
|
|
964
|
+
if (paneStatus) {
|
|
965
|
+
await this.tmux.killWindow();
|
|
966
|
+
}
|
|
967
|
+
// Detect claude-code background session conflict — recover without counting as crash
|
|
968
|
+
if (lastOutput && (lastOutput.includes("background agent") || lastOutput.includes("Session is currently running"))) {
|
|
969
|
+
if (!this.backgroundSessionRecoveryAttempted) {
|
|
970
|
+
this.backgroundSessionRecoveryAttempted = true;
|
|
971
|
+
this.logger.warn("Detected lingering background agent session — starting fresh (no resume)");
|
|
972
|
+
const sidFile = join(this.instanceDir, "session-id");
|
|
973
|
+
try {
|
|
974
|
+
unlinkSync(sidFile);
|
|
975
|
+
}
|
|
976
|
+
catch { }
|
|
977
|
+
this.skipResume = true;
|
|
978
|
+
await new Promise(r => setTimeout(r, 2_000));
|
|
979
|
+
try {
|
|
980
|
+
await this.spawnClaudeWindow();
|
|
981
|
+
this.setProcessStatus("running");
|
|
982
|
+
this.logger.info("Recovered from background session conflict");
|
|
983
|
+
this.emit("crash_respawn", this.name);
|
|
984
|
+
}
|
|
985
|
+
catch (err) {
|
|
986
|
+
this.logger.error({ err: err.message }, "Recovery from background session conflict failed");
|
|
987
|
+
}
|
|
988
|
+
return; // Don't count as crash
|
|
989
|
+
}
|
|
990
|
+
// Already attempted recovery — fall through to normal crash handling
|
|
991
|
+
}
|
|
992
|
+
// Detect a --continue/--resume failure (no conversation to resume). The
|
|
993
|
+
// session-id file persists across the crash, so a blind respawn would add
|
|
994
|
+
// --continue again and crash in the same way → loop. Clear the session id
|
|
995
|
+
// and skip resume so the next spawn starts fresh. (skipResume also stops
|
|
996
|
+
// saveSessionId below from resurrecting the id from statusline.json.)
|
|
997
|
+
if (lastOutput && /no conversation found|no conversation to (continue|resume)|no previous (session|conversation)|--continue/i.test(lastOutput)) {
|
|
998
|
+
this.logger.warn("Detected --continue/resume failure — clearing session-id; next spawn starts fresh");
|
|
930
999
|
try {
|
|
931
|
-
unlinkSync(
|
|
1000
|
+
unlinkSync(join(this.instanceDir, "session-id"));
|
|
932
1001
|
}
|
|
933
|
-
catch { }
|
|
1002
|
+
catch { /* may not exist */ }
|
|
934
1003
|
this.skipResume = true;
|
|
935
|
-
|
|
1004
|
+
}
|
|
1005
|
+
// Append to crash history
|
|
1006
|
+
this.appendCrashHistory({ exitCode, lastOutput, crashType, reason: nullReason });
|
|
1007
|
+
if (max_retries <= 0) {
|
|
1008
|
+
this.healthCheckPaused = true;
|
|
1009
|
+
this.logger.warn(`${cliLabel} window died — automatic restart is disabled`);
|
|
1010
|
+
return;
|
|
1011
|
+
}
|
|
1012
|
+
// Detect rapid crash: sliding window — 3+ crashes in 5 minutes
|
|
1013
|
+
this.crashTimestamps.push(Date.now());
|
|
1014
|
+
const crashWindowMs = 5 * 60_000;
|
|
1015
|
+
this.crashTimestamps = this.crashTimestamps.filter(t => t > Date.now() - crashWindowMs);
|
|
1016
|
+
if (this.crashTimestamps.length >= 3) {
|
|
1017
|
+
this.healthCheckPaused = true;
|
|
1018
|
+
this.logger.error({ crashesInWindow: this.crashTimestamps.length }, "3+ crashes in 5 minutes — pausing respawn");
|
|
1019
|
+
// P1: Persist crash state so next process restart skips resume
|
|
936
1020
|
try {
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
catch (err) {
|
|
943
|
-
this.logger.error({ err: err.message }, "Recovery from background session conflict failed");
|
|
1021
|
+
writeFileSync(join(this.instanceDir, "crash-state.json"), JSON.stringify({
|
|
1022
|
+
crashesInWindow: this.crashTimestamps.length,
|
|
1023
|
+
lastCrashAt: Date.now(),
|
|
1024
|
+
resumeDisabled: true,
|
|
1025
|
+
}));
|
|
944
1026
|
}
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
}
|
|
949
|
-
// Detect a --continue/--resume failure (no conversation to resume). The
|
|
950
|
-
// session-id file persists across the crash, so a blind respawn would add
|
|
951
|
-
// --continue again and crash in the same way → loop. Clear the session id
|
|
952
|
-
// and skip resume so the next spawn starts fresh. (skipResume also stops
|
|
953
|
-
// saveSessionId below from resurrecting the id from statusline.json.)
|
|
954
|
-
if (lastOutput && /no conversation found|no conversation to (continue|resume)|no previous (session|conversation)|--continue/i.test(lastOutput)) {
|
|
955
|
-
this.logger.warn("Detected --continue/resume failure — clearing session-id; next spawn starts fresh");
|
|
956
|
-
try {
|
|
957
|
-
unlinkSync(join(this.instanceDir, "session-id"));
|
|
1027
|
+
catch { /* best effort */ }
|
|
1028
|
+
this.emit("crash_loop", this.name);
|
|
1029
|
+
return; // don't schedule next — paused
|
|
958
1030
|
}
|
|
959
|
-
|
|
960
|
-
this.
|
|
961
|
-
|
|
962
|
-
// Append to crash history
|
|
963
|
-
this.appendCrashHistory({ exitCode, lastOutput, crashType, reason: nullReason });
|
|
964
|
-
if (max_retries <= 0) {
|
|
965
|
-
this.healthCheckPaused = true;
|
|
966
|
-
this.logger.warn(`${cliLabel} window died — automatic restart is disabled`);
|
|
967
|
-
return;
|
|
968
|
-
}
|
|
969
|
-
// Detect rapid crash: sliding window — 3+ crashes in 5 minutes
|
|
970
|
-
this.crashTimestamps.push(Date.now());
|
|
971
|
-
const crashWindowMs = 5 * 60_000;
|
|
972
|
-
this.crashTimestamps = this.crashTimestamps.filter(t => t > Date.now() - crashWindowMs);
|
|
973
|
-
if (this.crashTimestamps.length >= 3) {
|
|
974
|
-
this.healthCheckPaused = true;
|
|
975
|
-
this.logger.error({ crashesInWindow: this.crashTimestamps.length }, "3+ crashes in 5 minutes — pausing respawn");
|
|
976
|
-
// P1: Persist crash state so next process restart skips resume
|
|
977
|
-
try {
|
|
978
|
-
writeFileSync(join(this.instanceDir, "crash-state.json"), JSON.stringify({
|
|
979
|
-
crashesInWindow: this.crashTimestamps.length,
|
|
980
|
-
lastCrashAt: Date.now(),
|
|
981
|
-
resumeDisabled: true,
|
|
982
|
-
}));
|
|
1031
|
+
// Reset crash count if enough time has passed
|
|
1032
|
+
if (reset_after > 0 && Date.now() - this.lastCrashAt > reset_after) {
|
|
1033
|
+
this.crashCount = 0;
|
|
983
1034
|
}
|
|
984
|
-
|
|
985
|
-
this.
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
if (reset_after > 0 && Date.now() - this.lastCrashAt > reset_after) {
|
|
990
|
-
this.crashCount = 0;
|
|
991
|
-
}
|
|
992
|
-
this.crashCount++;
|
|
993
|
-
this.lastCrashAt = Date.now();
|
|
994
|
-
if (this.crashCount > max_retries) {
|
|
995
|
-
this.logger.error({ crashCount: this.crashCount, maxRetries: max_retries }, "Max crash retries exceeded — not respawning");
|
|
996
|
-
return; // don't schedule next — given up
|
|
997
|
-
}
|
|
998
|
-
// Calculate backoff delay
|
|
999
|
-
const delay = backoff === "exponential"
|
|
1000
|
-
? Math.min(1000 * Math.pow(2, this.crashCount - 1), 60_000)
|
|
1001
|
-
: 1000 * this.crashCount;
|
|
1002
|
-
this.logger.warn({ crashCount: this.crashCount, delay }, `${cliLabel} window died — respawning after backoff`);
|
|
1003
|
-
await new Promise(r => setTimeout(r, delay));
|
|
1004
|
-
try {
|
|
1005
|
-
this.saveSessionId();
|
|
1006
|
-
this.transcriptMonitor?.resetOffset();
|
|
1007
|
-
// Kill orphan MCP server from the crashed CLI session.
|
|
1008
|
-
// MCP server writes its PID to channel.mcp.pid on startup.
|
|
1009
|
-
try {
|
|
1010
|
-
const pidFile = join(this.instanceDir, "channel.mcp.pid");
|
|
1011
|
-
const pid = parseInt(readFileSync(pidFile, "utf-8").trim(), 10);
|
|
1012
|
-
process.kill(pid, "SIGTERM");
|
|
1013
|
-
this.logger.info({ pid }, "Killed orphan MCP server");
|
|
1035
|
+
this.crashCount++;
|
|
1036
|
+
this.lastCrashAt = Date.now();
|
|
1037
|
+
if (this.crashCount > max_retries) {
|
|
1038
|
+
this.logger.error({ crashCount: this.crashCount, maxRetries: max_retries }, "Max crash retries exceeded — not respawning");
|
|
1039
|
+
return; // don't schedule next — given up
|
|
1014
1040
|
}
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1041
|
+
// Calculate backoff delay
|
|
1042
|
+
const delay = backoff === "exponential"
|
|
1043
|
+
? Math.min(1000 * Math.pow(2, this.crashCount - 1), 60_000)
|
|
1044
|
+
: 1000 * this.crashCount;
|
|
1045
|
+
this.logger.warn({ crashCount: this.crashCount, delay }, `${cliLabel} window died — respawning after backoff`);
|
|
1046
|
+
await new Promise(r => setTimeout(r, delay));
|
|
1019
1047
|
try {
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1048
|
+
this.saveSessionId();
|
|
1049
|
+
this.transcriptMonitor?.resetOffset();
|
|
1050
|
+
// Kill orphan MCP server from the crashed CLI session.
|
|
1051
|
+
// MCP server writes its PID to channel.mcp.pid on startup.
|
|
1052
|
+
try {
|
|
1053
|
+
const pidFile = join(this.instanceDir, "channel.mcp.pid");
|
|
1054
|
+
const pid = parseInt(readFileSync(pidFile, "utf-8").trim(), 10);
|
|
1055
|
+
process.kill(pid, "SIGTERM");
|
|
1056
|
+
this.logger.info({ pid }, "Killed orphan MCP server");
|
|
1026
1057
|
}
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
// Try --resume first; spawnClaudeWindow falls back to fresh session if resume fails
|
|
1032
|
-
const resumed = await this.spawnClaudeWindow();
|
|
1033
|
-
if (!resumed) {
|
|
1034
|
-
// Resume failed → fresh session → inject snapshot for context
|
|
1035
|
-
await this.injectSnapshotMessage();
|
|
1036
|
-
}
|
|
1037
|
-
else {
|
|
1038
|
-
// Clean up stale snapshot — resume restored full context
|
|
1058
|
+
catch { /* no pid file or process already dead */ }
|
|
1059
|
+
// Kill any same-name windows before respawn to prevent orphans.
|
|
1060
|
+
// Wrapped in try-catch: if tmux server is dead, listWindows throws —
|
|
1061
|
+
// must not block spawnClaudeWindow (which calls ensureSession).
|
|
1039
1062
|
try {
|
|
1040
|
-
|
|
1063
|
+
const windows = await TmuxManager.listWindows(this.tmuxSessionName);
|
|
1064
|
+
for (const w of windows) {
|
|
1065
|
+
if (w.name === this.name) {
|
|
1066
|
+
const tm = new TmuxManager(this.tmuxSessionName, w.id);
|
|
1067
|
+
await tm.killWindow();
|
|
1068
|
+
}
|
|
1069
|
+
}
|
|
1041
1070
|
}
|
|
1042
|
-
catch { /* may
|
|
1071
|
+
catch { /* tmux server may be dead — ensureSession in trySpawn will recover */ }
|
|
1072
|
+
// Write snapshot before spawn — consumed only if resume fails
|
|
1073
|
+
this.writeRotationSnapshot("crash");
|
|
1074
|
+
// Try --resume first; spawnClaudeWindow falls back to fresh session if resume fails
|
|
1075
|
+
const resumed = await this.spawnClaudeWindow();
|
|
1076
|
+
if (!resumed) {
|
|
1077
|
+
// Resume failed → fresh session → inject snapshot for context
|
|
1078
|
+
await this.injectSnapshotMessage();
|
|
1079
|
+
}
|
|
1080
|
+
else {
|
|
1081
|
+
// Clean up stale snapshot — resume restored full context
|
|
1082
|
+
try {
|
|
1083
|
+
unlinkSync(join(this.instanceDir, "rotation-state.json"));
|
|
1084
|
+
}
|
|
1085
|
+
catch { /* may not exist */ }
|
|
1086
|
+
}
|
|
1087
|
+
this.setProcessStatus("running");
|
|
1088
|
+
this.logger.info({ resumed }, `Respawned ${cliLabel} window after crash`);
|
|
1089
|
+
this.emit("crash_respawn", this.name);
|
|
1090
|
+
}
|
|
1091
|
+
catch (err) {
|
|
1092
|
+
this.logger.error({ err }, `Failed to respawn ${cliLabel} window`);
|
|
1043
1093
|
}
|
|
1044
|
-
this.setProcessStatus("running");
|
|
1045
|
-
this.logger.info({ resumed }, `Respawned ${cliLabel} window after crash`);
|
|
1046
|
-
this.emit("crash_respawn", this.name);
|
|
1047
1094
|
}
|
|
1048
1095
|
catch (err) {
|
|
1049
|
-
this.logger.error({ err },
|
|
1096
|
+
this.logger.error({ err }, "Health check tick failed — continuing");
|
|
1097
|
+
// Surface it to the operator, not just the log — a health check that
|
|
1098
|
+
// keeps throwing means this instance is no longer being supervised.
|
|
1099
|
+
// Throttled: the tick repeats every ~30s, so an unnotified persistent
|
|
1100
|
+
// fault would otherwise post twice a minute forever.
|
|
1101
|
+
const now = Date.now();
|
|
1102
|
+
if (now - this.lastHealthErrorNotifyAt > HEALTH_ERROR_NOTIFY_INTERVAL_MS) {
|
|
1103
|
+
this.lastHealthErrorNotifyAt = now;
|
|
1104
|
+
this.emit("health_check_error", {
|
|
1105
|
+
name: this.name,
|
|
1106
|
+
message: err instanceof Error ? err.message : String(err),
|
|
1107
|
+
});
|
|
1108
|
+
}
|
|
1109
|
+
scheduleNext();
|
|
1110
|
+
return;
|
|
1050
1111
|
}
|
|
1051
1112
|
scheduleNext();
|
|
1052
1113
|
}, healthCheckIntervalMs);
|
|
@@ -1838,12 +1899,20 @@ export class Daemon extends EventEmitter {
|
|
|
1838
1899
|
// #77: show the sender's display name for readability, keeping the machine
|
|
1839
1900
|
// instance name in parens so the recipient's send_to_instance target is valid.
|
|
1840
1901
|
const fromLabel = meta.from_display ? `${meta.from_display} (${fromInstance})` : fromInstance;
|
|
1841
|
-
formatted = `[from:${fromLabel}] ${content}
|
|
1902
|
+
formatted = `[from:${fromLabel}] ${content}`;
|
|
1903
|
+
formatted += renderHandoffMetadata(meta);
|
|
1904
|
+
// A delegated task that requires a reply must not read like a chatty FYI —
|
|
1905
|
+
// the "you may stay silent" line is for the latter only.
|
|
1906
|
+
formatted += meta.requires_reply === "true"
|
|
1907
|
+
? "\n(A reply IS required: use report_result with the correlation_id above — or send_to_instance. Not direct text.)"
|
|
1908
|
+
: "\n(If you need to reply, use send_to_instance tool, NOT direct text. If there is nothing to add, you may stay silent.)";
|
|
1842
1909
|
}
|
|
1843
1910
|
else {
|
|
1844
1911
|
const via = meta.source ? ` via ${meta.source}` : "";
|
|
1845
1912
|
const idTag = meta.user_id ? `, id:${meta.user_id}` : "";
|
|
1846
|
-
formatted = `[user:${user}${via}${idTag}] ${content}
|
|
1913
|
+
formatted = `[user:${user}${via}${idTag}] ${content}`;
|
|
1914
|
+
formatted += renderHandoffMetadata(meta);
|
|
1915
|
+
formatted += "\n(Reply using the reply tool — do NOT respond with direct text)";
|
|
1847
1916
|
}
|
|
1848
1917
|
if (meta.reply_to_text) {
|
|
1849
1918
|
formatted += `\n(reply_to: "${meta.reply_to_text}")`;
|
|
@@ -1922,7 +1991,16 @@ export class Daemon extends EventEmitter {
|
|
|
1922
1991
|
}
|
|
1923
1992
|
else {
|
|
1924
1993
|
this.logger.debug("CLI busy — queuing message until idle");
|
|
1925
|
-
await this.controlClient.waitUntilIdle(windowId);
|
|
1994
|
+
const becameIdle = await this.controlClient.waitUntilIdle(windowId);
|
|
1995
|
+
if (!becameIdle) {
|
|
1996
|
+
// The pane never freed up. Report the failure instead of pasting into a
|
|
1997
|
+
// wedged CLI (where the text would sit unsubmitted and the next message
|
|
1998
|
+
// would land on top of it) — and instead of holding the queue silently.
|
|
1999
|
+
this.logger.error("Pane still busy after the idle wait — reporting delivery failure");
|
|
2000
|
+
if (status)
|
|
2001
|
+
this.emit("message_failed", status); // ❌
|
|
2002
|
+
return false;
|
|
2003
|
+
}
|
|
1926
2004
|
}
|
|
1927
2005
|
}
|
|
1928
2006
|
// Bug A: paste with backoff. Transient failures are usually a stale window id
|
|
@@ -2007,8 +2085,20 @@ export class Daemon extends EventEmitter {
|
|
|
2007
2085
|
await this.tmux.sendSpecialKey("Enter");
|
|
2008
2086
|
becameBusy = await this.confirmBusyAfterEnter(windowId, retryAt);
|
|
2009
2087
|
}
|
|
2010
|
-
if (becameBusy
|
|
2011
|
-
|
|
2088
|
+
if (becameBusy) {
|
|
2089
|
+
if (status)
|
|
2090
|
+
this.emit("message_confirmed", status); // ✅
|
|
2091
|
+
}
|
|
2092
|
+
else {
|
|
2093
|
+
// Both Enters were swallowed: the text is sitting UNSUBMITTED in the
|
|
2094
|
+
// CLI's input box. This used to return true, so the reaction stayed at 👀
|
|
2095
|
+
// forever and the next delivery pasted on top — submitting two messages
|
|
2096
|
+
// as one. Say so instead.
|
|
2097
|
+
this.logger.error("Message pasted but never submitted (no idle→busy after two Enters)");
|
|
2098
|
+
if (status)
|
|
2099
|
+
this.emit("message_failed", status); // ❌
|
|
2100
|
+
return false;
|
|
2101
|
+
}
|
|
2012
2102
|
}
|
|
2013
2103
|
else {
|
|
2014
2104
|
// No control client to observe output: fall back to the legacy double-Enter.
|