@songsid/agend 2.1.2-beta.9 → 2.1.3-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/access-path.js +15 -6
- package/dist/access-path.js.map +1 -1
- package/dist/agent-cli-instructions.md +1 -0
- package/dist/agent-cli.js +14 -0
- package/dist/agent-cli.js.map +1 -1
- package/dist/agent-endpoint.d.ts +2 -0
- package/dist/agent-endpoint.js +25 -1
- package/dist/agent-endpoint.js.map +1 -1
- package/dist/backend/antigravity.d.ts +20 -0
- package/dist/backend/antigravity.js +51 -4
- package/dist/backend/antigravity.js.map +1 -1
- package/dist/backend/claude-code.d.ts +60 -0
- package/dist/backend/claude-code.js +91 -4
- package/dist/backend/claude-code.js.map +1 -1
- package/dist/backend/codex.d.ts +26 -0
- package/dist/backend/codex.js +100 -0
- package/dist/backend/codex.js.map +1 -1
- package/dist/backend/grok.d.ts +25 -0
- package/dist/backend/grok.js +31 -0
- package/dist/backend/grok.js.map +1 -1
- package/dist/backend/kiro.d.ts +23 -0
- package/dist/backend/kiro.js +121 -2
- package/dist/backend/kiro.js.map +1 -1
- package/dist/backend/types.d.ts +60 -0
- package/dist/backend/types.js +13 -1
- package/dist/backend/types.js.map +1 -1
- package/dist/channel/adapters/discord.d.ts +15 -0
- package/dist/channel/adapters/discord.js +120 -8
- package/dist/channel/adapters/discord.js.map +1 -1
- package/dist/channel/adapters/telegram.d.ts +21 -0
- package/dist/channel/adapters/telegram.js +86 -0
- package/dist/channel/adapters/telegram.js.map +1 -1
- package/dist/channel/ipc-bridge.d.ts +9 -1
- package/dist/channel/ipc-bridge.js +12 -3
- package/dist/channel/ipc-bridge.js.map +1 -1
- package/dist/channel/ipc-timeouts.d.ts +46 -0
- package/dist/channel/ipc-timeouts.js +65 -0
- package/dist/channel/ipc-timeouts.js.map +1 -0
- package/dist/channel/mcp-server.js +25 -14
- package/dist/channel/mcp-server.js.map +1 -1
- package/dist/channel/mcp-tools.js +30 -1
- package/dist/channel/mcp-tools.js.map +1 -1
- package/dist/channel/reconnect-backoff.d.ts +17 -0
- package/dist/channel/reconnect-backoff.js +21 -0
- package/dist/channel/reconnect-backoff.js.map +1 -0
- package/dist/channel/types.d.ts +29 -0
- package/dist/classic-channel-manager.js +1 -5
- package/dist/classic-channel-manager.js.map +1 -1
- package/dist/cli.js +242 -91
- package/dist/cli.js.map +1 -1
- package/dist/completion.d.ts +27 -0
- package/dist/completion.js +121 -0
- package/dist/completion.js.map +1 -0
- package/dist/config-validator.js +21 -0
- package/dist/config-validator.js.map +1 -1
- package/dist/config.js +2 -0
- package/dist/config.js.map +1 -1
- package/dist/cost-guard.d.ts +3 -1
- package/dist/cost-guard.js +3 -1
- package/dist/cost-guard.js.map +1 -1
- package/dist/daemon.d.ts +298 -6
- package/dist/daemon.js +1157 -313
- package/dist/daemon.js.map +1 -1
- package/dist/event-log.d.ts +31 -0
- package/dist/event-log.js +96 -0
- package/dist/event-log.js.map +1 -1
- package/dist/fleet-context.d.ts +14 -0
- package/dist/fleet-lock.d.ts +28 -0
- package/dist/fleet-lock.js +130 -0
- package/dist/fleet-lock.js.map +1 -0
- package/dist/fleet-manager.d.ts +348 -6
- package/dist/fleet-manager.js +1859 -266
- package/dist/fleet-manager.js.map +1 -1
- package/dist/general-instance.d.ts +9 -0
- package/dist/general-instance.js +11 -0
- package/dist/general-instance.js.map +1 -0
- package/dist/general-knowledge/skills/cross-instance-messaging/SKILL.md +22 -0
- package/dist/general-knowledge/skills/model-discovery/SKILL.md +14 -23
- package/dist/general-knowledge/skills/session-management/SKILL.md +15 -34
- package/dist/hang-detector.d.ts +19 -13
- package/dist/hang-detector.js +19 -49
- package/dist/hang-detector.js.map +1 -1
- package/dist/instance-lifecycle.d.ts +58 -2
- package/dist/instance-lifecycle.js +231 -46
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/instructions.js +1 -1
- package/dist/locale.js +7 -1
- package/dist/locale.js.map +1 -1
- package/dist/mcp-liveness.d.ts +21 -0
- package/dist/mcp-liveness.js +27 -0
- package/dist/mcp-liveness.js.map +1 -0
- package/dist/outbound-handlers.d.ts +9 -0
- package/dist/outbound-handlers.js +194 -31
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/outbound-schemas.d.ts +11 -3
- package/dist/outbound-schemas.js +16 -2
- package/dist/outbound-schemas.js.map +1 -1
- package/dist/pane-write-lock.d.ts +48 -0
- package/dist/pane-write-lock.js +73 -0
- package/dist/pane-write-lock.js.map +1 -0
- package/dist/process-memory.d.ts +31 -0
- package/dist/process-memory.js +79 -0
- package/dist/process-memory.js.map +1 -0
- package/dist/quickstart.js +17 -16
- package/dist/quickstart.js.map +1 -1
- package/dist/reply-dedup.d.ts +41 -0
- package/dist/reply-dedup.js +0 -0
- package/dist/reply-dedup.js.map +1 -0
- package/dist/restart-progress.d.ts +41 -0
- package/dist/restart-progress.js +97 -0
- package/dist/restart-progress.js.map +1 -0
- package/dist/scheduler/db.js +3 -0
- package/dist/scheduler/db.js.map +1 -1
- package/dist/sd-notify.d.ts +27 -0
- package/dist/sd-notify.js +33 -1
- package/dist/sd-notify.js.map +1 -1
- package/dist/secret-file.d.ts +33 -0
- package/dist/secret-file.js +36 -0
- package/dist/secret-file.js.map +1 -0
- package/dist/service-installer.d.ts +28 -0
- package/dist/service-installer.js +72 -1
- package/dist/service-installer.js.map +1 -1
- package/dist/settings-api.js +5 -3
- package/dist/settings-api.js.map +1 -1
- package/dist/setup-wizard.js +9 -7
- package/dist/setup-wizard.js.map +1 -1
- package/dist/tmux-control.d.ts +58 -5
- package/dist/tmux-control.js +102 -14
- package/dist/tmux-control.js.map +1 -1
- package/dist/tmux-manager.d.ts +31 -1
- package/dist/tmux-manager.js +89 -28
- package/dist/tmux-manager.js.map +1 -1
- package/dist/topic-commands.d.ts +49 -9
- package/dist/topic-commands.js +270 -109
- package/dist/topic-commands.js.map +1 -1
- package/dist/tui-glyphs.d.ts +5 -1
- package/dist/tui-glyphs.js +6 -2
- package/dist/tui-glyphs.js.map +1 -1
- package/dist/types.d.ts +19 -2
- package/dist/ui/settings.html +1 -1
- package/dist/ui/view.html +123 -2
- package/dist/update-marker.d.ts +15 -0
- package/dist/update-marker.js +69 -0
- package/dist/update-marker.js.map +1 -0
- package/dist/usage/format-rich.d.ts +11 -0
- package/dist/usage/format-rich.js +121 -0
- package/dist/usage/format-rich.js.map +1 -0
- package/dist/usage/provider-alerts.d.ts +33 -0
- package/dist/usage/provider-alerts.js +57 -0
- package/dist/usage/provider-alerts.js.map +1 -0
- package/dist/usage/providers.d.ts +130 -0
- package/dist/usage/providers.js +1063 -0
- package/dist/usage/providers.js.map +1 -0
- package/dist/usage/statusline-usage.d.ts +20 -0
- package/dist/usage/statusline-usage.js +120 -0
- package/dist/usage/statusline-usage.js.map +1 -0
- package/dist/usage/usage-api.d.ts +50 -0
- package/dist/usage/usage-api.js +197 -0
- package/dist/usage/usage-api.js.map +1 -0
- package/dist/view-api.js +8 -3
- package/dist/view-api.js.map +1 -1
- package/dist/web-api.js +7 -3
- package/dist/web-api.js.map +1 -1
- package/dist/web-auth.d.ts +6 -0
- package/dist/web-auth.js +54 -0
- package/dist/web-auth.js.map +1 -0
- package/package.json +4 -1
- package/dist/channel/tool-tracker.d.ts +0 -13
- package/dist/channel/tool-tracker.js +0 -58
- package/dist/channel/tool-tracker.js.map +0 -1
- package/dist/daemon-entry.d.ts +0 -1
- package/dist/daemon-entry.js +0 -30
- package/dist/daemon-entry.js.map +0 -1
package/dist/daemon.js
CHANGED
|
@@ -1,19 +1,23 @@
|
|
|
1
1
|
import { join, dirname, basename, resolve } from "node:path";
|
|
2
|
-
import { mkdirSync, writeFileSync, readFileSync, existsSync, unlinkSync, rmSync, appendFileSync, statSync,
|
|
2
|
+
import { mkdirSync, writeFileSync, readFileSync, existsSync, unlinkSync, rmSync, appendFileSync, statSync, renameSync } from "node:fs";
|
|
3
3
|
import { fileURLToPath } from "node:url";
|
|
4
4
|
import { createHash, randomBytes } from "node:crypto";
|
|
5
5
|
import { EventEmitter } from "node:events";
|
|
6
6
|
import { rotateLogIfNeeded } from "./logger.js";
|
|
7
|
+
import { mcpServerState } from "./mcp-liveness.js";
|
|
7
8
|
import { clearPausedMarker, writePausedMarker } from "./pause-marker.js";
|
|
8
9
|
import { TmuxManager, resolveTmuxLogicalSize } from "./tmux-manager.js";
|
|
9
10
|
import { TranscriptMonitor } from "./transcript-monitor.js";
|
|
10
11
|
import { ContextGuardian } from "./context-guardian.js";
|
|
11
12
|
import { IpcServer } from "./channel/ipc-bridge.js";
|
|
13
|
+
import { daemonBudgetMs } from "./channel/ipc-timeouts.js";
|
|
12
14
|
import { MessageBus } from "./channel/message-bus.js";
|
|
13
15
|
import { shellQuote } from "./backend/types.js";
|
|
14
16
|
import { getTmuxSession } from "./config.js";
|
|
15
17
|
import { routeToolCall } from "./channel/tool-router.js";
|
|
16
18
|
import { HangDetector } from "./hang-detector.js";
|
|
19
|
+
import { writeSecretFile } from "./secret-file.js";
|
|
20
|
+
import { PaneWriteLock } from "./pane-write-lock.js";
|
|
17
21
|
import { buildFleetInstructions } from "./instructions.js";
|
|
18
22
|
const __filename = fileURLToPath(import.meta.url);
|
|
19
23
|
const __dirname = dirname(__filename);
|
|
@@ -38,9 +42,54 @@ export function buildInstructionReloadNotice(binaryName, instanceName, instanceD
|
|
|
38
42
|
export const DEFAULT_STUCK_TIMEOUT_MS = 10 * 60_000;
|
|
39
43
|
export const DEFAULT_STATE_IDLE_DEBOUNCE_MS = 2_000;
|
|
40
44
|
export const DEFAULT_STATE_SAFETY_SWEEP_MS = 60_000;
|
|
41
|
-
/** @deprecated State detection is event-driven; this now aliases the safety sweep. */
|
|
42
|
-
export const DEFAULT_STATE_POLL_INTERVAL_MS = DEFAULT_STATE_SAFETY_SWEEP_MS;
|
|
43
45
|
const LAST_INBOUND_FILE = "last-inbound-at";
|
|
46
|
+
/** Minimum gap between "health check is failing" notifications for one instance. */
|
|
47
|
+
const HEALTH_ERROR_NOTIFY_INTERVAL_MS = 10 * 60_000;
|
|
48
|
+
/**
|
|
49
|
+
* Whether two working directories belong to the same project, so a fleet-scoped
|
|
50
|
+
* decision recorded in one reaches the other. Covers the worktree/checkout
|
|
51
|
+
* layout AgEnD fleets actually use — `AgEnD`, `AgEnD-dev1`, `AgEnD-dev2`,
|
|
52
|
+
* `AgEnD-reviewer`, `AgEnD-main` are one project; `DouPo_Server` is not.
|
|
53
|
+
* Nesting counts too (a subdirectory of a project belongs to it).
|
|
54
|
+
*/
|
|
55
|
+
export function sameProjectFamily(a, b) {
|
|
56
|
+
const norm = (p) => p.replace(/\/+$/, "");
|
|
57
|
+
const x = norm(a), y = norm(b);
|
|
58
|
+
if (x === y)
|
|
59
|
+
return true;
|
|
60
|
+
if (x.startsWith(y + "/") || y.startsWith(x + "/"))
|
|
61
|
+
return true;
|
|
62
|
+
// Strip a trailing worktree/role suffix: "AgEnD-dev2" → "agend".
|
|
63
|
+
const family = (p) => {
|
|
64
|
+
const base = p.slice(p.lastIndexOf("/") + 1).toLowerCase();
|
|
65
|
+
return base.replace(/[-_](dev\d*|main|reviewer|tester|leader|sol|codex|worktree|wt\d*)$/i, "");
|
|
66
|
+
};
|
|
67
|
+
const fx = family(x), fy = family(y);
|
|
68
|
+
return fx.length > 0 && fx === fy;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Pick the decisions worth injecting into one instance's system prompt.
|
|
72
|
+
*
|
|
73
|
+
* `scope: "fleet"` previously bypassed the project check outright, so every
|
|
74
|
+
* instance carried every other project's playbook — measured at 8 of 14 here,
|
|
75
|
+
* resent on every API call. A fleet decision must now also be relevant: global
|
|
76
|
+
* (no project_root), the same project, or the same project family.
|
|
77
|
+
*
|
|
78
|
+
* `isDispatcher` (a general) opts out of that narrowing on purpose: it routes
|
|
79
|
+
* work across all projects, so cross-project rules ("X 文件操作由 Y 負責") are
|
|
80
|
+
* precisely what it must know. Narrowing a general would cause misrouting.
|
|
81
|
+
*/
|
|
82
|
+
export function selectRelevantDecisions(all, workDir, isDispatcher = false) {
|
|
83
|
+
return all.filter(d => {
|
|
84
|
+
if (d.project_root === workDir)
|
|
85
|
+
return true; // own project, any scope
|
|
86
|
+
if (d.scope !== "fleet")
|
|
87
|
+
return false; // project-scoped, elsewhere
|
|
88
|
+
if (!d.project_root)
|
|
89
|
+
return true; // truly global
|
|
90
|
+
return isDispatcher || sameProjectFamily(d.project_root, workDir);
|
|
91
|
+
});
|
|
92
|
+
}
|
|
44
93
|
/** Read the last real channel inbound timestamp persisted across daemon restarts. */
|
|
45
94
|
export function readLastInboundAt(instanceDir, now = Date.now()) {
|
|
46
95
|
try {
|
|
@@ -59,6 +108,40 @@ export function writeLastInboundAt(instanceDir, timestamp) {
|
|
|
59
108
|
writeFileSync(temp, String(timestamp));
|
|
60
109
|
renameSync(temp, target);
|
|
61
110
|
}
|
|
111
|
+
/**
|
|
112
|
+
* Render the handoff/routing metadata that rides along with an inbound message
|
|
113
|
+
* into the block actually pasted into the agent's pane.
|
|
114
|
+
*
|
|
115
|
+
* These fields were all populated by the sender (outbound-handlers builds them
|
|
116
|
+
* into `ipcMeta`) and delivered over IPC, but never rendered — so the receiving
|
|
117
|
+
* agent could not see them, and several documented flows could not work:
|
|
118
|
+
*
|
|
119
|
+
* - `report_result` requires a `correlation_id` the agent had no way to know, so
|
|
120
|
+
* the "correlation_id not recognized" warning fired on essentially every call.
|
|
121
|
+
* - Delegation cancel buttons are retired by correlation id, so they never retired.
|
|
122
|
+
* - `react`, `edit_message` and `reply.reply_to` need `message_id`, which the tool
|
|
123
|
+
* descriptions tell the agent to take "from the inbound block".
|
|
124
|
+
* - `download_attachment` needs `attachment_file_id`.
|
|
125
|
+
* - `requires_reply` was invisible, so a delegated task looked like an FYI.
|
|
126
|
+
*
|
|
127
|
+
* Only non-empty fields are emitted, so a plain user message gains at most a
|
|
128
|
+
* message_id line.
|
|
129
|
+
*/
|
|
130
|
+
export function renderHandoffMetadata(meta) {
|
|
131
|
+
const rows = [];
|
|
132
|
+
const add = (label, value) => {
|
|
133
|
+
if (value && value.trim())
|
|
134
|
+
rows.push(`${label}: ${value.trim()}`);
|
|
135
|
+
};
|
|
136
|
+
add("message_id", meta.message_id);
|
|
137
|
+
add("correlation_id", meta.correlation_id);
|
|
138
|
+
add("request_kind", meta.request_kind);
|
|
139
|
+
add("task_summary", meta.task_summary);
|
|
140
|
+
add("working_directory", meta.working_directory);
|
|
141
|
+
add("branch", meta.branch);
|
|
142
|
+
add("attachment_file_id", meta.attachment_file_id);
|
|
143
|
+
return rows.length ? `\n(${rows.join(" | ")})` : "";
|
|
144
|
+
}
|
|
62
145
|
/** Headless inactivity timer used by the daemon and unit tests. */
|
|
63
146
|
export class AutoPauseController {
|
|
64
147
|
thresholdMs;
|
|
@@ -103,37 +186,111 @@ export class AutoPauseController {
|
|
|
103
186
|
* Pane motion wins over a ready match because several backends keep their ready
|
|
104
187
|
* marker in a persistent header/footer while generating. A stable ready pane is
|
|
105
188
|
* idle; changing content is working; stable non-ready content eventually sticks.
|
|
189
|
+
*
|
|
190
|
+
* ## Why an optional busy pattern exists
|
|
191
|
+
*
|
|
192
|
+
* "Motion wins" degrades badly when a backend's ready marker is *always* on
|
|
193
|
+
* screen. `stuck` requires `!ready`, so for such a backend the state can only ever
|
|
194
|
+
* be idle or working — the stuck edge, and with it `handleStuckTransition` and the
|
|
195
|
+
* hang notification, is unreachable. A frozen CLI is then reported as idle, which
|
|
196
|
+
* also clears pending work: the message the user sent is quietly booked as done.
|
|
197
|
+
*
|
|
198
|
+
* claude-code was exactly this case (see ClaudeCodeBackend.getReadyPattern). A
|
|
199
|
+
* backend that can point at a marker meaning "generating right now" supplies
|
|
200
|
+
* `getBusyPattern()`, and that match vetoes ready regardless of what the ready
|
|
201
|
+
* pattern says.
|
|
106
202
|
*/
|
|
203
|
+
/**
|
|
204
|
+
* A shell command reduced to the part that is safe to show: the program, and a
|
|
205
|
+
* subcommand when there is one.
|
|
206
|
+
*
|
|
207
|
+
* curl -H "Bearer sk-ant-…" https://api.com → curl
|
|
208
|
+
* npm test --env=API_KEY=xxx → npm test
|
|
209
|
+
* git push origin main → git push
|
|
210
|
+
*
|
|
211
|
+
* Arguments are dropped wholesale rather than filtered, because there is no
|
|
212
|
+
* reliable way to tell a secret from an ordinary argument: tokens, bearer
|
|
213
|
+
* headers, connection strings and `--password=` values all look like text. The
|
|
214
|
+
* progress line is posted to a chat channel and, for a public one, so is
|
|
215
|
+
* anything in it — so the rule is "never the arguments", not "not the arguments
|
|
216
|
+
* that look dangerous".
|
|
217
|
+
*
|
|
218
|
+
* A second token is kept only when it is a bare word (`push`, `test`), which is
|
|
219
|
+
* what a subcommand looks like and what a flag, path, URL, assignment or quoted
|
|
220
|
+
* string does not. Everything after it goes, including `git push ORIGIN MAIN`.
|
|
221
|
+
*/
|
|
222
|
+
export function shellCommandLabel(command) {
|
|
223
|
+
// First line only: heredocs and multi-line scripts carry the payload.
|
|
224
|
+
const tokens = (command.split("\n")[0] ?? "").trim().split(/\s+/).filter(Boolean);
|
|
225
|
+
// `FOO=bar cmd …` — leading assignments are env, and their values are exactly
|
|
226
|
+
// the kind of thing being protected here.
|
|
227
|
+
while (tokens.length && /^[A-Za-z_][A-Za-z0-9_]*=/.test(tokens[0]))
|
|
228
|
+
tokens.shift();
|
|
229
|
+
const program = tokens[0];
|
|
230
|
+
if (!program)
|
|
231
|
+
return "";
|
|
232
|
+
const isBareWord = (t) => !!t && /^[A-Za-z][A-Za-z0-9_-]*$/.test(t);
|
|
233
|
+
const label = isBareWord(tokens[1]) ? `${program} ${tokens[1]}` : program;
|
|
234
|
+
return label.length > 40 ? `${label.slice(0, 39)}…` : label;
|
|
235
|
+
}
|
|
107
236
|
export class PaneStateMachine {
|
|
108
237
|
stuckTimeoutMs;
|
|
109
238
|
readyPattern;
|
|
239
|
+
busyPattern;
|
|
110
240
|
lastPaneHash = null;
|
|
111
241
|
lastPaneChangeAt;
|
|
112
242
|
lastObservedAt;
|
|
113
243
|
stateChangedAt;
|
|
114
244
|
currentState = "idle";
|
|
115
|
-
constructor(readyPattern, stuckTimeoutMs = DEFAULT_STUCK_TIMEOUT_MS, now = Date.now()) {
|
|
245
|
+
constructor(readyPattern, stuckTimeoutMs = DEFAULT_STUCK_TIMEOUT_MS, now = Date.now(), busyPattern) {
|
|
116
246
|
this.stuckTimeoutMs = stuckTimeoutMs;
|
|
117
247
|
// Stateful g/y regexes mutate lastIndex and can alternate true/false across
|
|
118
248
|
// polls. State detection must be deterministic for identical pane content.
|
|
119
249
|
this.readyPattern = new RegExp(readyPattern.source, readyPattern.flags.replace(/[gy]/g, ""));
|
|
250
|
+
this.busyPattern = busyPattern
|
|
251
|
+
? new RegExp(busyPattern.source, busyPattern.flags.replace(/[gy]/g, ""))
|
|
252
|
+
: null;
|
|
120
253
|
this.lastPaneChangeAt = now;
|
|
121
254
|
this.lastObservedAt = now;
|
|
122
255
|
this.stateChangedAt = now;
|
|
123
256
|
}
|
|
124
|
-
|
|
257
|
+
/** Ready, unless the backend can positively see that it is still generating. */
|
|
258
|
+
isReady(pane) {
|
|
259
|
+
if (this.busyPattern?.test(pane))
|
|
260
|
+
return false;
|
|
261
|
+
return this.readyPattern.test(pane);
|
|
262
|
+
}
|
|
263
|
+
/**
|
|
264
|
+
* @param now when this observation is being evaluated — drives the idle and
|
|
265
|
+
* stuck decisions.
|
|
266
|
+
* @param opts.changeAt when the content is believed to have changed, if that
|
|
267
|
+
* is earlier than `now` (the tmux output timestamp). Only the elapsed-time
|
|
268
|
+
* clock uses it; defaults to `now`.
|
|
269
|
+
* @param opts.settled the caller knows no output has arrived for the debounce
|
|
270
|
+
* window, so "the content differs from the last capture" says nothing about
|
|
271
|
+
* whether the CLI is still generating — decide on the patterns instead.
|
|
272
|
+
*
|
|
273
|
+
* Captures are event-driven, not periodic: the one taken 2s after output
|
|
274
|
+
* stops is compared against a capture that may be a minute old, so it
|
|
275
|
+
* *always* differs. Without this flag that difference reads as "still
|
|
276
|
+
* working" and a finished turn would not be seen as idle until the next 60s
|
|
277
|
+
* safety sweep. Output timestamps are the honest liveness signal here, and
|
|
278
|
+
* the caller has them.
|
|
279
|
+
*/
|
|
280
|
+
observe(pane, now = Date.now(), opts = {}) {
|
|
281
|
+
const { settled = false, changeAt = now } = opts;
|
|
125
282
|
const paneHash = createHash("sha256").update(pane).digest("hex");
|
|
126
283
|
const firstObservation = this.lastPaneHash === null;
|
|
127
284
|
const paneChanged = this.lastPaneHash !== paneHash;
|
|
128
285
|
if (paneChanged) {
|
|
129
286
|
this.lastPaneHash = paneHash;
|
|
130
|
-
this.lastPaneChangeAt =
|
|
287
|
+
this.lastPaneChangeAt = changeAt;
|
|
131
288
|
}
|
|
132
289
|
this.lastObservedAt = now;
|
|
133
|
-
const ready = this.
|
|
290
|
+
const ready = this.isReady(pane);
|
|
134
291
|
const nextState = firstObservation
|
|
135
292
|
? ready ? "idle" : "working"
|
|
136
|
-
: paneChanged
|
|
293
|
+
: paneChanged && !settled
|
|
137
294
|
? "working"
|
|
138
295
|
: ready
|
|
139
296
|
? "idle"
|
|
@@ -194,6 +351,23 @@ export class PendingWorkTracker {
|
|
|
194
351
|
const NORMAL_ENTER_SETTLE_MS = 500;
|
|
195
352
|
const FIRST_ENTER_SETTLE_MS = 1_750;
|
|
196
353
|
const FIRST_DELIVERY_WINDOW_MS = 5_000;
|
|
354
|
+
/** After busy native-queue paste+Enter, wait before checking the pane for silent loss. */
|
|
355
|
+
const NATIVE_QUEUE_PASTE_VERIFY_MS = 2_000;
|
|
356
|
+
/** Enter-confirmation poll: 10 × 200ms ≈ 2s of observed silence before giving up. */
|
|
357
|
+
const CONFIRM_BUSY_POLLS = 10;
|
|
358
|
+
const CONFIRM_BUSY_POLL_MS = 200;
|
|
359
|
+
/**
|
|
360
|
+
* Ceiling on the whole confirmation, including time re-earned after a control-mode
|
|
361
|
+
* reconnect. Without it a reconnect loop could restart the poll indefinitely and
|
|
362
|
+
* hold the pane write lock along with it.
|
|
363
|
+
*/
|
|
364
|
+
const CONFIRM_BUSY_MAX_WAIT_MS = 10_000;
|
|
365
|
+
/**
|
|
366
|
+
* How long a delivery waits for an in-flight spawn. Comfortably past the default
|
|
367
|
+
* 25s startup timeout plus dialog dismissal; past it we fall back to the old
|
|
368
|
+
* behaviour rather than holding a message indefinitely.
|
|
369
|
+
*/
|
|
370
|
+
const SPAWN_SETTLE_MAX_WAIT_MS = 60_000;
|
|
197
371
|
/**
|
|
198
372
|
* One-shot timing gate for the first paste after a CLI reaches its ready
|
|
199
373
|
* prompt. Recent tmux versions expose a short interval where the prompt is
|
|
@@ -238,6 +412,68 @@ export function sanitizePaneTail(pane, lineCount = 5) {
|
|
|
238
412
|
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "")
|
|
239
413
|
.slice(0, 200));
|
|
240
414
|
}
|
|
415
|
+
const INTERACTIVE_PROMPT_PATTERNS = [
|
|
416
|
+
{ kind: "sudo_password", pattern: /^\s*\[sudo\]\s+password\s+for\s+[^:\n]+:\s*$/im },
|
|
417
|
+
// macOS sudo and several package installers use this exact no-echo prompt.
|
|
418
|
+
// Tail-only + stability gating is what makes the otherwise-generic word safe.
|
|
419
|
+
{ kind: "password", pattern: /^\s*(?:Password|Passphrase):\s*$/im },
|
|
420
|
+
{ kind: "confirmation", pattern: /^[^\n]{0,180}(?:\([Yy]\/[Nn]\)|\[[Yy]\/[Nn]\]|\((?:yes|no)\/(?:yes|no)\))\s*:?\s*$/im },
|
|
421
|
+
{ kind: "confirmation", pattern: /^\s*Are you sure[^\n]{0,160}\((?:yes\/no)(?:\/\[[^\]]+\])?\)\??\s*$/im },
|
|
422
|
+
{ kind: "press_enter", pattern: /^\s*(?:Please\s+)?Press (?:the )?Enter(?: key)?(?: to [^\n]{0,120})?[.:…]?\s*$/im },
|
|
423
|
+
];
|
|
424
|
+
/**
|
|
425
|
+
* Detect terminal prompts that require a human, but only after the pane tail is
|
|
426
|
+
* unchanged and no control-mode output has arrived for the full grace period.
|
|
427
|
+
* This prevents prose such as "the installer asks [Y/n]" from notifying while
|
|
428
|
+
* an agent is still writing it.
|
|
429
|
+
*/
|
|
430
|
+
export class InteractivePromptDetector {
|
|
431
|
+
stableMs;
|
|
432
|
+
signature = null;
|
|
433
|
+
stableSince = 0;
|
|
434
|
+
lastOutputAt = 0;
|
|
435
|
+
notifiedSignature = null;
|
|
436
|
+
constructor(stableMs = 10_000) {
|
|
437
|
+
this.stableMs = stableMs;
|
|
438
|
+
}
|
|
439
|
+
observe(pane, now = Date.now(), outputAt = 0) {
|
|
440
|
+
const tail = sanitizePaneTail(pane, 5);
|
|
441
|
+
const tailText = tail.join("\n");
|
|
442
|
+
let matched = null;
|
|
443
|
+
for (const candidate of INTERACTIVE_PROMPT_PATTERNS) {
|
|
444
|
+
const match = tailText.match(candidate.pattern);
|
|
445
|
+
if (!match)
|
|
446
|
+
continue;
|
|
447
|
+
matched = { kind: candidate.kind, prompt: match[0].trim().slice(0, 200) };
|
|
448
|
+
break;
|
|
449
|
+
}
|
|
450
|
+
if (!matched) {
|
|
451
|
+
this.reset();
|
|
452
|
+
this.lastOutputAt = outputAt;
|
|
453
|
+
return null;
|
|
454
|
+
}
|
|
455
|
+
const signature = `${matched.kind}:${tailText}`;
|
|
456
|
+
const outputMoved = outputAt > this.lastOutputAt;
|
|
457
|
+
if (signature !== this.signature || outputMoved) {
|
|
458
|
+
this.signature = signature;
|
|
459
|
+
this.stableSince = now;
|
|
460
|
+
this.lastOutputAt = outputAt;
|
|
461
|
+
if (signature !== this.notifiedSignature)
|
|
462
|
+
this.notifiedSignature = null;
|
|
463
|
+
return null;
|
|
464
|
+
}
|
|
465
|
+
this.lastOutputAt = outputAt;
|
|
466
|
+
if (this.notifiedSignature === signature || now - this.stableSince < this.stableMs)
|
|
467
|
+
return null;
|
|
468
|
+
this.notifiedSignature = signature;
|
|
469
|
+
return matched;
|
|
470
|
+
}
|
|
471
|
+
reset() {
|
|
472
|
+
this.signature = null;
|
|
473
|
+
this.stableSince = 0;
|
|
474
|
+
this.notifiedSignature = null;
|
|
475
|
+
}
|
|
476
|
+
}
|
|
241
477
|
export class Daemon extends EventEmitter {
|
|
242
478
|
name;
|
|
243
479
|
config;
|
|
@@ -252,7 +488,6 @@ export class Daemon extends EventEmitter {
|
|
|
252
488
|
ipcServer = null;
|
|
253
489
|
messageBus;
|
|
254
490
|
transcriptMonitor = null;
|
|
255
|
-
toolTracker = null;
|
|
256
491
|
guardian = null;
|
|
257
492
|
adapter = null;
|
|
258
493
|
pendingIpcRequests = new Map();
|
|
@@ -263,6 +498,8 @@ export class Daemon extends EventEmitter {
|
|
|
263
498
|
// Pending ack: react 🫡 on first transcript activity after receiving a message
|
|
264
499
|
pendingAckMessage = null;
|
|
265
500
|
// Tool status tracking for channel adapter
|
|
501
|
+
/** Last activity published to the fleet manager; null when nothing is running. */
|
|
502
|
+
currentActivity = null;
|
|
266
503
|
toolStatusMessageId = null;
|
|
267
504
|
toolStatusLines = [];
|
|
268
505
|
toolStatusDebounce = null;
|
|
@@ -278,9 +515,19 @@ export class Daemon extends EventEmitter {
|
|
|
278
515
|
lastSpawnAt = 0;
|
|
279
516
|
crashTimestamps = [];
|
|
280
517
|
healthCheckPaused = false;
|
|
518
|
+
lastHealthErrorNotifyAt = 0;
|
|
281
519
|
/** CLI pane availability, independent from the daemon process and tri-state. */
|
|
282
520
|
processStatus = "running";
|
|
283
521
|
spawning = false;
|
|
522
|
+
/**
|
|
523
|
+
* Resolves when the in-flight spawn finishes; null when none is running.
|
|
524
|
+
*
|
|
525
|
+
* `spawning` alone can only be polled. Delivery needs to *wait*, and a boolean
|
|
526
|
+
* poll would either busy-loop or race the flag being cleared.
|
|
527
|
+
*/
|
|
528
|
+
spawnSettled = null;
|
|
529
|
+
resolveSpawnSettled = null;
|
|
530
|
+
spawnDepth = 0;
|
|
284
531
|
skipResume = false;
|
|
285
532
|
backgroundSessionRecoveryAttempted = false;
|
|
286
533
|
/** Whether the last spawn started a fresh session (not resumed). */
|
|
@@ -303,10 +550,18 @@ export class Daemon extends EventEmitter {
|
|
|
303
550
|
instanceStateIdleDebounceMs = DEFAULT_STATE_IDLE_DEBOUNCE_MS;
|
|
304
551
|
instanceStateStuckTimeoutMs = DEFAULT_STUCK_TIMEOUT_MS;
|
|
305
552
|
instanceStateReadyPattern = null;
|
|
553
|
+
instanceStateBusyPattern = null;
|
|
306
554
|
instanceStateMonitorActive = false;
|
|
307
555
|
statePollInFlight = false;
|
|
308
556
|
autoPauseController;
|
|
309
557
|
pauseRequested = false;
|
|
558
|
+
/**
|
|
559
|
+
* Sticky "pause as soon as possible" (auth failure). Unlike pauseRequested —
|
|
560
|
+
* which the state monitor clears whenever the pane is not idle — this survives
|
|
561
|
+
* busy/stuck states, because an auth error fires mid-turn and a plain pause()
|
|
562
|
+
* would silently no-op exactly when we most need to stop feeding the CLI.
|
|
563
|
+
*/
|
|
564
|
+
pausePending = false;
|
|
310
565
|
pauseWakeState = "active";
|
|
311
566
|
pauseWakeTransition = null;
|
|
312
567
|
// Model failover: override model on next spawn when rate-limited
|
|
@@ -316,7 +571,13 @@ export class Daemon extends EventEmitter {
|
|
|
316
571
|
recentEvents = [];
|
|
317
572
|
recentToolActivity = [];
|
|
318
573
|
snapshotConsumed = false;
|
|
574
|
+
/** Orders inbound channel messages against each other (queue-depth accounting
|
|
575
|
+
* and ⏳/👀/✅ reactions live here). It does NOT cover the other writers that
|
|
576
|
+
* reach the pane — {@link paneWriteLock} does. */
|
|
319
577
|
pasteLock = Promise.resolve();
|
|
578
|
+
/** Mutual exclusion for *every* write into the pane, whichever subsystem it
|
|
579
|
+
* comes from. See PaneWriteLock for why interleaving is destructive. */
|
|
580
|
+
paneWriteLock = new PaneWriteLock();
|
|
320
581
|
pendingInstructionsUpdate;
|
|
321
582
|
pendingInstructionsNotice = false;
|
|
322
583
|
// Whether the warmup steering-reload notice should be injected after spawn.
|
|
@@ -329,6 +590,26 @@ export class Daemon extends EventEmitter {
|
|
|
329
590
|
firstDeliveryDelay = new FirstDeliveryDelay();
|
|
330
591
|
// PTY error pattern monitoring
|
|
331
592
|
errorMonitorTimer = null;
|
|
593
|
+
interactivePromptDetector = new InteractivePromptDetector();
|
|
594
|
+
/** Same 5-min gate the error monitor uses, so a dead MCP server alerts once. */
|
|
595
|
+
static MCP_DEATH_COOLDOWN_MS = 5 * 60_000;
|
|
596
|
+
lastMcpDeathNotifiedAt = 0;
|
|
597
|
+
mcpDeathNotifiedForPid = null;
|
|
598
|
+
/**
|
|
599
|
+
* Ceiling on how long a dead MCP server waits for an idle window before the
|
|
600
|
+
* auto-restart fires anyway. A toolless instance cannot reply or report, so
|
|
601
|
+
* whatever a very long turn produces is stranded until the restart happens —
|
|
602
|
+
* past this point interrupting the turn costs less than staying mute.
|
|
603
|
+
*/
|
|
604
|
+
static MCP_RESTART_MAX_WAIT_MS = 30 * 60_000;
|
|
605
|
+
/**
|
|
606
|
+
* Sticky "restart to revive the MCP server as soon as the pane idles" — same
|
|
607
|
+
* shape as pausePending, because the death is usually detected mid-turn when
|
|
608
|
+
* an immediate restart would destroy in-flight work. Cleared when the server
|
|
609
|
+
* turns up alive again (operator restarted, CLI reconnected) before we fire.
|
|
610
|
+
*/
|
|
611
|
+
mcpRestartPending = false;
|
|
612
|
+
mcpRestartStaleTimer = null;
|
|
332
613
|
/** Prevent in-flight monitor callbacks from re-arming after a pause. */
|
|
333
614
|
runtimeMonitorsFrozen = false;
|
|
334
615
|
errorWaitingForRecovery = false; // true = error detected, waiting for ready pattern
|
|
@@ -340,6 +621,10 @@ export class Daemon extends EventEmitter {
|
|
|
340
621
|
return this.errorWaitingForRecovery || (this.healthCheckPaused && !this.isPaused) || Daemon.tmuxServerPaused;
|
|
341
622
|
}
|
|
342
623
|
get isPaused() { return this.pauseWakeState !== "active"; }
|
|
624
|
+
/** Current CLI process state for status surfaces and recovery commands. */
|
|
625
|
+
getProcessStatus() {
|
|
626
|
+
return this.processStatus;
|
|
627
|
+
}
|
|
343
628
|
get lastPausedAt() { return this.autoPauseController.lastPausedAt; }
|
|
344
629
|
getPauseWakeState() { return this.pauseWakeState; }
|
|
345
630
|
/** Whether this instance is in a crash loop (3+ consecutive crashes). */
|
|
@@ -455,9 +740,6 @@ export class Daemon extends EventEmitter {
|
|
|
455
740
|
if ((type === "fleet_schedule_response" || type === "fleet_outbound_response" || type === "fleet_decision_response" || type === "fleet_task_response" || type === "fleet_display_name_response" || type === "fleet_description_response") && msg.fleetRequestId) {
|
|
456
741
|
key = String(msg.fleetRequestId);
|
|
457
742
|
}
|
|
458
|
-
else if (type === "fleet_outbound_response" && msg.requestId != null) {
|
|
459
|
-
key = `fleet_out_${msg.requestId}`;
|
|
460
|
-
}
|
|
461
743
|
if (key && this.pendingIpcRequests.has(key)) {
|
|
462
744
|
const handler = this.pendingIpcRequests.get(key);
|
|
463
745
|
this.pendingIpcRequests.delete(key);
|
|
@@ -608,7 +890,12 @@ export class Daemon extends EventEmitter {
|
|
|
608
890
|
else {
|
|
609
891
|
await new Promise(r => setTimeout(r, 5000));
|
|
610
892
|
}
|
|
611
|
-
|
|
893
|
+
// This path only runs when pasteQueueDepth > 0 — i.e. exactly when a real
|
|
894
|
+
// delivery is already in flight or queued. Without the lock the notice and
|
|
895
|
+
// that delivery race into the same pane.
|
|
896
|
+
await this.paneWriteLock.run(async () => {
|
|
897
|
+
await this.tmux?.pasteText(buildInstructionReloadNotice(this.backend?.binaryName ?? "unknown", this.name, this.instanceDir), this.systemPasteOptions());
|
|
898
|
+
});
|
|
612
899
|
// Record the value the agent has now been told about so the next
|
|
613
900
|
// unchanged restart skips the reload.
|
|
614
901
|
try {
|
|
@@ -641,27 +928,31 @@ export class Daemon extends EventEmitter {
|
|
|
641
928
|
this.transcriptMonitor.on("tool_use", (name, input) => {
|
|
642
929
|
this.logger.debug({ tool: name }, "Tool use");
|
|
643
930
|
ackIfPending();
|
|
644
|
-
this.hangDetector?.recordActivity();
|
|
645
931
|
this.recordRecentEvent({ type: "tool_use", name, preview: this.summarizeTool(name, input) });
|
|
646
932
|
this.recordRecentToolActivity(this.summarizeTool(name, input));
|
|
933
|
+
this.publishActivity(this.summarizeTool(name, input));
|
|
647
934
|
});
|
|
648
935
|
this.transcriptMonitor.on("tool_result", (name, _output) => {
|
|
649
|
-
this.hangDetector?.recordActivity();
|
|
650
936
|
this.recordRecentEvent({ type: "tool_result", name });
|
|
937
|
+
// The tool finished; whatever comes next has not started yet. Better to
|
|
938
|
+
// show only elapsed time than to leave a stale "Bash: npm test" on screen.
|
|
939
|
+
this.publishActivity(null);
|
|
651
940
|
});
|
|
652
941
|
this.transcriptMonitor.on("assistant_text", (text) => {
|
|
653
942
|
this.logger.debug({ text: text.slice(0, 200) }, "Claude response");
|
|
654
943
|
ackIfPending();
|
|
655
|
-
this.hangDetector?.recordActivity();
|
|
656
944
|
this.recordRecentEvent({ type: "assistant_text", preview: text.slice(0, 100) });
|
|
657
945
|
});
|
|
658
946
|
this.transcriptMonitor.startPolling();
|
|
659
|
-
// HangDetector
|
|
660
|
-
//
|
|
661
|
-
//
|
|
947
|
+
// HangDetector is the notification bridge to the fleet manager; pane state
|
|
948
|
+
// transitions below are the sole source of hang events (see hang-detector.ts).
|
|
949
|
+
// `hang_detector.enabled: false` opts out of the notification entirely.
|
|
950
|
+
// NOTE: `hang_detector.timeout_minutes` is NOT read here — the stuck timeout
|
|
951
|
+
// the pane monitor actually uses is resolved separately below. The two have
|
|
952
|
+
// long been separate; documented rather than silently ignored.
|
|
662
953
|
const hangConfig = this.config.hang_detector;
|
|
663
954
|
if (hangConfig?.enabled !== false) {
|
|
664
|
-
this.hangDetector = new HangDetector(
|
|
955
|
+
this.hangDetector = new HangDetector();
|
|
665
956
|
}
|
|
666
957
|
// 8. Context guardian
|
|
667
958
|
const statusFile = join(this.instanceDir, "statusline.json");
|
|
@@ -669,7 +960,6 @@ export class Daemon extends EventEmitter {
|
|
|
669
960
|
this.guardian.startWatching();
|
|
670
961
|
this.guardian.on("status_update", () => {
|
|
671
962
|
this.saveSessionId();
|
|
672
|
-
this.hangDetector?.recordStatuslineUpdate();
|
|
673
963
|
});
|
|
674
964
|
// Context rotation removed: all CLI backends have built-in auto-compact.
|
|
675
965
|
// Crash recovery (health check + respawn with snapshot) is retained below.
|
|
@@ -689,6 +979,89 @@ export class Daemon extends EventEmitter {
|
|
|
689
979
|
this.startInstanceStateMonitor();
|
|
690
980
|
this.logger.info(`${this.name} ready`);
|
|
691
981
|
}
|
|
982
|
+
/**
|
|
983
|
+
* Detect a CRASHED MCP server and report it once.
|
|
984
|
+
*
|
|
985
|
+
* The MCP server writes its pid to channel.mcp.pid at startup and unlinks it on
|
|
986
|
+
* a clean exit, so "file present + pid dead" specifically means it died without
|
|
987
|
+
* cleaning up (crash / OOM / SIGKILL). A missing file is either "not started
|
|
988
|
+
* yet" or "exited cleanly" — neither is an incident, and an orphan exit implies
|
|
989
|
+
* the CLI itself died, which the crash detector already covers.
|
|
990
|
+
*
|
|
991
|
+
* The daemon deliberately does NOT try to respawn it: MCP is a stdio protocol
|
|
992
|
+
* where the CLI spawns the server and owns its pipes, so a daemon-spawned
|
|
993
|
+
* process would have no client reading it. Only the CLI can restore its own
|
|
994
|
+
* tools — hence notify, and let the operator decide about restarting.
|
|
995
|
+
*/
|
|
996
|
+
checkMcpServerAlive() {
|
|
997
|
+
if (this.isPaused)
|
|
998
|
+
return;
|
|
999
|
+
const status = mcpServerState(this.instanceDir);
|
|
1000
|
+
if (status.state === "unknown")
|
|
1001
|
+
return;
|
|
1002
|
+
if (status.state === "alive") {
|
|
1003
|
+
this.mcpDeathNotifiedForPid = null; // the CLI respawned it — re-arm
|
|
1004
|
+
this.clearMcpRestartRequest(); // tools are back — stand down a pending auto-restart
|
|
1005
|
+
return;
|
|
1006
|
+
}
|
|
1007
|
+
// Dead: report once per pid, and at most once per cooldown window.
|
|
1008
|
+
if (this.mcpDeathNotifiedForPid === status.pid)
|
|
1009
|
+
return;
|
|
1010
|
+
if (Date.now() - this.lastMcpDeathNotifiedAt < Daemon.MCP_DEATH_COOLDOWN_MS)
|
|
1011
|
+
return;
|
|
1012
|
+
this.mcpDeathNotifiedForPid = status.pid;
|
|
1013
|
+
this.lastMcpDeathNotifiedAt = Date.now();
|
|
1014
|
+
const autoRestart = this.config.mcp_auto_restart !== false;
|
|
1015
|
+
this.logger.error({ pid: status.pid, autoRestart }, "MCP server process is gone — instance has no agend tools");
|
|
1016
|
+
this.emit("mcp_died", { name: this.name, pid: status.pid, autoRestart });
|
|
1017
|
+
if (autoRestart)
|
|
1018
|
+
this.armMcpRestartWhenIdle();
|
|
1019
|
+
}
|
|
1020
|
+
/**
|
|
1021
|
+
* Arm an automatic instance restart to revive a dead MCP server. Fires on the
|
|
1022
|
+
* next busy→idle edge — or immediately when the pane is already idle, which is
|
|
1023
|
+
* the common case for a collab/chat instance parked between messages (no edge
|
|
1024
|
+
* would ever come). A turn that outlives MCP_RESTART_MAX_WAIT_MS is
|
|
1025
|
+
* force-restarted: see that constant for why staying mute costs more.
|
|
1026
|
+
*/
|
|
1027
|
+
armMcpRestartWhenIdle() {
|
|
1028
|
+
if (this.mcpRestartPending)
|
|
1029
|
+
return;
|
|
1030
|
+
this.mcpRestartPending = true;
|
|
1031
|
+
this.mcpRestartStaleTimer = setTimeout(() => {
|
|
1032
|
+
this.mcpRestartStaleTimer = null;
|
|
1033
|
+
if (!this.mcpRestartPending || this.runtimeMonitorsFrozen)
|
|
1034
|
+
return;
|
|
1035
|
+
this.fireMcpRestartRequest("stale_timeout");
|
|
1036
|
+
}, Daemon.MCP_RESTART_MAX_WAIT_MS);
|
|
1037
|
+
this.mcpRestartStaleTimer.unref?.();
|
|
1038
|
+
if (this.instanceState === "idle" && this.pasteQueueDepth === 0) {
|
|
1039
|
+
this.fireMcpRestartRequest("already_idle");
|
|
1040
|
+
}
|
|
1041
|
+
}
|
|
1042
|
+
fireMcpRestartRequest(trigger) {
|
|
1043
|
+
// A recovery is already reshaping the CLI: crash-loop handling
|
|
1044
|
+
// (healthCheckPaused), a respawn in flight (spawning), or a pause
|
|
1045
|
+
// (isPaused / frozen monitors). Each of those paths ends in a fresh CLI —
|
|
1046
|
+
// and with it a fresh MCP server — or in supervision ending, where a
|
|
1047
|
+
// restart on top would fight the recovery. Either way the revival restart
|
|
1048
|
+
// is moot: cancel it rather than defer it.
|
|
1049
|
+
if (this.healthCheckPaused || this.spawning || this.isPaused || this.runtimeMonitorsFrozen) {
|
|
1050
|
+
this.logger.warn({ trigger }, "MCP revival restart cancelled — instance is already pausing/spawning/recovering");
|
|
1051
|
+
this.clearMcpRestartRequest();
|
|
1052
|
+
return;
|
|
1053
|
+
}
|
|
1054
|
+
this.clearMcpRestartRequest();
|
|
1055
|
+
this.logger.warn({ trigger }, "Requesting instance restart to revive its MCP server");
|
|
1056
|
+
this.emit("mcp_restart_requested", { name: this.name, trigger });
|
|
1057
|
+
}
|
|
1058
|
+
clearMcpRestartRequest() {
|
|
1059
|
+
this.mcpRestartPending = false;
|
|
1060
|
+
if (this.mcpRestartStaleTimer) {
|
|
1061
|
+
clearTimeout(this.mcpRestartStaleTimer);
|
|
1062
|
+
this.mcpRestartStaleTimer = null;
|
|
1063
|
+
}
|
|
1064
|
+
}
|
|
692
1065
|
startHealthCheck() {
|
|
693
1066
|
if (this.runtimeMonitorsFrozen || this.healthCheckTimer)
|
|
694
1067
|
return;
|
|
@@ -704,254 +1077,294 @@ export class Daemon extends EventEmitter {
|
|
|
704
1077
|
this.healthCheckTimer = null;
|
|
705
1078
|
if (this.runtimeMonitorsFrozen)
|
|
706
1079
|
return;
|
|
707
|
-
//
|
|
708
|
-
//
|
|
709
|
-
//
|
|
710
|
-
//
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
1080
|
+
// The whole tick is guarded: an unguarded throw in here (a tmux hiccup,
|
|
1081
|
+
// ENOSPC on the crash-history write) became an unhandled rejection, which
|
|
1082
|
+
// the CLI turned into stopAll() + exit(1) for the ENTIRE fleet, and it also
|
|
1083
|
+
// ended this instance's health loop because scheduleNext() sat after the
|
|
1084
|
+
// throwing code. Catch, log, and keep checking.
|
|
1085
|
+
try {
|
|
1086
|
+
// Instance directory removed externally (e.g. `rm -rf ~/.agend/instances/<name>`).
|
|
1087
|
+
// Stop the loop permanently — otherwise every tick triggers a respawn, whose
|
|
1088
|
+
// writeRotationSnapshot fails with ENOENT and gets caught as "Failed to respawn",
|
|
1089
|
+
// spamming errors every ~30s forever.
|
|
1090
|
+
if (!existsSync(this.instanceDir)) {
|
|
1091
|
+
this.logger.warn({ instanceDir: this.instanceDir }, "Instance directory missing — stopping health check");
|
|
1092
|
+
this.healthCheckPaused = true;
|
|
1093
|
+
this.healthCheckTimer = null;
|
|
1094
|
+
this.emitSupervisionEnded("instance directory was removed from disk", "Recreate the instance, or delete it from fleet.yaml.");
|
|
1095
|
+
return;
|
|
1096
|
+
}
|
|
1097
|
+
if (!this.tmux || this.spawning || this.healthCheckPaused || Daemon.tmuxServerPaused) {
|
|
1098
|
+
scheduleNext();
|
|
1099
|
+
return;
|
|
1100
|
+
}
|
|
1101
|
+
// The CLI owns the MCP server process, so the daemon can only observe it.
|
|
1102
|
+
this.checkMcpServerAlive();
|
|
1103
|
+
// Human-readable backend label for logs (e.g. "claude", "kiro-cli")
|
|
1104
|
+
const cliLabel = this.backend?.binaryName ?? "CLI";
|
|
1105
|
+
let paneStatus = await this.tmux.getPaneStatus();
|
|
1106
|
+
// Auto-pause intentionally exits the pane process. A health tick that
|
|
1107
|
+
// began just before pause must not classify that exit as a crash.
|
|
1108
|
+
if (this.isPaused || this.pauseWakeState === "waking") {
|
|
1109
|
+
scheduleNext();
|
|
1110
|
+
return;
|
|
735
1111
|
}
|
|
736
|
-
scheduleNext();
|
|
737
|
-
return;
|
|
738
|
-
}
|
|
739
|
-
// A null status is ambiguous: it can be a transient `tmux list-panes`
|
|
740
|
-
// failure (e.g. tmux busy during a fleet-restart storm) rather than a
|
|
741
|
-
// real exit. Re-confirm once after a short delay before treating it as
|
|
742
|
-
// a crash. A non-null {alive:false} is a definite dead pane (real exit)
|
|
743
|
-
// and needs no recheck.
|
|
744
|
-
if (paneStatus === null) {
|
|
745
|
-
await new Promise(r => setTimeout(r, 1500));
|
|
746
|
-
paneStatus = await this.tmux.getPaneStatus();
|
|
747
1112
|
if (paneStatus?.alive) {
|
|
748
|
-
|
|
1113
|
+
// Instance output.log is fed by tmux pipe-pane and was previously never
|
|
1114
|
+
// rotated (only fleet.log / daemon.log were). Cap growth every tick.
|
|
1115
|
+
if (!this.config.lightweight) {
|
|
1116
|
+
rotateLogIfNeeded(join(this.instanceDir, "output.log"));
|
|
1117
|
+
}
|
|
749
1118
|
scheduleNext();
|
|
750
1119
|
return;
|
|
751
1120
|
}
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
this.healthCheckPaused = true;
|
|
763
|
-
return;
|
|
764
|
-
}
|
|
765
|
-
this.setProcessStatus("crashed");
|
|
766
|
-
// Distinguish tmux server crash from single window crash.
|
|
767
|
-
// nullReason records *why* getPaneStatus returned null (for diagnosing
|
|
768
|
-
// whether this was a real window loss or a transient query failure).
|
|
769
|
-
let crashType = "window";
|
|
770
|
-
let nullReason;
|
|
771
|
-
if (!paneStatus) {
|
|
772
|
-
const serverAlive = await TmuxManager.sessionExists(this.tmuxSessionName);
|
|
773
|
-
if (!serverAlive) {
|
|
774
|
-
crashType = "server";
|
|
775
|
-
nullReason = "server_gone";
|
|
776
|
-
this.logger.error(`tmux server died — all ${cliLabel} windows lost`);
|
|
777
|
-
// Fleet-level circuit breaker: pause all instances on repeated tmux server crashes
|
|
778
|
-
Daemon.tmuxServerCrashTimestamps.push(Date.now());
|
|
779
|
-
const cutoff = Date.now() - 5 * 60_000;
|
|
780
|
-
Daemon.tmuxServerCrashTimestamps = Daemon.tmuxServerCrashTimestamps.filter(t => t > cutoff);
|
|
781
|
-
if (Daemon.tmuxServerCrashTimestamps.length >= 2 && !Daemon.tmuxServerPaused) {
|
|
782
|
-
Daemon.tmuxServerPaused = true;
|
|
783
|
-
this.logger.error("Fleet-level tmux server circuit breaker triggered — pausing all respawns for 30s");
|
|
784
|
-
this.emit("tmux_server_crash", this.name);
|
|
785
|
-
if (!Daemon.tmuxServerRecoveryTimer) {
|
|
786
|
-
Daemon.tmuxServerRecoveryTimer = setTimeout(() => {
|
|
787
|
-
Daemon.tmuxServerRecoveryTimer = null;
|
|
788
|
-
Daemon.tmuxServerPaused = false;
|
|
789
|
-
}, 30_000);
|
|
790
|
-
}
|
|
1121
|
+
// A null status is ambiguous: it can be a transient `tmux list-panes`
|
|
1122
|
+
// failure (e.g. tmux busy during a fleet-restart storm) rather than a
|
|
1123
|
+
// real exit. Re-confirm once after a short delay before treating it as
|
|
1124
|
+
// a crash. A non-null {alive:false} is a definite dead pane (real exit)
|
|
1125
|
+
// and needs no recheck.
|
|
1126
|
+
if (paneStatus === null) {
|
|
1127
|
+
await new Promise(r => setTimeout(r, 1500));
|
|
1128
|
+
paneStatus = await this.tmux.getPaneStatus();
|
|
1129
|
+
if (paneStatus?.alive) {
|
|
1130
|
+
this.logger.debug(`[health] ${cliLabel} pane reported gone then alive on recheck — transient query failure, ignoring`);
|
|
791
1131
|
scheduleNext();
|
|
792
1132
|
return;
|
|
793
1133
|
}
|
|
794
|
-
await new Promise(r => setTimeout(r, 2_000)); // let session stabilize
|
|
795
1134
|
}
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
1135
|
+
// paneStatus === null → window gone entirely (e.g. tmux server crash)
|
|
1136
|
+
// paneStatus.alive === false → pane dead, exit code available
|
|
1137
|
+
const exitCode = paneStatus?.exitCode;
|
|
1138
|
+
this.logger.debug({ exitCode }, `[health] pane exited with code: ${exitCode}`);
|
|
1139
|
+
// Normal exit (e.g. user Ctrl+C or /exit) — no crash, no respawn
|
|
1140
|
+
if (paneStatus && exitCode === 0) {
|
|
1141
|
+
this.setProcessStatus("stopped");
|
|
1142
|
+
this.logger.info("CLI exited normally (code 0) — pausing health check");
|
|
1143
|
+
await this.tmux.killWindow();
|
|
1144
|
+
this.healthCheckPaused = true;
|
|
1145
|
+
this.emitSupervisionEnded("the CLI exited normally (code 0)", "Nothing crashed — start it again when you need it.");
|
|
1146
|
+
return;
|
|
1147
|
+
}
|
|
1148
|
+
// Distinguish tmux server crash from single window crash.
|
|
1149
|
+
// nullReason records *why* getPaneStatus returned null (for diagnosing
|
|
1150
|
+
// whether this was a real window loss or a transient query failure).
|
|
1151
|
+
let crashType = "window";
|
|
1152
|
+
let nullReason;
|
|
1153
|
+
if (!paneStatus) {
|
|
1154
|
+
const serverAlive = await TmuxManager.sessionExists(this.tmuxSessionName);
|
|
1155
|
+
if (!serverAlive) {
|
|
1156
|
+
crashType = "server";
|
|
1157
|
+
nullReason = "server_gone";
|
|
1158
|
+
this.logger.error(`tmux server died — all ${cliLabel} windows lost`);
|
|
1159
|
+
// Fleet-level circuit breaker: pause all instances on repeated tmux server crashes
|
|
1160
|
+
Daemon.tmuxServerCrashTimestamps.push(Date.now());
|
|
1161
|
+
const cutoff = Date.now() - 5 * 60_000;
|
|
1162
|
+
Daemon.tmuxServerCrashTimestamps = Daemon.tmuxServerCrashTimestamps.filter(t => t > cutoff);
|
|
1163
|
+
if (Daemon.tmuxServerCrashTimestamps.length >= 2 && !Daemon.tmuxServerPaused) {
|
|
1164
|
+
Daemon.tmuxServerPaused = true;
|
|
1165
|
+
this.logger.error("Fleet-level tmux server circuit breaker triggered — pausing all respawns for 30s");
|
|
1166
|
+
this.emit("tmux_server_crash", this.name);
|
|
1167
|
+
if (!Daemon.tmuxServerRecoveryTimer) {
|
|
1168
|
+
Daemon.tmuxServerRecoveryTimer = setTimeout(() => {
|
|
1169
|
+
Daemon.tmuxServerRecoveryTimer = null;
|
|
1170
|
+
Daemon.tmuxServerPaused = false;
|
|
1171
|
+
}, 30_000);
|
|
1172
|
+
}
|
|
1173
|
+
scheduleNext();
|
|
1174
|
+
return;
|
|
1175
|
+
}
|
|
1176
|
+
await new Promise(r => setTimeout(r, 2_000)); // let session stabilize
|
|
804
1177
|
}
|
|
805
|
-
|
|
806
|
-
|
|
1178
|
+
else {
|
|
1179
|
+
// null but server alive: window-level disappearance. Probe whether
|
|
1180
|
+
// the window truly no longer exists vs a transient query glitch.
|
|
1181
|
+
nullReason = "no_window";
|
|
1182
|
+
try {
|
|
1183
|
+
const windows = await TmuxManager.listWindows(this.tmuxSessionName);
|
|
1184
|
+
const currentWindowId = this.tmux.getWindowId();
|
|
1185
|
+
if (windows.some(w => w.id === currentWindowId)) {
|
|
1186
|
+
// The exact window still exists, so `list-panes` was the query
|
|
1187
|
+
// that glitched. Keep the process/state intact and retry on
|
|
1188
|
+
// the next health tick instead of killing a live Kiro TUI.
|
|
1189
|
+
this.logger.warn({ windowId: currentWindowId }, `${cliLabel} pane status unavailable but window is present — deferring crash recovery`);
|
|
1190
|
+
scheduleNext();
|
|
1191
|
+
return;
|
|
1192
|
+
}
|
|
1193
|
+
if (windows.some(w => w.name === this.name))
|
|
1194
|
+
nullReason = "same_name_other_window";
|
|
1195
|
+
}
|
|
1196
|
+
catch {
|
|
1197
|
+
nullReason = "query_error";
|
|
1198
|
+
}
|
|
1199
|
+
this.logger.warn({ exitCode, nullReason }, `${cliLabel} window not found (tmux server alive)`);
|
|
807
1200
|
}
|
|
808
|
-
this.logger.warn({ exitCode, nullReason }, `${cliLabel} window not found (tmux server alive)`);
|
|
809
1201
|
}
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
1202
|
+
else {
|
|
1203
|
+
this.logger.warn({ exitCode }, `${cliLabel} process exited`);
|
|
1204
|
+
}
|
|
1205
|
+
this.setProcessStatus("crashed");
|
|
1206
|
+
// Capture last output before killing. Best-effort even when the pane is
|
|
1207
|
+
// gone (paneStatus null) — gives the crash record something to diagnose
|
|
1208
|
+
// from instead of an empty lastOutput.
|
|
1209
|
+
let lastOutput;
|
|
1210
|
+
try {
|
|
1211
|
+
const raw = await this.tmux.capturePaneWithHistory(50);
|
|
1212
|
+
// Strip ANSI escape codes for readability
|
|
1213
|
+
const cleaned = raw.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
|
|
1214
|
+
lastOutput = cleaned.trimEnd() || undefined;
|
|
1215
|
+
}
|
|
1216
|
+
catch { /* best effort — pane may already be gone */ }
|
|
1217
|
+
// Kill the dead window (remain-on-exit keeps it around) before respawn
|
|
1218
|
+
if (paneStatus) {
|
|
1219
|
+
await this.tmux.killWindow();
|
|
1220
|
+
}
|
|
1221
|
+
// Detect claude-code background session conflict — recover without counting as crash
|
|
1222
|
+
if (lastOutput && (lastOutput.includes("background agent") || lastOutput.includes("Session is currently running"))) {
|
|
1223
|
+
if (!this.backgroundSessionRecoveryAttempted) {
|
|
1224
|
+
this.backgroundSessionRecoveryAttempted = true;
|
|
1225
|
+
this.logger.warn("Detected lingering background agent session — starting fresh (no resume)");
|
|
1226
|
+
const sidFile = join(this.instanceDir, "session-id");
|
|
1227
|
+
try {
|
|
1228
|
+
unlinkSync(sidFile);
|
|
1229
|
+
}
|
|
1230
|
+
catch { }
|
|
1231
|
+
this.skipResume = true;
|
|
1232
|
+
await new Promise(r => setTimeout(r, 2_000));
|
|
1233
|
+
try {
|
|
1234
|
+
await this.spawnClaudeWindow();
|
|
1235
|
+
this.setProcessStatus("running");
|
|
1236
|
+
this.logger.info("Recovered from background session conflict");
|
|
1237
|
+
this.emit("crash_respawn", this.name);
|
|
1238
|
+
}
|
|
1239
|
+
catch (err) {
|
|
1240
|
+
this.logger.error({ err: err.message }, "Recovery from background session conflict failed");
|
|
1241
|
+
}
|
|
1242
|
+
return; // Don't count as crash
|
|
1243
|
+
}
|
|
1244
|
+
// Already attempted recovery — fall through to normal crash handling
|
|
1245
|
+
}
|
|
1246
|
+
// Detect a --continue/--resume failure (no conversation to resume). The
|
|
1247
|
+
// session-id file persists across the crash, so a blind respawn would add
|
|
1248
|
+
// --continue again and crash in the same way → loop. Clear the session id
|
|
1249
|
+
// and skip resume so the next spawn starts fresh. (skipResume also stops
|
|
1250
|
+
// saveSessionId below from resurrecting the id from statusline.json.)
|
|
1251
|
+
if (lastOutput && /no conversation found|no conversation to (continue|resume)|no previous (session|conversation)|--continue/i.test(lastOutput)) {
|
|
1252
|
+
this.logger.warn("Detected --continue/resume failure — clearing session-id; next spawn starts fresh");
|
|
835
1253
|
try {
|
|
836
|
-
unlinkSync(
|
|
1254
|
+
unlinkSync(join(this.instanceDir, "session-id"));
|
|
837
1255
|
}
|
|
838
|
-
catch { }
|
|
1256
|
+
catch { /* may not exist */ }
|
|
839
1257
|
this.skipResume = true;
|
|
840
|
-
|
|
1258
|
+
}
|
|
1259
|
+
// Append to crash history
|
|
1260
|
+
this.appendCrashHistory({ exitCode, lastOutput, crashType, reason: nullReason });
|
|
1261
|
+
if (max_retries <= 0) {
|
|
1262
|
+
this.healthCheckPaused = true;
|
|
1263
|
+
this.logger.warn(`${cliLabel} window died — automatic restart is disabled`);
|
|
1264
|
+
this.emitSupervisionEnded("the CLI died and automatic restart is disabled (restart_policy.max_retries is 0)", "Start it manually, or raise max_retries in fleet.yaml.");
|
|
1265
|
+
return;
|
|
1266
|
+
}
|
|
1267
|
+
// Detect rapid crash: sliding window — 3+ crashes in 5 minutes
|
|
1268
|
+
this.crashTimestamps.push(Date.now());
|
|
1269
|
+
const crashWindowMs = 5 * 60_000;
|
|
1270
|
+
this.crashTimestamps = this.crashTimestamps.filter(t => t > Date.now() - crashWindowMs);
|
|
1271
|
+
if (this.crashTimestamps.length >= 3) {
|
|
1272
|
+
this.healthCheckPaused = true;
|
|
1273
|
+
this.logger.error({ crashesInWindow: this.crashTimestamps.length }, "3+ crashes in 5 minutes — pausing respawn");
|
|
1274
|
+
// P1: Persist crash state so next process restart skips resume
|
|
841
1275
|
try {
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
1276
|
+
writeFileSync(join(this.instanceDir, "crash-state.json"), JSON.stringify({
|
|
1277
|
+
crashesInWindow: this.crashTimestamps.length,
|
|
1278
|
+
lastCrashAt: Date.now(),
|
|
1279
|
+
resumeDisabled: true,
|
|
1280
|
+
}));
|
|
846
1281
|
}
|
|
847
|
-
catch
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
return; // Don't count as crash
|
|
1282
|
+
catch { /* best effort */ }
|
|
1283
|
+
this.emit("crash_loop", this.name);
|
|
1284
|
+
return; // don't schedule next — paused
|
|
851
1285
|
}
|
|
852
|
-
//
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
// session-id file persists across the crash, so a blind respawn would add
|
|
856
|
-
// --continue again and crash in the same way → loop. Clear the session id
|
|
857
|
-
// and skip resume so the next spawn starts fresh. (skipResume also stops
|
|
858
|
-
// saveSessionId below from resurrecting the id from statusline.json.)
|
|
859
|
-
if (lastOutput && /no conversation found|no conversation to (continue|resume)|no previous (session|conversation)|--continue/i.test(lastOutput)) {
|
|
860
|
-
this.logger.warn("Detected --continue/resume failure — clearing session-id; next spawn starts fresh");
|
|
861
|
-
try {
|
|
862
|
-
unlinkSync(join(this.instanceDir, "session-id"));
|
|
1286
|
+
// Reset crash count if enough time has passed
|
|
1287
|
+
if (reset_after > 0 && Date.now() - this.lastCrashAt > reset_after) {
|
|
1288
|
+
this.crashCount = 0;
|
|
863
1289
|
}
|
|
864
|
-
|
|
865
|
-
this.
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
this.logger.warn(`${cliLabel} window died — automatic restart is disabled`);
|
|
872
|
-
return;
|
|
873
|
-
}
|
|
874
|
-
// Detect rapid crash: sliding window — 3+ crashes in 5 minutes
|
|
875
|
-
this.crashTimestamps.push(Date.now());
|
|
876
|
-
const crashWindowMs = 5 * 60_000;
|
|
877
|
-
this.crashTimestamps = this.crashTimestamps.filter(t => t > Date.now() - crashWindowMs);
|
|
878
|
-
if (this.crashTimestamps.length >= 3) {
|
|
879
|
-
this.healthCheckPaused = true;
|
|
880
|
-
this.logger.error({ crashesInWindow: this.crashTimestamps.length }, "3+ crashes in 5 minutes — pausing respawn");
|
|
881
|
-
// P1: Persist crash state so next process restart skips resume
|
|
882
|
-
try {
|
|
883
|
-
writeFileSync(join(this.instanceDir, "crash-state.json"), JSON.stringify({
|
|
884
|
-
crashesInWindow: this.crashTimestamps.length,
|
|
885
|
-
lastCrashAt: Date.now(),
|
|
886
|
-
resumeDisabled: true,
|
|
887
|
-
}));
|
|
1290
|
+
this.crashCount++;
|
|
1291
|
+
this.lastCrashAt = Date.now();
|
|
1292
|
+
if (this.crashCount > max_retries) {
|
|
1293
|
+
this.logger.error({ crashCount: this.crashCount, maxRetries: max_retries }, "Max crash retries exceeded — not respawning");
|
|
1294
|
+
this.healthCheckPaused = true;
|
|
1295
|
+
this.emitSupervisionEnded(`it crashed ${this.crashCount} times, exceeding restart_policy.max_retries (${max_retries})`, "Check the logs for the cause, then restart it.");
|
|
1296
|
+
return; // don't schedule next — given up
|
|
888
1297
|
}
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
this.crashCount = 0;
|
|
896
|
-
}
|
|
897
|
-
this.crashCount++;
|
|
898
|
-
this.lastCrashAt = Date.now();
|
|
899
|
-
if (this.crashCount > max_retries) {
|
|
900
|
-
this.logger.error({ crashCount: this.crashCount, maxRetries: max_retries }, "Max crash retries exceeded — not respawning");
|
|
901
|
-
return; // don't schedule next — given up
|
|
902
|
-
}
|
|
903
|
-
// Calculate backoff delay
|
|
904
|
-
const delay = backoff === "exponential"
|
|
905
|
-
? Math.min(1000 * Math.pow(2, this.crashCount - 1), 60_000)
|
|
906
|
-
: 1000 * this.crashCount;
|
|
907
|
-
this.logger.warn({ crashCount: this.crashCount, delay }, `${cliLabel} window died — respawning after backoff`);
|
|
908
|
-
await new Promise(r => setTimeout(r, delay));
|
|
909
|
-
try {
|
|
910
|
-
this.saveSessionId();
|
|
911
|
-
this.transcriptMonitor?.resetOffset();
|
|
912
|
-
// Kill orphan MCP server from the crashed CLI session.
|
|
913
|
-
// MCP server writes its PID to channel.mcp.pid on startup.
|
|
1298
|
+
// Calculate backoff delay
|
|
1299
|
+
const delay = backoff === "exponential"
|
|
1300
|
+
? Math.min(1000 * Math.pow(2, this.crashCount - 1), 60_000)
|
|
1301
|
+
: 1000 * this.crashCount;
|
|
1302
|
+
this.logger.warn({ crashCount: this.crashCount, delay }, `${cliLabel} window died — respawning after backoff`);
|
|
1303
|
+
await new Promise(r => setTimeout(r, delay));
|
|
914
1304
|
try {
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
try {
|
|
925
|
-
const windows = await TmuxManager.listWindows(this.tmuxSessionName);
|
|
926
|
-
for (const w of windows) {
|
|
927
|
-
if (w.name === this.name) {
|
|
928
|
-
const tm = new TmuxManager(this.tmuxSessionName, w.id);
|
|
929
|
-
await tm.killWindow();
|
|
930
|
-
}
|
|
1305
|
+
this.saveSessionId();
|
|
1306
|
+
this.transcriptMonitor?.resetOffset();
|
|
1307
|
+
// Kill orphan MCP server from the crashed CLI session.
|
|
1308
|
+
// MCP server writes its PID to channel.mcp.pid on startup.
|
|
1309
|
+
try {
|
|
1310
|
+
const pidFile = join(this.instanceDir, "channel.mcp.pid");
|
|
1311
|
+
const pid = parseInt(readFileSync(pidFile, "utf-8").trim(), 10);
|
|
1312
|
+
process.kill(pid, "SIGTERM");
|
|
1313
|
+
this.logger.info({ pid }, "Killed orphan MCP server");
|
|
931
1314
|
}
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
// Try --resume first; spawnClaudeWindow falls back to fresh session if resume fails
|
|
937
|
-
const resumed = await this.spawnClaudeWindow();
|
|
938
|
-
if (!resumed) {
|
|
939
|
-
// Resume failed → fresh session → inject snapshot for context
|
|
940
|
-
await this.injectSnapshotMessage();
|
|
941
|
-
}
|
|
942
|
-
else {
|
|
943
|
-
// Clean up stale snapshot — resume restored full context
|
|
1315
|
+
catch { /* no pid file or process already dead */ }
|
|
1316
|
+
// Kill any same-name windows before respawn to prevent orphans.
|
|
1317
|
+
// Wrapped in try-catch: if tmux server is dead, listWindows throws —
|
|
1318
|
+
// must not block spawnClaudeWindow (which calls ensureSession).
|
|
944
1319
|
try {
|
|
945
|
-
|
|
1320
|
+
const windows = await TmuxManager.listWindows(this.tmuxSessionName);
|
|
1321
|
+
for (const w of windows) {
|
|
1322
|
+
if (w.name === this.name) {
|
|
1323
|
+
const tm = new TmuxManager(this.tmuxSessionName, w.id);
|
|
1324
|
+
await tm.killWindow();
|
|
1325
|
+
}
|
|
1326
|
+
}
|
|
946
1327
|
}
|
|
947
|
-
catch { /* may
|
|
1328
|
+
catch { /* tmux server may be dead — ensureSession in trySpawn will recover */ }
|
|
1329
|
+
// Write snapshot before spawn — consumed only if resume fails
|
|
1330
|
+
this.writeRotationSnapshot("crash");
|
|
1331
|
+
// Try --resume first; spawnClaudeWindow falls back to fresh session if resume fails
|
|
1332
|
+
const resumed = await this.spawnClaudeWindow();
|
|
1333
|
+
if (!resumed) {
|
|
1334
|
+
// Resume failed → fresh session → inject snapshot for context
|
|
1335
|
+
await this.injectSnapshotMessage();
|
|
1336
|
+
}
|
|
1337
|
+
else {
|
|
1338
|
+
// Clean up stale snapshot — resume restored full context
|
|
1339
|
+
try {
|
|
1340
|
+
unlinkSync(join(this.instanceDir, "rotation-state.json"));
|
|
1341
|
+
}
|
|
1342
|
+
catch { /* may not exist */ }
|
|
1343
|
+
}
|
|
1344
|
+
this.setProcessStatus("running");
|
|
1345
|
+
this.logger.info({ resumed }, `Respawned ${cliLabel} window after crash`);
|
|
1346
|
+
this.emit("crash_respawn", this.name);
|
|
1347
|
+
}
|
|
1348
|
+
catch (err) {
|
|
1349
|
+
this.logger.error({ err }, `Failed to respawn ${cliLabel} window`);
|
|
948
1350
|
}
|
|
949
|
-
this.setProcessStatus("running");
|
|
950
|
-
this.logger.info({ resumed }, `Respawned ${cliLabel} window after crash`);
|
|
951
|
-
this.emit("crash_respawn", this.name);
|
|
952
1351
|
}
|
|
953
1352
|
catch (err) {
|
|
954
|
-
this.logger.error({ err },
|
|
1353
|
+
this.logger.error({ err }, "Health check tick failed — continuing");
|
|
1354
|
+
// Surface it to the operator, not just the log — a health check that
|
|
1355
|
+
// keeps throwing means this instance is no longer being supervised.
|
|
1356
|
+
// Throttled: the tick repeats every ~30s, so an unnotified persistent
|
|
1357
|
+
// fault would otherwise post twice a minute forever.
|
|
1358
|
+
const now = Date.now();
|
|
1359
|
+
if (now - this.lastHealthErrorNotifyAt > HEALTH_ERROR_NOTIFY_INTERVAL_MS) {
|
|
1360
|
+
this.lastHealthErrorNotifyAt = now;
|
|
1361
|
+
this.emit("health_check_error", {
|
|
1362
|
+
name: this.name,
|
|
1363
|
+
message: err instanceof Error ? err.message : String(err),
|
|
1364
|
+
});
|
|
1365
|
+
}
|
|
1366
|
+
scheduleNext();
|
|
1367
|
+
return;
|
|
955
1368
|
}
|
|
956
1369
|
scheduleNext();
|
|
957
1370
|
}, healthCheckIntervalMs);
|
|
@@ -963,6 +1376,21 @@ export class Daemon extends EventEmitter {
|
|
|
963
1376
|
* A dead remain-on-exit pane still contains the old ready prompt, so allowing
|
|
964
1377
|
* the pane monitor to capture it would re-create a false idle state.
|
|
965
1378
|
*/
|
|
1379
|
+
/**
|
|
1380
|
+
* Announce that this instance is no longer being supervised.
|
|
1381
|
+
*
|
|
1382
|
+
* Four health-check exits set `healthCheckPaused = true` and returned without
|
|
1383
|
+
* telling anyone: a clean CLI exit, `max_retries <= 0`, crash retries exhausted,
|
|
1384
|
+
* and the instance directory being deleted. Only the crash-LOOP case emitted an
|
|
1385
|
+
* event, so a fleet could quietly contain a dead instance that still looked fine
|
|
1386
|
+
* on the dashboard — while messages routed to it queued or failed with a bare ❌.
|
|
1387
|
+
*
|
|
1388
|
+
* `crash_loop` already had this treatment; this is the same bridge for the other
|
|
1389
|
+
* four, carrying a human-readable cause and the operator's next step.
|
|
1390
|
+
*/
|
|
1391
|
+
emitSupervisionEnded(reason, remedy) {
|
|
1392
|
+
this.emit("supervision_ended", { name: this.name, reason, remedy });
|
|
1393
|
+
}
|
|
966
1394
|
setProcessStatus(status) {
|
|
967
1395
|
if (this.processStatus === status)
|
|
968
1396
|
return;
|
|
@@ -1001,13 +1429,12 @@ export class Daemon extends EventEmitter {
|
|
|
1001
1429
|
return;
|
|
1002
1430
|
const patterns = this.backend?.getErrorPatterns?.() ?? [];
|
|
1003
1431
|
const dialogs = this.backend?.getRuntimeDialogs?.() ?? [];
|
|
1004
|
-
if (!patterns.length && !dialogs.length)
|
|
1005
|
-
return;
|
|
1006
1432
|
if (!this.tmux)
|
|
1007
1433
|
return;
|
|
1008
1434
|
if (!this.backend)
|
|
1009
1435
|
return; // lightweight mode has no backend
|
|
1010
1436
|
const readyPattern = this.backend.getReadyPattern();
|
|
1437
|
+
const busyPattern = this.backend.getBusyPattern?.() ?? null;
|
|
1011
1438
|
this.errorMonitorTimer = setInterval(async () => {
|
|
1012
1439
|
if (!this.tmux || this.spawning)
|
|
1013
1440
|
return;
|
|
@@ -1016,24 +1443,41 @@ export class Daemon extends EventEmitter {
|
|
|
1016
1443
|
if (!alive)
|
|
1017
1444
|
return;
|
|
1018
1445
|
const pane = await this.tmux.capturePane();
|
|
1446
|
+
const interactivePrompt = this.interactivePromptDetector.observe(pane, Date.now(), this.instanceStateLastOutputAt);
|
|
1447
|
+
if (interactivePrompt) {
|
|
1448
|
+
this.logger.warn(interactivePrompt, "Interactive terminal prompt is waiting for human input");
|
|
1449
|
+
this.emit("interactive_prompt", { name: this.name, ...interactivePrompt });
|
|
1450
|
+
// A prompt is not an error and must not enter the PTY recovery gate.
|
|
1451
|
+
// Continue scanning real errors in this same snapshot.
|
|
1452
|
+
}
|
|
1019
1453
|
// Auto-dismiss runtime dialogs (e.g. Codex rate limit model switch)
|
|
1020
1454
|
for (const dialog of dialogs) {
|
|
1021
1455
|
if (!dialog.pattern.test(pane))
|
|
1022
1456
|
continue;
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1457
|
+
// These keys go straight into the pane. Sent while a message delivery is
|
|
1458
|
+
// mid-transaction, an `Escape` wipes the pasted text and an `Enter`
|
|
1459
|
+
// submits it half-composed — the user sees a message that vanished. Skip
|
|
1460
|
+
// (not queue) when the pane is busy: this poller runs every 5s, and the
|
|
1461
|
+
// dialog will still be on screen next tick.
|
|
1462
|
+
const dismissed = await this.paneWriteLock.tryRun(async () => {
|
|
1463
|
+
this.logger.info(`Auto-dismissing runtime dialog: ${dialog.description}`);
|
|
1464
|
+
const SPECIAL_KEYS = new Set(["Up", "Down", "Enter", "Escape", "Right", "Left"]);
|
|
1465
|
+
for (const key of dialog.keys) {
|
|
1466
|
+
if (SPECIAL_KEYS.has(key)) {
|
|
1467
|
+
await this.tmux.sendSpecialKey(key);
|
|
1468
|
+
}
|
|
1469
|
+
else {
|
|
1470
|
+
await this.tmux.pasteText(key, this.systemPasteOptions());
|
|
1471
|
+
}
|
|
1472
|
+
await new Promise(r => setTimeout(r, 200));
|
|
1031
1473
|
}
|
|
1032
|
-
|
|
1474
|
+
});
|
|
1475
|
+
if (!dismissed) {
|
|
1476
|
+
this.logger.debug({ dialog: dialog.description }, "Dialog dismissal deferred — pane write in flight");
|
|
1033
1477
|
}
|
|
1034
|
-
return; // Dialog
|
|
1478
|
+
return; // Dialog handled (or deliberately deferred): skip error checks this cycle
|
|
1035
1479
|
}
|
|
1036
|
-
this.evaluateErrorPatterns(pane, patterns, readyPattern);
|
|
1480
|
+
this.evaluateErrorPatterns(pane, patterns, readyPattern, Date.now(), busyPattern);
|
|
1037
1481
|
}
|
|
1038
1482
|
catch {
|
|
1039
1483
|
// capturePane can fail if window is transitioning — ignore
|
|
@@ -1041,7 +1485,7 @@ export class Daemon extends EventEmitter {
|
|
|
1041
1485
|
}, 5_000); // Check every 5 seconds (runtime dialogs need fast response)
|
|
1042
1486
|
}
|
|
1043
1487
|
/** Evaluate one pane snapshot. Kept synchronous so state-machine edges are unit-testable. */
|
|
1044
|
-
evaluateErrorPatterns(pane, patterns, readyPattern, now = Date.now()) {
|
|
1488
|
+
evaluateErrorPatterns(pane, patterns, readyPattern, now = Date.now(), busyPattern = null) {
|
|
1045
1489
|
// Count occurrences across the WHOLE pane (not just text after the last
|
|
1046
1490
|
// ready prompt). Clone with `g` so stateful backend regexes cannot leak
|
|
1047
1491
|
// lastIndex between monitor cycles.
|
|
@@ -1049,14 +1493,34 @@ export class Daemon extends EventEmitter {
|
|
|
1049
1493
|
const flags = pattern.flags.includes("g") ? pattern.flags : pattern.flags + "g";
|
|
1050
1494
|
return (pane.match(new RegExp(pattern.source, flags)) || []).length;
|
|
1051
1495
|
};
|
|
1496
|
+
// Same veto as the state machine: a backend whose ready marker is permanently
|
|
1497
|
+
// on screen would otherwise "recover" on the very first tick after the error,
|
|
1498
|
+
// rebaselining the occurrence count while the error is still displayed — one
|
|
1499
|
+
// notification, a false recovery log, then silence.
|
|
1500
|
+
const looksReady = () => !busyPattern?.test(pane) && readyPattern.test(pane);
|
|
1052
1501
|
// State: waiting for recovery. A missing/outdated ready pattern must not
|
|
1053
1502
|
// suppress every future error forever, so the gate has a hard deadline.
|
|
1054
1503
|
if (this.errorWaitingForRecovery) {
|
|
1055
|
-
if (
|
|
1504
|
+
if (looksReady()) {
|
|
1056
1505
|
const downtime = Math.round((now - this.errorDetectedAt) / 1000);
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1506
|
+
// Re-baseline EVERY pattern, not just the one that fired. Patterns
|
|
1507
|
+
// legitimately overlap — kiro prints one header ("having trouble
|
|
1508
|
+
// responding") above the specific cause — and only the first match is
|
|
1509
|
+
// reported before the `break`, leaving the others at a stale baseline.
|
|
1510
|
+
// Rebasing just the active one meant the first scan after recovery saw
|
|
1511
|
+
// count > seen on the very text we already reported and fired a second,
|
|
1512
|
+
// contradictory notification ("Rate limit") for an incident the user had
|
|
1513
|
+
// just been told about. Recovery means the pane is now history, so all
|
|
1514
|
+
// of it is history. Deliberately NOT done at detection time: a pattern
|
|
1515
|
+
// skipped for cooldown must keep its count unconsumed so it can still
|
|
1516
|
+
// fire once that cooldown expires.
|
|
1517
|
+
for (const ep of patterns) {
|
|
1518
|
+
const seen = countMatches(ep.pattern);
|
|
1519
|
+
// Only patterns actually present in the pane. Storing zeros would be a
|
|
1520
|
+
// no-op for detection (count 0 === absent) but grows the map with an
|
|
1521
|
+
// entry per pattern per recovery.
|
|
1522
|
+
if (seen > 0)
|
|
1523
|
+
this.lastErrorCount.set(Daemon.errorPatternKey(ep), seen);
|
|
1060
1524
|
}
|
|
1061
1525
|
this.clearErrorRecoveryGate();
|
|
1062
1526
|
this.logger.info({ downtime_s: downtime }, "PTY error recovered — agent is ready again");
|
|
@@ -1151,6 +1615,12 @@ export class Daemon extends EventEmitter {
|
|
|
1151
1615
|
* Interrupt the CLI's current generation (cancel button / `/cancel`).
|
|
1152
1616
|
* Direct tmux key event (not a paste) so it registers as the interrupt key.
|
|
1153
1617
|
* kiro-cli interrupts on Ctrl+C; the others (claude-code, codex, …) on Escape.
|
|
1618
|
+
*
|
|
1619
|
+
* Deliberately NOT taken through `paneWriteLock`. Cancel is the user's way out
|
|
1620
|
+
* of a pane that is busy or wedged — queueing it behind the very delivery chain
|
|
1621
|
+
* it exists to unblock would make the button useless exactly when it is needed.
|
|
1622
|
+
* The cost is accepted: an Escape landing between a paste and its Enter discards
|
|
1623
|
+
* that message, which is what the user asked for anyway.
|
|
1154
1624
|
*/
|
|
1155
1625
|
async sendEscape() {
|
|
1156
1626
|
const cancelKey = this.backend?.getCancelKey() ?? "Escape";
|
|
@@ -1188,7 +1658,6 @@ export class Daemon extends EventEmitter {
|
|
|
1188
1658
|
this.toolStatusDebounce = null;
|
|
1189
1659
|
}
|
|
1190
1660
|
this.pendingIpcRequests.clear();
|
|
1191
|
-
this.hangDetector?.stop();
|
|
1192
1661
|
if (this.adapter)
|
|
1193
1662
|
await this.adapter.stop();
|
|
1194
1663
|
// Notify MCP servers of graceful shutdown (prevents reconnect attempts)
|
|
@@ -1215,7 +1684,12 @@ export class Daemon extends EventEmitter {
|
|
|
1215
1684
|
if (!killed)
|
|
1216
1685
|
this.logger.warn("CLI did not exit gracefully within 3s, force killing window");
|
|
1217
1686
|
// Always kill window — remain-on-exit keeps dead panes around after CLI exits
|
|
1687
|
+
const stoppedWindowId = this.tmux.getWindowId();
|
|
1218
1688
|
await this.tmux.killWindow();
|
|
1689
|
+
// The control client is shared across the fleet and outlives this daemon, so
|
|
1690
|
+
// a registration left behind here is re-resolved on every reconnect forever.
|
|
1691
|
+
if (stoppedWindowId)
|
|
1692
|
+
this.controlClient?.unregisterWindow(stoppedWindowId);
|
|
1219
1693
|
const windowIdFile = join(this.instanceDir, "window-id");
|
|
1220
1694
|
try {
|
|
1221
1695
|
unlinkSync(windowIdFile);
|
|
@@ -1258,6 +1732,16 @@ export class Daemon extends EventEmitter {
|
|
|
1258
1732
|
};
|
|
1259
1733
|
}
|
|
1260
1734
|
/** Gracefully stop the CLI while keeping its remain-on-exit tmux window. */
|
|
1735
|
+
/**
|
|
1736
|
+
* Mark the instance to pause as soon as its pane is idle. Used for auth
|
|
1737
|
+
* failures: pause() alone no-ops while the CLI is busy/stuck, which is the
|
|
1738
|
+
* usual state when the error surfaces. Cleared by a successful pause or wake.
|
|
1739
|
+
*/
|
|
1740
|
+
requestPauseWhenIdle() {
|
|
1741
|
+
if (this.pauseWakeState === "paused")
|
|
1742
|
+
return;
|
|
1743
|
+
this.pausePending = true;
|
|
1744
|
+
}
|
|
1261
1745
|
async pause() {
|
|
1262
1746
|
if (this.pauseWakeState === "paused")
|
|
1263
1747
|
return;
|
|
@@ -1327,6 +1811,9 @@ export class Daemon extends EventEmitter {
|
|
|
1327
1811
|
}
|
|
1328
1812
|
/** Respawn the CLI in the preserved window and block until its prompt is ready. */
|
|
1329
1813
|
async wake(timeoutMs = 30_000) {
|
|
1814
|
+
// An explicit wake (e.g. after the user re-logs in) cancels a deferred
|
|
1815
|
+
// auth pause — otherwise the instance would pause again the moment it idles.
|
|
1816
|
+
this.pausePending = false;
|
|
1330
1817
|
if (this.pauseWakeState === "active")
|
|
1331
1818
|
return;
|
|
1332
1819
|
if (this.pauseWakeState === "pausing")
|
|
@@ -1338,7 +1825,7 @@ export class Daemon extends EventEmitter {
|
|
|
1338
1825
|
return;
|
|
1339
1826
|
}
|
|
1340
1827
|
this.pauseWakeState = "waking";
|
|
1341
|
-
this.
|
|
1828
|
+
this.beginSpawn();
|
|
1342
1829
|
const transition = this.autoPauseController.wakeOnDeliver(async () => {
|
|
1343
1830
|
let timeout;
|
|
1344
1831
|
try {
|
|
@@ -1379,7 +1866,7 @@ export class Daemon extends EventEmitter {
|
|
|
1379
1866
|
throw err;
|
|
1380
1867
|
}
|
|
1381
1868
|
finally {
|
|
1382
|
-
this.
|
|
1869
|
+
this.endSpawn();
|
|
1383
1870
|
if (this.pauseWakeTransition === transition)
|
|
1384
1871
|
this.pauseWakeTransition = null;
|
|
1385
1872
|
}
|
|
@@ -1391,6 +1878,10 @@ export class Daemon extends EventEmitter {
|
|
|
1391
1878
|
// observations between enqueue and paste must not clear a newer inbound.
|
|
1392
1879
|
if (snapshot.state === "idle" && previous !== "idle") {
|
|
1393
1880
|
this.pendingWork.recordIdle(snapshot.observedAt);
|
|
1881
|
+
// The turn is over. A transcript can end on a tool_use with no matching
|
|
1882
|
+
// tool_result (interrupted, crashed, cancelled), which would otherwise leave
|
|
1883
|
+
// the last tool pinned to the progress line for the rest of the session.
|
|
1884
|
+
this.publishActivity(null);
|
|
1394
1885
|
}
|
|
1395
1886
|
if (snapshot.state !== previous) {
|
|
1396
1887
|
this.logger.info({
|
|
@@ -1406,6 +1897,18 @@ export class Daemon extends EventEmitter {
|
|
|
1406
1897
|
}
|
|
1407
1898
|
if (snapshot.state !== "idle")
|
|
1408
1899
|
this.pauseRequested = false;
|
|
1900
|
+
// A pause deferred by an auth failure: retry the moment the pane settles.
|
|
1901
|
+
if (this.pausePending && snapshot.state === "idle" && this.pasteQueueDepth === 0) {
|
|
1902
|
+
this.pausePending = false;
|
|
1903
|
+
this.emit("auto_pause_requested", { name: this.name, idleSince: snapshot.stateChangedAt });
|
|
1904
|
+
return;
|
|
1905
|
+
}
|
|
1906
|
+
// A restart deferred by a dead MCP server: the turn it must not interrupt is
|
|
1907
|
+
// over. Skip auto-pause evaluation this tick — the restart supersedes it.
|
|
1908
|
+
if (this.mcpRestartPending && snapshot.state === "idle" && this.pasteQueueDepth === 0) {
|
|
1909
|
+
this.fireMcpRestartRequest("idle_edge");
|
|
1910
|
+
return;
|
|
1911
|
+
}
|
|
1409
1912
|
if (!this.pauseRequested && this.pasteQueueDepth === 0 && this.autoPauseController.observe(snapshot.state)) {
|
|
1410
1913
|
this.pauseRequested = true;
|
|
1411
1914
|
this.emit("auto_pause_requested", { name: this.name, idleSince: snapshot.stateChangedAt });
|
|
@@ -1472,9 +1975,29 @@ export class Daemon extends EventEmitter {
|
|
|
1472
1975
|
|| (this.instanceStateLastOutputAt > 0 && this.instanceStateLastOutputAt >= captureStartedAt))
|
|
1473
1976
|
return;
|
|
1474
1977
|
const observedChangeAt = expectedOutputAt || captureStartedAt;
|
|
1475
|
-
|
|
1476
|
-
|
|
1978
|
+
// One observation per capture. This used to call observe() twice with the
|
|
1979
|
+
// same pane: the second call always saw an unchanged pane, so the
|
|
1980
|
+
// "content moved → still working" branch could never fire and the state
|
|
1981
|
+
// came down to the busy/ready patterns alone. That is why a single
|
|
1982
|
+
// mismatched spinner frame was enough to report idle mid-turn.
|
|
1983
|
+
const settled = this.instanceStateLastOutputAt === 0
|
|
1984
|
+
|| captureStartedAt - this.instanceStateLastOutputAt >= this.instanceStateIdleDebounceMs;
|
|
1985
|
+
// The two times are genuinely different and both matter: the content
|
|
1986
|
+
// changed when tmux reported output (observedChangeAt), but idle/stuck are
|
|
1987
|
+
// decisions about *now*. The old double-observe expressed that by calling
|
|
1988
|
+
// observe twice, which silently disabled the "content moved" branch.
|
|
1989
|
+
const snapshot = this.instanceStateMachine.observe(pane, Date.now(), {
|
|
1990
|
+
settled,
|
|
1991
|
+
changeAt: observedChangeAt,
|
|
1992
|
+
});
|
|
1477
1993
|
this.applyInstanceStateSnapshot(snapshot, pane);
|
|
1994
|
+
// Backends without a transcript feed can still say what they are doing, if
|
|
1995
|
+
// their TUI names it. Free-riding on a capture that already happened: no
|
|
1996
|
+
// extra tmux call. Cadence is the capture cadence (idle debounce + the 60s
|
|
1997
|
+
// safety sweep), which matches the progress ticker's own 60s interval.
|
|
1998
|
+
if (this.backend?.getPaneActivity) {
|
|
1999
|
+
this.publishActivity(snapshot.state === "idle" ? null : this.backend.getPaneActivity(pane));
|
|
2000
|
+
}
|
|
1478
2001
|
if (snapshot.state === "idle") {
|
|
1479
2002
|
this.clearInstanceStateStuckTimer();
|
|
1480
2003
|
}
|
|
@@ -1529,7 +2052,8 @@ export class Daemon extends EventEmitter {
|
|
|
1529
2052
|
? rawConfig.idle_debounce_ms
|
|
1530
2053
|
: DEFAULT_STATE_IDLE_DEBOUNCE_MS;
|
|
1531
2054
|
this.instanceStateReadyPattern = this.backend.getReadyPattern();
|
|
1532
|
-
this.
|
|
2055
|
+
this.instanceStateBusyPattern = this.backend.getBusyPattern?.() ?? null;
|
|
2056
|
+
this.instanceStateMachine = new PaneStateMachine(this.instanceStateReadyPattern, this.instanceStateStuckTimeoutMs, Date.now(), this.instanceStateBusyPattern);
|
|
1533
2057
|
this.instanceStateLastOutputAt = 0;
|
|
1534
2058
|
this.instanceStateMonitorActive = true;
|
|
1535
2059
|
if (this.controlClient) {
|
|
@@ -1570,6 +2094,12 @@ export class Daemon extends EventEmitter {
|
|
|
1570
2094
|
paneTail: sanitizePaneTail(pane),
|
|
1571
2095
|
readyPattern: readyPattern.toString(),
|
|
1572
2096
|
readyMatched: deterministicReadyPattern.test(pane),
|
|
2097
|
+
// A pane that is stuck *while* the ready marker matches means the busy
|
|
2098
|
+
// pattern is what held it back — worth seeing when tuning either regex.
|
|
2099
|
+
busyPattern: this.instanceStateBusyPattern?.toString(),
|
|
2100
|
+
busyMatched: this.instanceStateBusyPattern
|
|
2101
|
+
? new RegExp(this.instanceStateBusyPattern.source, this.instanceStateBusyPattern.flags.replace(/[gy]/g, "")).test(pane)
|
|
2102
|
+
: undefined,
|
|
1573
2103
|
unchangedForMs: snapshot.unchangedForMs,
|
|
1574
2104
|
pendingWork: this.pendingWork.hasPendingWork(),
|
|
1575
2105
|
};
|
|
@@ -1591,6 +2121,11 @@ export class Daemon extends EventEmitter {
|
|
|
1591
2121
|
clearInterval(this.errorMonitorTimer);
|
|
1592
2122
|
this.errorMonitorTimer = null;
|
|
1593
2123
|
}
|
|
2124
|
+
// A pause or stop tears the CLI down anyway — the respawn brings a fresh MCP
|
|
2125
|
+
// server, so a deferred MCP-revival restart is moot (and its timer must not
|
|
2126
|
+
// fire into a stopped daemon).
|
|
2127
|
+
this.clearMcpRestartRequest();
|
|
2128
|
+
this.interactivePromptDetector.reset();
|
|
1594
2129
|
this.stopInstanceStateMonitor();
|
|
1595
2130
|
this.transcriptMonitor?.stop();
|
|
1596
2131
|
this.guardian?.stop();
|
|
@@ -1612,6 +2147,40 @@ export class Daemon extends EventEmitter {
|
|
|
1612
2147
|
return this.messageBus;
|
|
1613
2148
|
}
|
|
1614
2149
|
// ── Tool status tracking ──────────────────────────────────────
|
|
2150
|
+
/**
|
|
2151
|
+
* Tell the fleet manager what this instance is doing right now, for the live
|
|
2152
|
+
* progress line on the cancel button.
|
|
2153
|
+
*
|
|
2154
|
+
* Purely cosmetic — nothing decides anything from it, so it is fine that only
|
|
2155
|
+
* backends with a transcript feed report at all (claude-code today). For the
|
|
2156
|
+
* rest the progress line keeps showing just elapsed time.
|
|
2157
|
+
*
|
|
2158
|
+
* Repeats are dropped: the ticker only edits the channel message when the text
|
|
2159
|
+
* changes, and a stream of identical broadcasts would defeat that.
|
|
2160
|
+
*/
|
|
2161
|
+
publishActivity(activity) {
|
|
2162
|
+
const next = activity && activity.trim() ? activity.trim() : null;
|
|
2163
|
+
if (next === this.currentActivity)
|
|
2164
|
+
return;
|
|
2165
|
+
this.currentActivity = next;
|
|
2166
|
+
this.ipcServer?.broadcast({
|
|
2167
|
+
type: "instance_activity",
|
|
2168
|
+
instanceName: this.name,
|
|
2169
|
+
activity: next,
|
|
2170
|
+
});
|
|
2171
|
+
}
|
|
2172
|
+
/**
|
|
2173
|
+
* Options for every system-initiated paste (startup notice, session snapshot,
|
|
2174
|
+
* runtime-dialog keys).
|
|
2175
|
+
*
|
|
2176
|
+
* One place, so the three call sites cannot drift apart on it. Backends with a
|
|
2177
|
+
* native input queue do not get the retry Enter: on those, a second bare Enter
|
|
2178
|
+
* is not the no-op the retry assumed but a queue mutation — the same reason
|
|
2179
|
+
* `deliverMessage` refuses to probe for busy with one.
|
|
2180
|
+
*/
|
|
2181
|
+
systemPasteOptions() {
|
|
2182
|
+
return { retryEnter: this.backend?.supportsQueuedInput?.() !== true };
|
|
2183
|
+
}
|
|
1615
2184
|
summarizeTool(name, input) {
|
|
1616
2185
|
const inp = input;
|
|
1617
2186
|
if (!inp)
|
|
@@ -1622,8 +2191,9 @@ export class Daemon extends EventEmitter {
|
|
|
1622
2191
|
return `Edit ${inp.file_path ?? ""}`;
|
|
1623
2192
|
if (name === "Write")
|
|
1624
2193
|
return `Write ${inp.file_path ?? ""}`;
|
|
2194
|
+
// Command name only — never the arguments. See shellCommandLabel.
|
|
1625
2195
|
if (name === "Bash")
|
|
1626
|
-
return `$ ${String(inp.command ?? "")
|
|
2196
|
+
return `$ ${shellCommandLabel(String(inp.command ?? ""))}`;
|
|
1627
2197
|
if (name === "Glob")
|
|
1628
2198
|
return `Glob ${inp.pattern ?? ""}`;
|
|
1629
2199
|
if (name === "Grep")
|
|
@@ -1701,7 +2271,6 @@ export class Daemon extends EventEmitter {
|
|
|
1701
2271
|
writeFileSync(join(this.instanceDir, "prev-instructions"), this.pendingInstructionsUpdate);
|
|
1702
2272
|
this.pendingInstructionsUpdate = undefined;
|
|
1703
2273
|
}
|
|
1704
|
-
this.hangDetector?.recordInbound();
|
|
1705
2274
|
this.pendingWork.recordInbound();
|
|
1706
2275
|
// v3: record user messages for rotation snapshot
|
|
1707
2276
|
this.recordRecentUserMessage(content, meta);
|
|
@@ -1724,12 +2293,26 @@ export class Daemon extends EventEmitter {
|
|
|
1724
2293
|
// #77: show the sender's display name for readability, keeping the machine
|
|
1725
2294
|
// instance name in parens so the recipient's send_to_instance target is valid.
|
|
1726
2295
|
const fromLabel = meta.from_display ? `${meta.from_display} (${fromInstance})` : fromInstance;
|
|
1727
|
-
formatted = `[from:${fromLabel}] ${content}
|
|
2296
|
+
formatted = `[from:${fromLabel}] ${content}`;
|
|
2297
|
+
formatted += renderHandoffMetadata(meta);
|
|
2298
|
+
// A delegated task that requires a reply must not read like a chatty FYI —
|
|
2299
|
+
// the "you may stay silent" line is for the latter only.
|
|
2300
|
+
formatted += meta.requires_reply === "true"
|
|
2301
|
+
? "\n(A reply IS required: use report_result with the correlation_id above — or send_to_instance. Not direct text.)"
|
|
2302
|
+
: "\n(If you need to reply, use send_to_instance tool, NOT direct text. If there is nothing to add, you may stay silent.)";
|
|
1728
2303
|
}
|
|
1729
2304
|
else {
|
|
1730
2305
|
const via = meta.source ? ` via ${meta.source}` : "";
|
|
1731
2306
|
const idTag = meta.user_id ? `, id:${meta.user_id}` : "";
|
|
1732
|
-
formatted = `[user:${user}${via}${idTag}] ${content}
|
|
2307
|
+
formatted = `[user:${user}${via}${idTag}] ${content}`;
|
|
2308
|
+
// Reactions queued since the last real message (#432). One leading line of
|
|
2309
|
+
// context, present only when something is pending — a reaction no longer
|
|
2310
|
+
// costs a turn of its own.
|
|
2311
|
+
if (meta.pending_reactions) {
|
|
2312
|
+
formatted = `[Recent reactions: ${meta.pending_reactions}]\n${formatted}`;
|
|
2313
|
+
}
|
|
2314
|
+
formatted += renderHandoffMetadata(meta);
|
|
2315
|
+
formatted += "\n(Reply using the reply tool — do NOT respond with direct text)";
|
|
1733
2316
|
}
|
|
1734
2317
|
if (meta.reply_to_text) {
|
|
1735
2318
|
formatted += `\n(reply_to: "${meta.reply_to_text}")`;
|
|
@@ -1793,6 +2376,8 @@ export class Daemon extends EventEmitter {
|
|
|
1793
2376
|
// Odd number of fences = unclosed. Remove all code fences from the message.
|
|
1794
2377
|
formatted = formatted.replace(/```/g, "");
|
|
1795
2378
|
}
|
|
2379
|
+
// Before anything reads the window id: a spawn in progress is about to change it.
|
|
2380
|
+
await this.waitForSpawnToSettle();
|
|
1796
2381
|
let windowId = this.getWindowId();
|
|
1797
2382
|
const supportsQueuedInput = this.backend?.supportsQueuedInput?.() === true;
|
|
1798
2383
|
let handingOffToNativeQueue = false;
|
|
@@ -1808,9 +2393,33 @@ export class Daemon extends EventEmitter {
|
|
|
1808
2393
|
}
|
|
1809
2394
|
else {
|
|
1810
2395
|
this.logger.debug("CLI busy — queuing message until idle");
|
|
1811
|
-
await this.controlClient.waitUntilIdle(windowId);
|
|
2396
|
+
const becameIdle = await this.controlClient.waitUntilIdle(windowId);
|
|
2397
|
+
if (!becameIdle) {
|
|
2398
|
+
// The pane never freed up. Report the failure instead of pasting into a
|
|
2399
|
+
// wedged CLI (where the text would sit unsubmitted and the next message
|
|
2400
|
+
// would land on top of it) — and instead of holding the queue silently.
|
|
2401
|
+
this.logger.error("Pane still busy after the idle wait — reporting delivery failure");
|
|
2402
|
+
if (status)
|
|
2403
|
+
this.emit("message_failed", status); // ❌
|
|
2404
|
+
return false;
|
|
2405
|
+
}
|
|
1812
2406
|
}
|
|
1813
2407
|
}
|
|
2408
|
+
// Everything above is *waiting*; everything below *writes*. Only the write is
|
|
2409
|
+
// held under the pane lock — holding it across the idle wait (up to 30 min)
|
|
2410
|
+
// would starve the runtime-dialog dismisser, which is often the very thing
|
|
2411
|
+
// that would let the pane go idle again.
|
|
2412
|
+
return this.paneWriteLock.run(() => this.writeMessageToPane(formatted, windowId, handingOffToNativeQueue, status));
|
|
2413
|
+
}
|
|
2414
|
+
/**
|
|
2415
|
+
* The write half of a delivery: paste → Enter → confirm, with retries.
|
|
2416
|
+
*
|
|
2417
|
+
* Always called under {@link paneWriteLock}. Split out from `deliverMessage`
|
|
2418
|
+
* precisely so the lock's scope is visible at the call site rather than being
|
|
2419
|
+
* an invariant maintained by comments.
|
|
2420
|
+
*/
|
|
2421
|
+
async writeMessageToPane(formatted, initialWindowId, handingOffToNativeQueue, status) {
|
|
2422
|
+
let windowId = initialWindowId;
|
|
1814
2423
|
// Bug A: paste with backoff. Transient failures are usually a stale window id
|
|
1815
2424
|
// after a crash/respawn — recover by name and retry (max 3 attempts, 2s apart).
|
|
1816
2425
|
const maxAttempts = 3;
|
|
@@ -1831,19 +2440,79 @@ export class Daemon extends EventEmitter {
|
|
|
1831
2440
|
this.logger.debug({ enterSettleMs }, "First delivery after ready — extending Enter settle delay");
|
|
1832
2441
|
}
|
|
1833
2442
|
await new Promise(r => setTimeout(r, enterSettleMs));
|
|
1834
|
-
|
|
2443
|
+
let enterAt = Date.now();
|
|
1835
2444
|
await this.tmux.sendSpecialKey("Enter");
|
|
2445
|
+
// Kiro's legacy TUI can swallow Enter while it is still processing a large
|
|
2446
|
+
// paste — not only during the post-ready redraw (#479): on slower hosts it
|
|
2447
|
+
// happens on ordinary deliveries too (v2.1.2 stable, WSL2 + tmux 3.4).
|
|
2448
|
+
// The busy confirmation below cannot be trusted to catch that: it accepts
|
|
2449
|
+
// ANY output after Enter, and the paste's own late render satisfies it, so
|
|
2450
|
+
// the message is confirmed ✅ while the text sits unsubmitted. Kiro has no
|
|
2451
|
+
// native input queue and a bare Enter is a no-op both at an empty prompt
|
|
2452
|
+
// and during generation (both verified live on kiro-cli 2.16.1), so the
|
|
2453
|
+
// defensive retry runs on EVERY delivery — which is also what the legacy
|
|
2454
|
+
// pasteText path has always done for queue-less backends. enterAt is
|
|
2455
|
+
// re-baselined to the second Enter so leftover paste-render output between
|
|
2456
|
+
// the two cannot be what "confirms" the submission.
|
|
2457
|
+
if (this.backend?.requiresDeliveryEnterRetry?.() === true) {
|
|
2458
|
+
await new Promise(r => setTimeout(r, 1_000));
|
|
2459
|
+
enterAt = Date.now();
|
|
2460
|
+
await this.tmux.sendSpecialKey("Enter");
|
|
2461
|
+
this.logger.debug("Sent defensive Enter retry (queue-less TUI can swallow the first)");
|
|
2462
|
+
}
|
|
1836
2463
|
if (status)
|
|
1837
2464
|
this.emit("message_delivered", status); // 👀
|
|
1838
|
-
//
|
|
1839
|
-
//
|
|
1840
|
-
//
|
|
1841
|
-
//
|
|
2465
|
+
// Busy queue-capable CLIs (codex) may accept paste without an idle→busy
|
|
2466
|
+
// transition. Do NOT probe for busy — a second bare Enter can mutate the
|
|
2467
|
+
// queue. Instead verify the paste actually landed in the pane (TUI redraw
|
|
2468
|
+
// can silently swallow it). Idle submissions keep the swallowed-Enter path.
|
|
1842
2469
|
if (handingOffToNativeQueue) {
|
|
2470
|
+
await new Promise(r => setTimeout(r, NATIVE_QUEUE_PASTE_VERIFY_MS));
|
|
2471
|
+
if (await this.nativeQueuePasteVisible(formatted)) {
|
|
2472
|
+
if (status)
|
|
2473
|
+
this.emit("message_confirmed", status); // ✅ native queue accepted
|
|
2474
|
+
return true;
|
|
2475
|
+
}
|
|
2476
|
+
// Silent loss: fall back once to the normal idle-gated path.
|
|
2477
|
+
this.logger.warn("Native-queue paste not visible in pane — retrying via idle-gated delivery");
|
|
2478
|
+
if (windowId && this.controlClient) {
|
|
2479
|
+
await this.controlClient.waitUntilIdle(windowId);
|
|
2480
|
+
}
|
|
2481
|
+
const repasted = await this.tmux.pasteBuffer(formatted);
|
|
2482
|
+
if (!repasted) {
|
|
2483
|
+
this.logger.error("Idle-gated redelivery paste failed after native-queue silent loss");
|
|
2484
|
+
if (status)
|
|
2485
|
+
this.emit("message_failed", status); // ❌
|
|
2486
|
+
return false;
|
|
2487
|
+
}
|
|
2488
|
+
await new Promise(r => setTimeout(r, NORMAL_ENTER_SETTLE_MS));
|
|
2489
|
+
const retryAt = Date.now();
|
|
2490
|
+
await this.tmux.sendSpecialKey("Enter");
|
|
2491
|
+
if (windowId && this.controlClient) {
|
|
2492
|
+
let becameBusy = await this.confirmBusyAfterEnter(windowId, retryAt);
|
|
2493
|
+
if (!becameBusy) {
|
|
2494
|
+
this.logger.warn("No idle→busy after idle-gated redelivery — re-sending Enter once");
|
|
2495
|
+
const retry2At = Date.now();
|
|
2496
|
+
await this.tmux.sendSpecialKey("Enter");
|
|
2497
|
+
becameBusy = await this.confirmBusyAfterEnter(windowId, retry2At);
|
|
2498
|
+
}
|
|
2499
|
+
if (becameBusy) {
|
|
2500
|
+
if (status)
|
|
2501
|
+
this.emit("message_confirmed", status); // ✅
|
|
2502
|
+
return true;
|
|
2503
|
+
}
|
|
2504
|
+
}
|
|
2505
|
+
else if (await this.nativeQueuePasteVisible(formatted)) {
|
|
2506
|
+
if (status)
|
|
2507
|
+
this.emit("message_confirmed", status); // ✅
|
|
2508
|
+
return true;
|
|
2509
|
+
}
|
|
2510
|
+
this.logger.error("Idle-gated redelivery also failed after native-queue silent loss");
|
|
1843
2511
|
if (status)
|
|
1844
|
-
this.emit("
|
|
2512
|
+
this.emit("message_failed", status); // ❌
|
|
2513
|
+
return false;
|
|
1845
2514
|
}
|
|
1846
|
-
|
|
2515
|
+
if (windowId && this.controlClient) {
|
|
1847
2516
|
let becameBusy = await this.confirmBusyAfterEnter(windowId, enterAt);
|
|
1848
2517
|
if (!becameBusy) {
|
|
1849
2518
|
this.logger.warn("No idle→busy transition after Enter — re-sending Enter once");
|
|
@@ -1851,8 +2520,20 @@ export class Daemon extends EventEmitter {
|
|
|
1851
2520
|
await this.tmux.sendSpecialKey("Enter");
|
|
1852
2521
|
becameBusy = await this.confirmBusyAfterEnter(windowId, retryAt);
|
|
1853
2522
|
}
|
|
1854
|
-
if (becameBusy
|
|
1855
|
-
|
|
2523
|
+
if (becameBusy) {
|
|
2524
|
+
if (status)
|
|
2525
|
+
this.emit("message_confirmed", status); // ✅
|
|
2526
|
+
}
|
|
2527
|
+
else {
|
|
2528
|
+
// Both Enters were swallowed: the text is sitting UNSUBMITTED in the
|
|
2529
|
+
// CLI's input box. This used to return true, so the reaction stayed at 👀
|
|
2530
|
+
// forever and the next delivery pasted on top — submitting two messages
|
|
2531
|
+
// as one. Say so instead.
|
|
2532
|
+
this.logger.error("Message pasted but never submitted (no idle→busy after two Enters)");
|
|
2533
|
+
if (status)
|
|
2534
|
+
this.emit("message_failed", status); // ❌
|
|
2535
|
+
return false;
|
|
2536
|
+
}
|
|
1856
2537
|
}
|
|
1857
2538
|
else {
|
|
1858
2539
|
// No control client to observe output: fall back to the legacy double-Enter.
|
|
@@ -1868,8 +2549,36 @@ export class Daemon extends EventEmitter {
|
|
|
1868
2549
|
this.emit("message_failed", status); // ❌
|
|
1869
2550
|
return false;
|
|
1870
2551
|
}
|
|
2552
|
+
/**
|
|
2553
|
+
* True when a busy native-queue paste appears to have landed: Codex shows a
|
|
2554
|
+
* `↳` queue marker, or a distinctive slice of the pasted text is on screen.
|
|
2555
|
+
* Used only to detect silent paste loss — not as a general ready check.
|
|
2556
|
+
*/
|
|
2557
|
+
async nativeQueuePasteVisible(formatted) {
|
|
2558
|
+
if (!this.tmux)
|
|
2559
|
+
return false;
|
|
2560
|
+
try {
|
|
2561
|
+
const pane = await this.tmux.capturePane();
|
|
2562
|
+
if (pane.includes("↳"))
|
|
2563
|
+
return true;
|
|
2564
|
+
for (const line of formatted.split(/\r?\n/)) {
|
|
2565
|
+
const t = line.trim();
|
|
2566
|
+
if (t.length >= 8 && pane.includes(t))
|
|
2567
|
+
return true;
|
|
2568
|
+
}
|
|
2569
|
+
const compact = formatted.replace(/\s+/g, " ").trim();
|
|
2570
|
+
if (compact.length >= 8 && pane.includes(compact.slice(0, Math.min(80, compact.length)))) {
|
|
2571
|
+
return true;
|
|
2572
|
+
}
|
|
2573
|
+
return false;
|
|
2574
|
+
}
|
|
2575
|
+
catch {
|
|
2576
|
+
return false;
|
|
2577
|
+
}
|
|
2578
|
+
}
|
|
1871
2579
|
/** Re-resolve this instance's tmux window by name (stale id after crash/respawn). */
|
|
1872
2580
|
async recoverWindow() {
|
|
2581
|
+
const previousWindowId = this.tmux?.getWindowId();
|
|
1873
2582
|
try {
|
|
1874
2583
|
const windows = await TmuxManager.listWindows(this.tmuxSessionName);
|
|
1875
2584
|
const match = windows.find(w => w.name === this.name);
|
|
@@ -1877,6 +2586,12 @@ export class Daemon extends EventEmitter {
|
|
|
1877
2586
|
return undefined;
|
|
1878
2587
|
this.tmux = new TmuxManager(this.tmuxSessionName, match.id, resolveTmuxLogicalSize(this.config.terminal));
|
|
1879
2588
|
writeFileSync(join(this.instanceDir, "window-id"), match.id);
|
|
2589
|
+
// The window we were talking to is gone; leaving it registered means the
|
|
2590
|
+
// control client re-resolves a dead id — one tmux subprocess — on every
|
|
2591
|
+
// reconnect, for the life of the fleet process.
|
|
2592
|
+
if (previousWindowId && previousWindowId !== match.id) {
|
|
2593
|
+
this.controlClient?.unregisterWindow(previousWindowId);
|
|
2594
|
+
}
|
|
1880
2595
|
await this.controlClient?.registerWindow(match.id);
|
|
1881
2596
|
this.bindInstanceStateOutputListener(match.id);
|
|
1882
2597
|
this.logger.info({ windowId: match.id }, "Recovered window ID for message delivery");
|
|
@@ -1887,12 +2602,38 @@ export class Daemon extends EventEmitter {
|
|
|
1887
2602
|
return undefined;
|
|
1888
2603
|
}
|
|
1889
2604
|
}
|
|
1890
|
-
/**
|
|
2605
|
+
/**
|
|
2606
|
+
* Poll up to ~2s (200ms × 10) for the pane to emit output after `since`.
|
|
2607
|
+
*
|
|
2608
|
+
* A `false` here is not "no output" — it is read as **"the message was pasted but
|
|
2609
|
+
* never submitted"**, which re-sends Enter and then reports ❌ to the user. So a
|
|
2610
|
+
* false negative costs a possible double submit and a failure notice for a
|
|
2611
|
+
* message that actually arrived.
|
|
2612
|
+
*
|
|
2613
|
+
* That is exactly what a control-mode reconnect used to produce. `connect()`
|
|
2614
|
+
* drops every output timestamp, so `hasOutputSince` answers `false` for a pane
|
|
2615
|
+
* that did react — the evidence was thrown away, not absent. Blind time is
|
|
2616
|
+
* therefore not counted against the budget: after a reset the poll restarts from
|
|
2617
|
+
* the moment observation resumed. A hard wall-clock cap keeps a reconnect loop
|
|
2618
|
+
* from extending this forever.
|
|
2619
|
+
*/
|
|
1891
2620
|
async confirmBusyAfterEnter(windowId, since) {
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
2621
|
+
const client = this.controlClient;
|
|
2622
|
+
const hardDeadline = Date.now() + CONFIRM_BUSY_MAX_WAIT_MS;
|
|
2623
|
+
let observedFrom = since;
|
|
2624
|
+
let polls = 0;
|
|
2625
|
+
while (polls < CONFIRM_BUSY_POLLS) {
|
|
2626
|
+
await new Promise(r => setTimeout(r, CONFIRM_BUSY_POLL_MS));
|
|
2627
|
+
if (client.hasOutputSince(windowId, observedFrom))
|
|
1895
2628
|
return true;
|
|
2629
|
+
const resetAt = client.getObservationResetAt();
|
|
2630
|
+
if (resetAt > observedFrom && Date.now() < hardDeadline) {
|
|
2631
|
+
this.logger.debug({ windowId, resetAt }, "Control mode reconnected mid-confirmation — restarting the Enter check from when we could see again");
|
|
2632
|
+
observedFrom = resetAt;
|
|
2633
|
+
polls = 0;
|
|
2634
|
+
continue;
|
|
2635
|
+
}
|
|
2636
|
+
polls++;
|
|
1896
2637
|
}
|
|
1897
2638
|
return false;
|
|
1898
2639
|
}
|
|
@@ -1963,8 +2704,8 @@ export class Daemon extends EventEmitter {
|
|
|
1963
2704
|
});
|
|
1964
2705
|
const timeout = setTimeout(() => {
|
|
1965
2706
|
this.pendingIpcRequests.delete(fleetReqId);
|
|
1966
|
-
respond(null,
|
|
1967
|
-
},
|
|
2707
|
+
respond(null, `Task operation timed out after ${daemonBudgetMs(tool) / 1000}s`);
|
|
2708
|
+
}, daemonBudgetMs(tool));
|
|
1968
2709
|
this.pendingIpcRequests.set(fleetReqId, (respMsg) => {
|
|
1969
2710
|
clearTimeout(timeout);
|
|
1970
2711
|
respond(respMsg.result, respMsg.error);
|
|
@@ -1986,8 +2727,8 @@ export class Daemon extends EventEmitter {
|
|
|
1986
2727
|
});
|
|
1987
2728
|
const timeout = setTimeout(() => {
|
|
1988
2729
|
this.pendingIpcRequests.delete(fleetReqId);
|
|
1989
|
-
respond(null,
|
|
1990
|
-
},
|
|
2730
|
+
respond(null, `Decision operation timed out after ${daemonBudgetMs(tool) / 1000}s`);
|
|
2731
|
+
}, daemonBudgetMs(tool));
|
|
1991
2732
|
this.pendingIpcRequests.set(fleetReqId, (respMsg) => {
|
|
1992
2733
|
clearTimeout(timeout);
|
|
1993
2734
|
respond(respMsg.result, respMsg.error);
|
|
@@ -2013,8 +2754,8 @@ export class Daemon extends EventEmitter {
|
|
|
2013
2754
|
// Wait for fleet_schedule_response via pending request map
|
|
2014
2755
|
const timeout = setTimeout(() => {
|
|
2015
2756
|
this.pendingIpcRequests.delete(fleetReqId);
|
|
2016
|
-
respond(null,
|
|
2017
|
-
},
|
|
2757
|
+
respond(null, `Schedule operation timed out after ${daemonBudgetMs(tool) / 1000}s`);
|
|
2758
|
+
}, daemonBudgetMs(tool));
|
|
2018
2759
|
this.pendingIpcRequests.set(fleetReqId, (respMsg) => {
|
|
2019
2760
|
clearTimeout(timeout);
|
|
2020
2761
|
respond(respMsg.result, respMsg.error);
|
|
@@ -2035,7 +2776,9 @@ export class Daemon extends EventEmitter {
|
|
|
2035
2776
|
fleetRequestId: fleetReqId,
|
|
2036
2777
|
senderSessionName,
|
|
2037
2778
|
});
|
|
2038
|
-
|
|
2779
|
+
// Shared table, so delete_instance and the deployment tools get the same
|
|
2780
|
+
// budget here as the MCP client expects (the ad-hoc list omitted them).
|
|
2781
|
+
const crossTimeoutMs = daemonBudgetMs(tool);
|
|
2039
2782
|
const timeout = setTimeout(() => {
|
|
2040
2783
|
this.pendingIpcRequests.delete(fleetReqId);
|
|
2041
2784
|
respond(null, `Cross-instance operation timed out after ${crossTimeoutMs / 1000}s`);
|
|
@@ -2062,7 +2805,11 @@ export class Daemon extends EventEmitter {
|
|
|
2062
2805
|
}
|
|
2063
2806
|
if (this.lastChatId) {
|
|
2064
2807
|
args.chat_id = this.lastChatId;
|
|
2065
|
-
|
|
2808
|
+
// Discord messages live in the channel/thread id, not the guild id in
|
|
2809
|
+
// chat_id. This is required for react/edit as well as reply. In
|
|
2810
|
+
// particular, general_topic deliberately has no configured reply thread,
|
|
2811
|
+
// so FleetManager cannot reconstruct this address from fleet.yaml.
|
|
2812
|
+
if (this.lastThreadId)
|
|
2066
2813
|
args.thread_id = this.lastThreadId;
|
|
2067
2814
|
}
|
|
2068
2815
|
}
|
|
@@ -2075,11 +2822,20 @@ export class Daemon extends EventEmitter {
|
|
|
2075
2822
|
// from prematurely resolving their pending requests when they receive the broadcast.
|
|
2076
2823
|
const fleetReqId = `tool_${requestId}`;
|
|
2077
2824
|
const outboundKey = fleetReqId;
|
|
2078
|
-
this.ipcServer?.broadcast({
|
|
2825
|
+
this.ipcServer?.broadcast({
|
|
2826
|
+
type: "fleet_outbound",
|
|
2827
|
+
tool,
|
|
2828
|
+
args,
|
|
2829
|
+
fleetRequestId: fleetReqId,
|
|
2830
|
+
// Preserve the exact adapter world that supplied the chat context. An
|
|
2831
|
+
// instance binding is only a fallback: persisted/runtime context can be
|
|
2832
|
+
// from a secondary world, and message ids are not portable across bots.
|
|
2833
|
+
adapterId: this.lastAdapterId,
|
|
2834
|
+
});
|
|
2079
2835
|
const timeout = setTimeout(() => {
|
|
2080
2836
|
this.pendingIpcRequests.delete(outboundKey);
|
|
2081
|
-
respond(null,
|
|
2082
|
-
},
|
|
2837
|
+
respond(null, `Fleet outbound timed out after ${daemonBudgetMs(tool) / 1000}s`);
|
|
2838
|
+
}, daemonBudgetMs(tool));
|
|
2083
2839
|
this.pendingIpcRequests.set(outboundKey, (respMsg) => {
|
|
2084
2840
|
clearTimeout(timeout);
|
|
2085
2841
|
respond(respMsg.result, respMsg.error);
|
|
@@ -2143,7 +2899,14 @@ export class Daemon extends EventEmitter {
|
|
|
2143
2899
|
try {
|
|
2144
2900
|
const all = JSON.parse(process.env.AGEND_DECISIONS);
|
|
2145
2901
|
const workDir = this.config.working_directory;
|
|
2146
|
-
|
|
2902
|
+
// `scope: "fleet"` used to bypass the project check entirely, so a
|
|
2903
|
+
// single-project worker inherited every other project's playbook (8 of
|
|
2904
|
+
// 14 injected here were foreign) — resent on every API call. Now a
|
|
2905
|
+
// fleet decision must also be RELEVANT: global (no project_root), the
|
|
2906
|
+
// same project, or the same project family (worktrees/checkouts of one
|
|
2907
|
+
// repo). A general is exempt: it dispatches across all projects, so
|
|
2908
|
+
// cross-project routing rules are exactly what it needs.
|
|
2909
|
+
decisions = selectRelevantDecisions(all, workDir, this.config.general_topic === true);
|
|
2147
2910
|
// Stable ordering so identical decision sets always build byte-identical
|
|
2148
2911
|
// instructions — otherwise source ordering jitter flips the warmup hash.
|
|
2149
2912
|
decisions.sort((a, b) => a.title.localeCompare(b.title));
|
|
@@ -2161,8 +2924,14 @@ export class Daemon extends EventEmitter {
|
|
|
2161
2924
|
AGEND_BACKEND: this.runtimeIdentity?.backend ?? this.config.backend ?? this.backend?.binaryName ?? "unknown",
|
|
2162
2925
|
AGEND_MODEL: this.runtimeIdentity?.model ?? this.config.model ?? "default",
|
|
2163
2926
|
};
|
|
2164
|
-
|
|
2165
|
-
|
|
2927
|
+
// A general is a dispatcher, not a worker: default it to the `general` tool
|
|
2928
|
+
// profile instead of `full`. Every tool's schema is resent on every API call,
|
|
2929
|
+
// so full (44 tools) costs a general ~22k chars per turn — the single largest
|
|
2930
|
+
// slice of its prompt. Explicit config still wins.
|
|
2931
|
+
const defaultToolSet = this.config.general_topic ? "general" : undefined;
|
|
2932
|
+
const toolSet = this.config.tool_set ?? defaultToolSet;
|
|
2933
|
+
if (toolSet)
|
|
2934
|
+
mcpEnv.AGEND_TOOL_SET = toolSet;
|
|
2166
2935
|
if (this.config.display_name)
|
|
2167
2936
|
mcpEnv.AGEND_DISPLAY_NAME = this.config.display_name;
|
|
2168
2937
|
if (this.config.description)
|
|
@@ -2230,6 +2999,7 @@ export class Daemon extends EventEmitter {
|
|
|
2230
2999
|
},
|
|
2231
3000
|
skipPermissions: this.config.skipPermissions,
|
|
2232
3001
|
model: this.modelOverride ?? this.config.model,
|
|
3002
|
+
effort: this.config.effort,
|
|
2233
3003
|
kiroUi: this.config.kiro_ui,
|
|
2234
3004
|
skipResume: this.skipResume,
|
|
2235
3005
|
instructions,
|
|
@@ -2255,7 +3025,9 @@ export class Daemon extends EventEmitter {
|
|
|
2255
3025
|
// Small delay to let the CLI fully render its ready prompt
|
|
2256
3026
|
await new Promise(r => setTimeout(r, 1_000));
|
|
2257
3027
|
try {
|
|
2258
|
-
|
|
3028
|
+
// Messages can arrive during a restart and be queued on pasteLock before the
|
|
3029
|
+
// snapshot lands; both write to the pane, so both go through the same lock.
|
|
3030
|
+
await this.paneWriteLock.run(() => this.tmux.pasteText(`[system:session-snapshot]\n${snapshot}\n\nThis is a background context restore — do NOT reply to or acknowledge this message. Simply resume normal operation when the next user or instance message arrives.`, this.systemPasteOptions()));
|
|
2259
3031
|
this.logger.info("Injected session snapshot as first message");
|
|
2260
3032
|
this.emit("snapshot_injected", this.name);
|
|
2261
3033
|
}
|
|
@@ -2264,9 +3036,69 @@ export class Daemon extends EventEmitter {
|
|
|
2264
3036
|
this.emit("snapshot_failed", this.name);
|
|
2265
3037
|
}
|
|
2266
3038
|
}
|
|
3039
|
+
/**
|
|
3040
|
+
* Depth, not a flag: the wake path marks a spawn and then calls trySpawn, which
|
|
3041
|
+
* marks another. With a plain boolean the inner one's completion would clear the
|
|
3042
|
+
* latch while the outer spawn was still dismissing dialogs, releasing delivery
|
|
3043
|
+
* into exactly the window this exists to close.
|
|
3044
|
+
*/
|
|
3045
|
+
beginSpawn() {
|
|
3046
|
+
this.spawnDepth++;
|
|
3047
|
+
this.spawning = true;
|
|
3048
|
+
if (!this.spawnSettled) {
|
|
3049
|
+
this.spawnSettled = new Promise(resolve => { this.resolveSpawnSettled = resolve; });
|
|
3050
|
+
}
|
|
3051
|
+
}
|
|
3052
|
+
endSpawn() {
|
|
3053
|
+
this.spawnDepth = Math.max(0, this.spawnDepth - 1);
|
|
3054
|
+
if (this.spawnDepth > 0)
|
|
3055
|
+
return; // an enclosing spawn is still running
|
|
3056
|
+
this.spawning = false;
|
|
3057
|
+
const resolve = this.resolveSpawnSettled;
|
|
3058
|
+
this.spawnSettled = null;
|
|
3059
|
+
this.resolveSpawnSettled = null;
|
|
3060
|
+
resolve?.();
|
|
3061
|
+
}
|
|
3062
|
+
/**
|
|
3063
|
+
* Hold until the CLI has finished starting up.
|
|
3064
|
+
*
|
|
3065
|
+
* Startup is not a quiet period: `dismissDialogsUntilReady` is clicking through
|
|
3066
|
+
* trust prompts and session pickers. A pane showing a modal dialog produces no
|
|
3067
|
+
* output, so `waitUntilIdle` reports it idle and a queued message gets pasted
|
|
3068
|
+
* *into the dialog* — where the text is discarded and the Enter picks a menu
|
|
3069
|
+
* item. The paste and the Enter both "succeed", so this loses the message
|
|
3070
|
+
* without even a ❌.
|
|
3071
|
+
*
|
|
3072
|
+
* The pane write lock (#414) does not cover this: it serialises each key
|
|
3073
|
+
* sequence, but it is released between one dialog and the next.
|
|
3074
|
+
*
|
|
3075
|
+
* Bounded, and called BEFORE the pane lock is taken — waiting on the spawn while
|
|
3076
|
+
* holding the lock the spawn itself needs would deadlock.
|
|
3077
|
+
*/
|
|
3078
|
+
async waitForSpawnToSettle() {
|
|
3079
|
+
const settled = this.spawnSettled;
|
|
3080
|
+
if (!settled)
|
|
3081
|
+
return;
|
|
3082
|
+
this.logger.debug("Holding delivery until the CLI has finished starting up");
|
|
3083
|
+
let timer;
|
|
3084
|
+
const cap = new Promise(resolve => {
|
|
3085
|
+
timer = setTimeout(resolve, SPAWN_SETTLE_MAX_WAIT_MS);
|
|
3086
|
+
timer.unref?.();
|
|
3087
|
+
});
|
|
3088
|
+
try {
|
|
3089
|
+
await Promise.race([settled, cap]);
|
|
3090
|
+
}
|
|
3091
|
+
finally {
|
|
3092
|
+
if (timer)
|
|
3093
|
+
clearTimeout(timer);
|
|
3094
|
+
}
|
|
3095
|
+
if (this.spawning) {
|
|
3096
|
+
this.logger.warn("CLI still starting after the delivery hold — delivering anyway");
|
|
3097
|
+
}
|
|
3098
|
+
}
|
|
2267
3099
|
/** Spawn a CLI window. Returns true if --resume was used successfully. */
|
|
2268
3100
|
async spawnClaudeWindow() {
|
|
2269
|
-
this.
|
|
3101
|
+
this.beginSpawn();
|
|
2270
3102
|
let resumedSuccessfully = false;
|
|
2271
3103
|
try {
|
|
2272
3104
|
this.toolStatusLines = [];
|
|
@@ -2303,7 +3135,7 @@ export class Daemon extends EventEmitter {
|
|
|
2303
3135
|
this.backgroundSessionRecoveryAttempted = false;
|
|
2304
3136
|
}
|
|
2305
3137
|
finally {
|
|
2306
|
-
this.
|
|
3138
|
+
this.endSpawn();
|
|
2307
3139
|
}
|
|
2308
3140
|
return resumedSuccessfully;
|
|
2309
3141
|
}
|
|
@@ -2374,11 +3206,13 @@ export class Daemon extends EventEmitter {
|
|
|
2374
3206
|
// the global web token) from impersonating instances.
|
|
2375
3207
|
const agentTokenPath = join(this.instanceDir, "agent.token");
|
|
2376
3208
|
const agentToken = randomBytes(32).toString("hex");
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
|
|
3209
|
+
const tokenWrite = writeSecretFile(agentTokenPath, agentToken);
|
|
3210
|
+
if (!tokenWrite.ok) {
|
|
3211
|
+
// Do not fail the spawn over it — an instance that cannot start is worse
|
|
3212
|
+
// than one whose token is readable. But say so, loudly: the whole point of
|
|
3213
|
+
// this token is that other local processes cannot use it.
|
|
3214
|
+
this.logger.error({ path: agentTokenPath, mode: tokenWrite.mode?.toString(8), reason: tokenWrite.reason }, "Agent token file is not owner-only — other local users can impersonate this instance");
|
|
2380
3215
|
}
|
|
2381
|
-
catch { }
|
|
2382
3216
|
// AGEND_HOME points the child's agent-cli at the same data dir the daemon
|
|
2383
3217
|
// is using, so it can locate <instanceDir>/agent.token.
|
|
2384
3218
|
const agendHome = join(this.instanceDir, "..", "..");
|
|
@@ -2396,7 +3230,12 @@ export class Daemon extends EventEmitter {
|
|
|
2396
3230
|
windowId = this.tmux.getWindowId();
|
|
2397
3231
|
}
|
|
2398
3232
|
else {
|
|
3233
|
+
// A crash respawn makes a brand new window. The dead one stays registered
|
|
3234
|
+
// unless we say so — the respawn branch above already does.
|
|
3235
|
+
const retired = this.tmux.getWindowId();
|
|
2399
3236
|
windowId = await this.tmux.createWindow(cmd, resolvedCwd, this.name);
|
|
3237
|
+
if (retired && retired !== windowId)
|
|
3238
|
+
this.controlClient?.unregisterWindow(retired);
|
|
2400
3239
|
}
|
|
2401
3240
|
writeFileSync(join(this.instanceDir, "window-id"), windowId);
|
|
2402
3241
|
// Enable remain-on-exit to capture exit codes on crash
|
|
@@ -2462,15 +3301,20 @@ export class Daemon extends EventEmitter {
|
|
|
2462
3301
|
for (const dialog of startupDialogs) {
|
|
2463
3302
|
if (dialog.pattern.test(pane)) {
|
|
2464
3303
|
this.logger.debug(`Dismissing startup dialog: ${dialog.description}`);
|
|
2465
|
-
|
|
2466
|
-
|
|
2467
|
-
|
|
3304
|
+
// Restart is exactly when inbound messages pile up, and nothing gates
|
|
3305
|
+
// delivery on `spawning`. Take the pane lock for the key sequence so a
|
|
3306
|
+
// queued message cannot be pasted into a half-dismissed trust dialog.
|
|
3307
|
+
await this.paneWriteLock.run(async () => {
|
|
3308
|
+
for (const key of dialog.keys) {
|
|
3309
|
+
if (key === "Up" || key === "Down" || key === "Enter" || key === "Escape") {
|
|
3310
|
+
await this.tmux.sendSpecialKey(key);
|
|
3311
|
+
}
|
|
3312
|
+
else {
|
|
3313
|
+
await this.tmux.sendKeys(key);
|
|
3314
|
+
}
|
|
3315
|
+
await new Promise(r => setTimeout(r, 200));
|
|
2468
3316
|
}
|
|
2469
|
-
|
|
2470
|
-
await this.tmux.sendKeys(key);
|
|
2471
|
-
}
|
|
2472
|
-
await new Promise(r => setTimeout(r, 200));
|
|
2473
|
-
}
|
|
3317
|
+
});
|
|
2474
3318
|
// Wait for next screen to render
|
|
2475
3319
|
if (this.controlClient) {
|
|
2476
3320
|
const wid = readFileSync(join(this.instanceDir, "window-id"), "utf-8").trim();
|