@songsid/agend 2.1.1-beta.2 → 2.1.1-beta.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/README.zh-TW.md +1 -0
- package/dist/access-path.js +15 -6
- package/dist/access-path.js.map +1 -1
- package/dist/agent-cli-instructions.md +9 -0
- package/dist/backend/antigravity.d.ts +4 -1
- package/dist/backend/antigravity.js +24 -13
- package/dist/backend/antigravity.js.map +1 -1
- package/dist/backend/claude-code.d.ts +37 -0
- package/dist/backend/claude-code.js +47 -8
- package/dist/backend/claude-code.js.map +1 -1
- package/dist/backend/codex.d.ts +17 -0
- package/dist/backend/codex.js +279 -62
- package/dist/backend/codex.js.map +1 -1
- package/dist/backend/gemini-cli.js +4 -7
- package/dist/backend/gemini-cli.js.map +1 -1
- package/dist/backend/grok.d.ts +24 -0
- package/dist/backend/grok.js +57 -8
- package/dist/backend/grok.js.map +1 -1
- package/dist/backend/kiro.d.ts +27 -0
- package/dist/backend/kiro.js +96 -15
- package/dist/backend/kiro.js.map +1 -1
- package/dist/backend/opencode.js +4 -7
- package/dist/backend/opencode.js.map +1 -1
- package/dist/backend/types.d.ts +44 -0
- package/dist/backend/types.js +3 -3
- package/dist/backend/types.js.map +1 -1
- package/dist/channel/adapters/discord.d.ts +15 -0
- package/dist/channel/adapters/discord.js +94 -6
- package/dist/channel/adapters/discord.js.map +1 -1
- package/dist/channel/adapters/telegram.d.ts +21 -0
- package/dist/channel/adapters/telegram.js +132 -15
- package/dist/channel/adapters/telegram.js.map +1 -1
- package/dist/channel/ipc-bridge.d.ts +17 -1
- package/dist/channel/ipc-bridge.js +53 -17
- package/dist/channel/ipc-bridge.js.map +1 -1
- package/dist/channel/ipc-timeouts.d.ts +40 -0
- package/dist/channel/ipc-timeouts.js +58 -0
- package/dist/channel/ipc-timeouts.js.map +1 -0
- package/dist/channel/mcp-server.js +30 -14
- package/dist/channel/mcp-server.js.map +1 -1
- package/dist/channel/mcp-tools.js +28 -1
- package/dist/channel/mcp-tools.js.map +1 -1
- package/dist/channel/message-queue.d.ts +1 -0
- package/dist/channel/message-queue.js +49 -17
- package/dist/channel/message-queue.js.map +1 -1
- package/dist/channel/reconnect-backoff.d.ts +17 -0
- package/dist/channel/reconnect-backoff.js +21 -0
- package/dist/channel/reconnect-backoff.js.map +1 -0
- package/dist/channel/types.d.ts +29 -0
- package/dist/classic-channel-manager.js +1 -5
- package/dist/classic-channel-manager.js.map +1 -1
- package/dist/cli.js +148 -60
- package/dist/cli.js.map +1 -1
- package/dist/completion.d.ts +27 -0
- package/dist/completion.js +121 -0
- package/dist/completion.js.map +1 -0
- package/dist/config-validator.js +27 -0
- package/dist/config-validator.js.map +1 -1
- package/dist/config.js +6 -0
- package/dist/config.js.map +1 -1
- package/dist/cost-guard.d.ts +3 -1
- package/dist/cost-guard.js +3 -1
- package/dist/cost-guard.js.map +1 -1
- package/dist/daemon.d.ts +230 -13
- package/dist/daemon.js +975 -335
- package/dist/daemon.js.map +1 -1
- package/dist/event-log.d.ts +31 -0
- package/dist/event-log.js +96 -0
- package/dist/event-log.js.map +1 -1
- package/dist/fleet-context.d.ts +2 -0
- package/dist/fleet-manager.d.ts +209 -4
- package/dist/fleet-manager.js +813 -84
- package/dist/fleet-manager.js.map +1 -1
- package/dist/general-knowledge/skills/cross-instance-messaging/SKILL.md +22 -0
- package/dist/general-knowledge/skills/model-discovery/SKILL.md +14 -23
- package/dist/general-knowledge/skills/session-management/SKILL.md +15 -34
- package/dist/hang-detector.d.ts +19 -13
- package/dist/hang-detector.js +19 -49
- package/dist/hang-detector.js.map +1 -1
- package/dist/instance-lifecycle.d.ts +21 -1
- package/dist/instance-lifecycle.js +141 -7
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/instructions.d.ts +5 -0
- package/dist/instructions.js +10 -5
- package/dist/instructions.js.map +1 -1
- package/dist/locale.js +3 -0
- package/dist/locale.js.map +1 -1
- package/dist/logger.js +14 -0
- package/dist/logger.js.map +1 -1
- package/dist/mcp-liveness.d.ts +21 -0
- package/dist/mcp-liveness.js +27 -0
- package/dist/mcp-liveness.js.map +1 -0
- package/dist/outbound-handlers.d.ts +16 -0
- package/dist/outbound-handlers.js +121 -27
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/outbound-schemas.d.ts +7 -2
- package/dist/outbound-schemas.js +13 -2
- package/dist/outbound-schemas.js.map +1 -1
- package/dist/pane-write-lock.d.ts +48 -0
- package/dist/pane-write-lock.js +73 -0
- package/dist/pane-write-lock.js.map +1 -0
- package/dist/process-memory.d.ts +31 -0
- package/dist/process-memory.js +79 -0
- package/dist/process-memory.js.map +1 -0
- package/dist/quickstart.js +17 -16
- package/dist/quickstart.js.map +1 -1
- package/dist/scheduler/db.js +3 -0
- package/dist/scheduler/db.js.map +1 -1
- package/dist/sd-notify.d.ts +45 -0
- package/dist/sd-notify.js +74 -3
- package/dist/sd-notify.js.map +1 -1
- package/dist/secret-file.d.ts +33 -0
- package/dist/secret-file.js +36 -0
- package/dist/secret-file.js.map +1 -0
- package/dist/setup-wizard.js +9 -7
- package/dist/setup-wizard.js.map +1 -1
- package/dist/tmux-control.d.ts +58 -5
- package/dist/tmux-control.js +107 -13
- package/dist/tmux-control.js.map +1 -1
- package/dist/tmux-manager.d.ts +47 -2
- package/dist/tmux-manager.js +103 -8
- package/dist/tmux-manager.js.map +1 -1
- package/dist/topic-commands.d.ts +43 -1
- package/dist/topic-commands.js +196 -66
- package/dist/topic-commands.js.map +1 -1
- package/dist/tui-glyphs.d.ts +25 -0
- package/dist/tui-glyphs.js +26 -0
- package/dist/tui-glyphs.js.map +1 -0
- package/dist/types.d.ts +19 -0
- package/dist/ui/view.html +291 -33
- package/dist/update-check.d.ts +22 -0
- package/dist/update-check.js +44 -0
- package/dist/update-check.js.map +1 -0
- package/dist/usage/providers.d.ts +50 -0
- package/dist/usage/providers.js +611 -0
- package/dist/usage/providers.js.map +1 -0
- package/dist/usage/usage-api.d.ts +50 -0
- package/dist/usage/usage-api.js +111 -0
- package/dist/usage/usage-api.js.map +1 -0
- package/dist/view-api.d.ts +10 -0
- package/dist/view-api.js +67 -14
- package/dist/view-api.js.map +1 -1
- package/dist/web-api.js +5 -2
- package/dist/web-api.js.map +1 -1
- package/package.json +4 -1
- package/templates/systemd.service.ejs +9 -1
- package/dist/channel/tool-tracker.d.ts +0 -13
- package/dist/channel/tool-tracker.js +0 -58
- package/dist/channel/tool-tracker.js.map +0 -1
- package/dist/daemon-entry.d.ts +0 -1
- package/dist/daemon-entry.js +0 -30
- package/dist/daemon-entry.js.map +0 -1
- package/dist/fleet-system-prompt.d.ts +0 -11
- package/dist/fleet-system-prompt.js +0 -61
- package/dist/fleet-system-prompt.js.map +0 -1
package/dist/daemon.js
CHANGED
|
@@ -1,18 +1,23 @@
|
|
|
1
1
|
import { join, dirname, basename, resolve } from "node:path";
|
|
2
|
-
import { mkdirSync, writeFileSync, readFileSync, existsSync, unlinkSync, rmSync, appendFileSync, statSync,
|
|
2
|
+
import { mkdirSync, writeFileSync, readFileSync, existsSync, unlinkSync, rmSync, appendFileSync, statSync, renameSync } from "node:fs";
|
|
3
3
|
import { fileURLToPath } from "node:url";
|
|
4
4
|
import { createHash, randomBytes } from "node:crypto";
|
|
5
5
|
import { EventEmitter } from "node:events";
|
|
6
|
+
import { rotateLogIfNeeded } from "./logger.js";
|
|
7
|
+
import { mcpServerState } from "./mcp-liveness.js";
|
|
6
8
|
import { clearPausedMarker, writePausedMarker } from "./pause-marker.js";
|
|
7
|
-
import { TmuxManager } from "./tmux-manager.js";
|
|
9
|
+
import { TmuxManager, resolveTmuxLogicalSize } from "./tmux-manager.js";
|
|
8
10
|
import { TranscriptMonitor } from "./transcript-monitor.js";
|
|
9
11
|
import { ContextGuardian } from "./context-guardian.js";
|
|
10
12
|
import { IpcServer } from "./channel/ipc-bridge.js";
|
|
13
|
+
import { daemonBudgetMs } from "./channel/ipc-timeouts.js";
|
|
11
14
|
import { MessageBus } from "./channel/message-bus.js";
|
|
12
15
|
import { shellQuote } from "./backend/types.js";
|
|
13
16
|
import { getTmuxSession } from "./config.js";
|
|
14
17
|
import { routeToolCall } from "./channel/tool-router.js";
|
|
15
18
|
import { HangDetector } from "./hang-detector.js";
|
|
19
|
+
import { writeSecretFile } from "./secret-file.js";
|
|
20
|
+
import { PaneWriteLock } from "./pane-write-lock.js";
|
|
16
21
|
import { buildFleetInstructions } from "./instructions.js";
|
|
17
22
|
const __filename = fileURLToPath(import.meta.url);
|
|
18
23
|
const __dirname = dirname(__filename);
|
|
@@ -37,9 +42,54 @@ export function buildInstructionReloadNotice(binaryName, instanceName, instanceD
|
|
|
37
42
|
export const DEFAULT_STUCK_TIMEOUT_MS = 10 * 60_000;
|
|
38
43
|
export const DEFAULT_STATE_IDLE_DEBOUNCE_MS = 2_000;
|
|
39
44
|
export const DEFAULT_STATE_SAFETY_SWEEP_MS = 60_000;
|
|
40
|
-
/** @deprecated State detection is event-driven; this now aliases the safety sweep. */
|
|
41
|
-
export const DEFAULT_STATE_POLL_INTERVAL_MS = DEFAULT_STATE_SAFETY_SWEEP_MS;
|
|
42
45
|
const LAST_INBOUND_FILE = "last-inbound-at";
|
|
46
|
+
/** Minimum gap between "health check is failing" notifications for one instance. */
|
|
47
|
+
const HEALTH_ERROR_NOTIFY_INTERVAL_MS = 10 * 60_000;
|
|
48
|
+
/**
|
|
49
|
+
* Whether two working directories belong to the same project, so a fleet-scoped
|
|
50
|
+
* decision recorded in one reaches the other. Covers the worktree/checkout
|
|
51
|
+
* layout AgEnD fleets actually use — `AgEnD`, `AgEnD-dev1`, `AgEnD-dev2`,
|
|
52
|
+
* `AgEnD-reviewer`, `AgEnD-main` are one project; `DouPo_Server` is not.
|
|
53
|
+
* Nesting counts too (a subdirectory of a project belongs to it).
|
|
54
|
+
*/
|
|
55
|
+
export function sameProjectFamily(a, b) {
|
|
56
|
+
const norm = (p) => p.replace(/\/+$/, "");
|
|
57
|
+
const x = norm(a), y = norm(b);
|
|
58
|
+
if (x === y)
|
|
59
|
+
return true;
|
|
60
|
+
if (x.startsWith(y + "/") || y.startsWith(x + "/"))
|
|
61
|
+
return true;
|
|
62
|
+
// Strip a trailing worktree/role suffix: "AgEnD-dev2" → "agend".
|
|
63
|
+
const family = (p) => {
|
|
64
|
+
const base = p.slice(p.lastIndexOf("/") + 1).toLowerCase();
|
|
65
|
+
return base.replace(/[-_](dev\d*|main|reviewer|tester|leader|sol|codex|worktree|wt\d*)$/i, "");
|
|
66
|
+
};
|
|
67
|
+
const fx = family(x), fy = family(y);
|
|
68
|
+
return fx.length > 0 && fx === fy;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Pick the decisions worth injecting into one instance's system prompt.
|
|
72
|
+
*
|
|
73
|
+
* `scope: "fleet"` previously bypassed the project check outright, so every
|
|
74
|
+
* instance carried every other project's playbook — measured at 8 of 14 here,
|
|
75
|
+
* resent on every API call. A fleet decision must now also be relevant: global
|
|
76
|
+
* (no project_root), the same project, or the same project family.
|
|
77
|
+
*
|
|
78
|
+
* `isDispatcher` (a general) opts out of that narrowing on purpose: it routes
|
|
79
|
+
* work across all projects, so cross-project rules ("X 文件操作由 Y 負責") are
|
|
80
|
+
* precisely what it must know. Narrowing a general would cause misrouting.
|
|
81
|
+
*/
|
|
82
|
+
export function selectRelevantDecisions(all, workDir, isDispatcher = false) {
|
|
83
|
+
return all.filter(d => {
|
|
84
|
+
if (d.project_root === workDir)
|
|
85
|
+
return true; // own project, any scope
|
|
86
|
+
if (d.scope !== "fleet")
|
|
87
|
+
return false; // project-scoped, elsewhere
|
|
88
|
+
if (!d.project_root)
|
|
89
|
+
return true; // truly global
|
|
90
|
+
return isDispatcher || sameProjectFamily(d.project_root, workDir);
|
|
91
|
+
});
|
|
92
|
+
}
|
|
43
93
|
/** Read the last real channel inbound timestamp persisted across daemon restarts. */
|
|
44
94
|
export function readLastInboundAt(instanceDir, now = Date.now()) {
|
|
45
95
|
try {
|
|
@@ -58,6 +108,40 @@ export function writeLastInboundAt(instanceDir, timestamp) {
|
|
|
58
108
|
writeFileSync(temp, String(timestamp));
|
|
59
109
|
renameSync(temp, target);
|
|
60
110
|
}
|
|
111
|
+
/**
|
|
112
|
+
* Render the handoff/routing metadata that rides along with an inbound message
|
|
113
|
+
* into the block actually pasted into the agent's pane.
|
|
114
|
+
*
|
|
115
|
+
* These fields were all populated by the sender (outbound-handlers builds them
|
|
116
|
+
* into `ipcMeta`) and delivered over IPC, but never rendered — so the receiving
|
|
117
|
+
* agent could not see them, and several documented flows could not work:
|
|
118
|
+
*
|
|
119
|
+
* - `report_result` requires a `correlation_id` the agent had no way to know, so
|
|
120
|
+
* the "correlation_id not recognized" warning fired on essentially every call.
|
|
121
|
+
* - Delegation cancel buttons are retired by correlation id, so they never retired.
|
|
122
|
+
* - `react`, `edit_message` and `reply.reply_to` need `message_id`, which the tool
|
|
123
|
+
* descriptions tell the agent to take "from the inbound block".
|
|
124
|
+
* - `download_attachment` needs `attachment_file_id`.
|
|
125
|
+
* - `requires_reply` was invisible, so a delegated task looked like an FYI.
|
|
126
|
+
*
|
|
127
|
+
* Only non-empty fields are emitted, so a plain user message gains at most a
|
|
128
|
+
* message_id line.
|
|
129
|
+
*/
|
|
130
|
+
export function renderHandoffMetadata(meta) {
|
|
131
|
+
const rows = [];
|
|
132
|
+
const add = (label, value) => {
|
|
133
|
+
if (value && value.trim())
|
|
134
|
+
rows.push(`${label}: ${value.trim()}`);
|
|
135
|
+
};
|
|
136
|
+
add("message_id", meta.message_id);
|
|
137
|
+
add("correlation_id", meta.correlation_id);
|
|
138
|
+
add("request_kind", meta.request_kind);
|
|
139
|
+
add("task_summary", meta.task_summary);
|
|
140
|
+
add("working_directory", meta.working_directory);
|
|
141
|
+
add("branch", meta.branch);
|
|
142
|
+
add("attachment_file_id", meta.attachment_file_id);
|
|
143
|
+
return rows.length ? `\n(${rows.join(" | ")})` : "";
|
|
144
|
+
}
|
|
61
145
|
/** Headless inactivity timer used by the daemon and unit tests. */
|
|
62
146
|
export class AutoPauseController {
|
|
63
147
|
thresholdMs;
|
|
@@ -102,24 +186,47 @@ export class AutoPauseController {
|
|
|
102
186
|
* Pane motion wins over a ready match because several backends keep their ready
|
|
103
187
|
* marker in a persistent header/footer while generating. A stable ready pane is
|
|
104
188
|
* idle; changing content is working; stable non-ready content eventually sticks.
|
|
189
|
+
*
|
|
190
|
+
* ## Why an optional busy pattern exists
|
|
191
|
+
*
|
|
192
|
+
* "Motion wins" degrades badly when a backend's ready marker is *always* on
|
|
193
|
+
* screen. `stuck` requires `!ready`, so for such a backend the state can only ever
|
|
194
|
+
* be idle or working — the stuck edge, and with it `handleStuckTransition` and the
|
|
195
|
+
* hang notification, is unreachable. A frozen CLI is then reported as idle, which
|
|
196
|
+
* also clears pending work: the message the user sent is quietly booked as done.
|
|
197
|
+
*
|
|
198
|
+
* claude-code was exactly this case (see ClaudeCodeBackend.getReadyPattern). A
|
|
199
|
+
* backend that can point at a marker meaning "generating right now" supplies
|
|
200
|
+
* `getBusyPattern()`, and that match vetoes ready regardless of what the ready
|
|
201
|
+
* pattern says.
|
|
105
202
|
*/
|
|
106
203
|
export class PaneStateMachine {
|
|
107
204
|
stuckTimeoutMs;
|
|
108
205
|
readyPattern;
|
|
206
|
+
busyPattern;
|
|
109
207
|
lastPaneHash = null;
|
|
110
208
|
lastPaneChangeAt;
|
|
111
209
|
lastObservedAt;
|
|
112
210
|
stateChangedAt;
|
|
113
211
|
currentState = "idle";
|
|
114
|
-
constructor(readyPattern, stuckTimeoutMs = DEFAULT_STUCK_TIMEOUT_MS, now = Date.now()) {
|
|
212
|
+
constructor(readyPattern, stuckTimeoutMs = DEFAULT_STUCK_TIMEOUT_MS, now = Date.now(), busyPattern) {
|
|
115
213
|
this.stuckTimeoutMs = stuckTimeoutMs;
|
|
116
214
|
// Stateful g/y regexes mutate lastIndex and can alternate true/false across
|
|
117
215
|
// polls. State detection must be deterministic for identical pane content.
|
|
118
216
|
this.readyPattern = new RegExp(readyPattern.source, readyPattern.flags.replace(/[gy]/g, ""));
|
|
217
|
+
this.busyPattern = busyPattern
|
|
218
|
+
? new RegExp(busyPattern.source, busyPattern.flags.replace(/[gy]/g, ""))
|
|
219
|
+
: null;
|
|
119
220
|
this.lastPaneChangeAt = now;
|
|
120
221
|
this.lastObservedAt = now;
|
|
121
222
|
this.stateChangedAt = now;
|
|
122
223
|
}
|
|
224
|
+
/** Ready, unless the backend can positively see that it is still generating. */
|
|
225
|
+
isReady(pane) {
|
|
226
|
+
if (this.busyPattern?.test(pane))
|
|
227
|
+
return false;
|
|
228
|
+
return this.readyPattern.test(pane);
|
|
229
|
+
}
|
|
123
230
|
observe(pane, now = Date.now()) {
|
|
124
231
|
const paneHash = createHash("sha256").update(pane).digest("hex");
|
|
125
232
|
const firstObservation = this.lastPaneHash === null;
|
|
@@ -129,7 +236,7 @@ export class PaneStateMachine {
|
|
|
129
236
|
this.lastPaneChangeAt = now;
|
|
130
237
|
}
|
|
131
238
|
this.lastObservedAt = now;
|
|
132
|
-
const ready = this.
|
|
239
|
+
const ready = this.isReady(pane);
|
|
133
240
|
const nextState = firstObservation
|
|
134
241
|
? ready ? "idle" : "working"
|
|
135
242
|
: paneChanged
|
|
@@ -193,6 +300,23 @@ export class PendingWorkTracker {
|
|
|
193
300
|
const NORMAL_ENTER_SETTLE_MS = 500;
|
|
194
301
|
const FIRST_ENTER_SETTLE_MS = 1_750;
|
|
195
302
|
const FIRST_DELIVERY_WINDOW_MS = 5_000;
|
|
303
|
+
/** After busy native-queue paste+Enter, wait before checking the pane for silent loss. */
|
|
304
|
+
const NATIVE_QUEUE_PASTE_VERIFY_MS = 2_000;
|
|
305
|
+
/** Enter-confirmation poll: 10 × 200ms ≈ 2s of observed silence before giving up. */
|
|
306
|
+
const CONFIRM_BUSY_POLLS = 10;
|
|
307
|
+
const CONFIRM_BUSY_POLL_MS = 200;
|
|
308
|
+
/**
|
|
309
|
+
* Ceiling on the whole confirmation, including time re-earned after a control-mode
|
|
310
|
+
* reconnect. Without it a reconnect loop could restart the poll indefinitely and
|
|
311
|
+
* hold the pane write lock along with it.
|
|
312
|
+
*/
|
|
313
|
+
const CONFIRM_BUSY_MAX_WAIT_MS = 10_000;
|
|
314
|
+
/**
|
|
315
|
+
* How long a delivery waits for an in-flight spawn. Comfortably past the default
|
|
316
|
+
* 25s startup timeout plus dialog dismissal; past it we fall back to the old
|
|
317
|
+
* behaviour rather than holding a message indefinitely.
|
|
318
|
+
*/
|
|
319
|
+
const SPAWN_SETTLE_MAX_WAIT_MS = 60_000;
|
|
196
320
|
/**
|
|
197
321
|
* One-shot timing gate for the first paste after a CLI reaches its ready
|
|
198
322
|
* prompt. Recent tmux versions expose a short interval where the prompt is
|
|
@@ -244,13 +368,13 @@ export class Daemon extends EventEmitter {
|
|
|
244
368
|
topicMode;
|
|
245
369
|
backend;
|
|
246
370
|
controlClient;
|
|
371
|
+
runtimeIdentity;
|
|
247
372
|
logger;
|
|
248
373
|
tmuxSessionName;
|
|
249
374
|
tmux = null;
|
|
250
375
|
ipcServer = null;
|
|
251
376
|
messageBus;
|
|
252
377
|
transcriptMonitor = null;
|
|
253
|
-
toolTracker = null;
|
|
254
378
|
guardian = null;
|
|
255
379
|
adapter = null;
|
|
256
380
|
pendingIpcRequests = new Map();
|
|
@@ -261,6 +385,8 @@ export class Daemon extends EventEmitter {
|
|
|
261
385
|
// Pending ack: react 🫡 on first transcript activity after receiving a message
|
|
262
386
|
pendingAckMessage = null;
|
|
263
387
|
// Tool status tracking for channel adapter
|
|
388
|
+
/** Last activity published to the fleet manager; null when nothing is running. */
|
|
389
|
+
currentActivity = null;
|
|
264
390
|
toolStatusMessageId = null;
|
|
265
391
|
toolStatusLines = [];
|
|
266
392
|
toolStatusDebounce = null;
|
|
@@ -276,9 +402,19 @@ export class Daemon extends EventEmitter {
|
|
|
276
402
|
lastSpawnAt = 0;
|
|
277
403
|
crashTimestamps = [];
|
|
278
404
|
healthCheckPaused = false;
|
|
405
|
+
lastHealthErrorNotifyAt = 0;
|
|
279
406
|
/** CLI pane availability, independent from the daemon process and tri-state. */
|
|
280
407
|
processStatus = "running";
|
|
281
408
|
spawning = false;
|
|
409
|
+
/**
|
|
410
|
+
* Resolves when the in-flight spawn finishes; null when none is running.
|
|
411
|
+
*
|
|
412
|
+
* `spawning` alone can only be polled. Delivery needs to *wait*, and a boolean
|
|
413
|
+
* poll would either busy-loop or race the flag being cleared.
|
|
414
|
+
*/
|
|
415
|
+
spawnSettled = null;
|
|
416
|
+
resolveSpawnSettled = null;
|
|
417
|
+
spawnDepth = 0;
|
|
282
418
|
skipResume = false;
|
|
283
419
|
backgroundSessionRecoveryAttempted = false;
|
|
284
420
|
/** Whether the last spawn started a fresh session (not resumed). */
|
|
@@ -301,10 +437,18 @@ export class Daemon extends EventEmitter {
|
|
|
301
437
|
instanceStateIdleDebounceMs = DEFAULT_STATE_IDLE_DEBOUNCE_MS;
|
|
302
438
|
instanceStateStuckTimeoutMs = DEFAULT_STUCK_TIMEOUT_MS;
|
|
303
439
|
instanceStateReadyPattern = null;
|
|
440
|
+
instanceStateBusyPattern = null;
|
|
304
441
|
instanceStateMonitorActive = false;
|
|
305
442
|
statePollInFlight = false;
|
|
306
443
|
autoPauseController;
|
|
307
444
|
pauseRequested = false;
|
|
445
|
+
/**
|
|
446
|
+
* Sticky "pause as soon as possible" (auth failure). Unlike pauseRequested —
|
|
447
|
+
* which the state monitor clears whenever the pane is not idle — this survives
|
|
448
|
+
* busy/stuck states, because an auth error fires mid-turn and a plain pause()
|
|
449
|
+
* would silently no-op exactly when we most need to stop feeding the CLI.
|
|
450
|
+
*/
|
|
451
|
+
pausePending = false;
|
|
308
452
|
pauseWakeState = "active";
|
|
309
453
|
pauseWakeTransition = null;
|
|
310
454
|
// Model failover: override model on next spawn when rate-limited
|
|
@@ -314,7 +458,13 @@ export class Daemon extends EventEmitter {
|
|
|
314
458
|
recentEvents = [];
|
|
315
459
|
recentToolActivity = [];
|
|
316
460
|
snapshotConsumed = false;
|
|
461
|
+
/** Orders inbound channel messages against each other (queue-depth accounting
|
|
462
|
+
* and ⏳/👀/✅ reactions live here). It does NOT cover the other writers that
|
|
463
|
+
* reach the pane — {@link paneWriteLock} does. */
|
|
317
464
|
pasteLock = Promise.resolve();
|
|
465
|
+
/** Mutual exclusion for *every* write into the pane, whichever subsystem it
|
|
466
|
+
* comes from. See PaneWriteLock for why interleaving is destructive. */
|
|
467
|
+
paneWriteLock = new PaneWriteLock();
|
|
318
468
|
pendingInstructionsUpdate;
|
|
319
469
|
pendingInstructionsNotice = false;
|
|
320
470
|
// Whether the warmup steering-reload notice should be injected after spawn.
|
|
@@ -327,6 +477,10 @@ export class Daemon extends EventEmitter {
|
|
|
327
477
|
firstDeliveryDelay = new FirstDeliveryDelay();
|
|
328
478
|
// PTY error pattern monitoring
|
|
329
479
|
errorMonitorTimer = null;
|
|
480
|
+
/** Same 5-min gate the error monitor uses, so a dead MCP server alerts once. */
|
|
481
|
+
static MCP_DEATH_COOLDOWN_MS = 5 * 60_000;
|
|
482
|
+
lastMcpDeathNotifiedAt = 0;
|
|
483
|
+
mcpDeathNotifiedForPid = null;
|
|
330
484
|
/** Prevent in-flight monitor callbacks from re-arming after a pause. */
|
|
331
485
|
runtimeMonitorsFrozen = false;
|
|
332
486
|
errorWaitingForRecovery = false; // true = error detected, waiting for ready pattern
|
|
@@ -371,7 +525,7 @@ export class Daemon extends EventEmitter {
|
|
|
371
525
|
this.errorRecoveryDeadlineAt = 0;
|
|
372
526
|
this.activeErrorPatternKey = null;
|
|
373
527
|
}
|
|
374
|
-
constructor(name, config, instanceDir, topicMode = false, backend, controlClient, rootLogger) {
|
|
528
|
+
constructor(name, config, instanceDir, topicMode = false, backend, controlClient, rootLogger, runtimeIdentity) {
|
|
375
529
|
super();
|
|
376
530
|
this.name = name;
|
|
377
531
|
this.config = config;
|
|
@@ -379,8 +533,14 @@ export class Daemon extends EventEmitter {
|
|
|
379
533
|
this.topicMode = topicMode;
|
|
380
534
|
this.backend = backend;
|
|
381
535
|
this.controlClient = controlClient;
|
|
536
|
+
this.runtimeIdentity = runtimeIdentity;
|
|
382
537
|
if (!rootLogger)
|
|
383
538
|
throw new Error("Daemon requires a shared root logger");
|
|
539
|
+
this.runtimeIdentity ??= {
|
|
540
|
+
kind: "fleet-topic",
|
|
541
|
+
backend: config.backend ?? backend?.binaryName ?? "unknown",
|
|
542
|
+
model: config.model ?? "default",
|
|
543
|
+
};
|
|
384
544
|
this.logger = rootLogger.child({ instance: name }, { level: config.log_level });
|
|
385
545
|
this.tmuxSessionName = getTmuxSession();
|
|
386
546
|
this.messageBus = new MessageBus();
|
|
@@ -447,9 +607,6 @@ export class Daemon extends EventEmitter {
|
|
|
447
607
|
if ((type === "fleet_schedule_response" || type === "fleet_outbound_response" || type === "fleet_decision_response" || type === "fleet_task_response" || type === "fleet_display_name_response" || type === "fleet_description_response") && msg.fleetRequestId) {
|
|
448
608
|
key = String(msg.fleetRequestId);
|
|
449
609
|
}
|
|
450
|
-
else if (type === "fleet_outbound_response" && msg.requestId != null) {
|
|
451
|
-
key = `fleet_out_${msg.requestId}`;
|
|
452
|
-
}
|
|
453
610
|
if (key && this.pendingIpcRequests.has(key)) {
|
|
454
611
|
const handler = this.pendingIpcRequests.get(key);
|
|
455
612
|
this.pendingIpcRequests.delete(key);
|
|
@@ -543,7 +700,7 @@ export class Daemon extends EventEmitter {
|
|
|
543
700
|
});
|
|
544
701
|
// 2. Tmux — ensure session, create window if not alive
|
|
545
702
|
await TmuxManager.ensureSession(this.tmuxSessionName);
|
|
546
|
-
this.tmux = new TmuxManager(this.tmuxSessionName, "");
|
|
703
|
+
this.tmux = new TmuxManager(this.tmuxSessionName, "", resolveTmuxLogicalSize(this.config.terminal));
|
|
547
704
|
// Strategy A: always start fresh Claude window (MCP server has no reconnection)
|
|
548
705
|
// Kill any existing window from previous run
|
|
549
706
|
const windowIdFile = join(this.instanceDir, "window-id");
|
|
@@ -600,7 +757,12 @@ export class Daemon extends EventEmitter {
|
|
|
600
757
|
else {
|
|
601
758
|
await new Promise(r => setTimeout(r, 5000));
|
|
602
759
|
}
|
|
603
|
-
|
|
760
|
+
// This path only runs when pasteQueueDepth > 0 — i.e. exactly when a real
|
|
761
|
+
// delivery is already in flight or queued. Without the lock the notice and
|
|
762
|
+
// that delivery race into the same pane.
|
|
763
|
+
await this.paneWriteLock.run(async () => {
|
|
764
|
+
await this.tmux?.pasteText(buildInstructionReloadNotice(this.backend?.binaryName ?? "unknown", this.name, this.instanceDir), this.systemPasteOptions());
|
|
765
|
+
});
|
|
604
766
|
// Record the value the agent has now been told about so the next
|
|
605
767
|
// unchanged restart skips the reload.
|
|
606
768
|
try {
|
|
@@ -612,8 +774,12 @@ export class Daemon extends EventEmitter {
|
|
|
612
774
|
catch { /* non-fatal */ }
|
|
613
775
|
})();
|
|
614
776
|
if (!this.config.lightweight) {
|
|
615
|
-
// 3. Pipe-pane for prompt detection
|
|
777
|
+
// 3. Pipe-pane for prompt detection. Rotate first so a ballooned log from a
|
|
778
|
+
// previous stuck splash (hundreds of MB of ANSI frames) is truncated before
|
|
779
|
+
// we attach — pipe-pane uses `cat >>` on the same inode, so copytruncate
|
|
780
|
+
// keeps the writer attached after size resets.
|
|
616
781
|
const outputLog = join(this.instanceDir, "output.log");
|
|
782
|
+
rotateLogIfNeeded(outputLog);
|
|
617
783
|
await this.tmux.pipeOutput(outputLog).catch(() => { });
|
|
618
784
|
// 4. Transcript monitor
|
|
619
785
|
this.transcriptMonitor = new TranscriptMonitor(this.instanceDir, this.logger);
|
|
@@ -629,27 +795,31 @@ export class Daemon extends EventEmitter {
|
|
|
629
795
|
this.transcriptMonitor.on("tool_use", (name, input) => {
|
|
630
796
|
this.logger.debug({ tool: name }, "Tool use");
|
|
631
797
|
ackIfPending();
|
|
632
|
-
this.hangDetector?.recordActivity();
|
|
633
798
|
this.recordRecentEvent({ type: "tool_use", name, preview: this.summarizeTool(name, input) });
|
|
634
799
|
this.recordRecentToolActivity(this.summarizeTool(name, input));
|
|
800
|
+
this.publishActivity(this.summarizeTool(name, input));
|
|
635
801
|
});
|
|
636
802
|
this.transcriptMonitor.on("tool_result", (name, _output) => {
|
|
637
|
-
this.hangDetector?.recordActivity();
|
|
638
803
|
this.recordRecentEvent({ type: "tool_result", name });
|
|
804
|
+
// The tool finished; whatever comes next has not started yet. Better to
|
|
805
|
+
// show only elapsed time than to leave a stale "Bash: npm test" on screen.
|
|
806
|
+
this.publishActivity(null);
|
|
639
807
|
});
|
|
640
808
|
this.transcriptMonitor.on("assistant_text", (text) => {
|
|
641
809
|
this.logger.debug({ text: text.slice(0, 200) }, "Claude response");
|
|
642
810
|
ackIfPending();
|
|
643
|
-
this.hangDetector?.recordActivity();
|
|
644
811
|
this.recordRecentEvent({ type: "assistant_text", preview: text.slice(0, 100) });
|
|
645
812
|
});
|
|
646
813
|
this.transcriptMonitor.startPolling();
|
|
647
|
-
// HangDetector
|
|
648
|
-
//
|
|
649
|
-
//
|
|
814
|
+
// HangDetector is the notification bridge to the fleet manager; pane state
|
|
815
|
+
// transitions below are the sole source of hang events (see hang-detector.ts).
|
|
816
|
+
// `hang_detector.enabled: false` opts out of the notification entirely.
|
|
817
|
+
// NOTE: `hang_detector.timeout_minutes` is NOT read here — the stuck timeout
|
|
818
|
+
// the pane monitor actually uses is resolved separately below. The two have
|
|
819
|
+
// long been separate; documented rather than silently ignored.
|
|
650
820
|
const hangConfig = this.config.hang_detector;
|
|
651
821
|
if (hangConfig?.enabled !== false) {
|
|
652
|
-
this.hangDetector = new HangDetector(
|
|
822
|
+
this.hangDetector = new HangDetector();
|
|
653
823
|
}
|
|
654
824
|
// 8. Context guardian
|
|
655
825
|
const statusFile = join(this.instanceDir, "statusline.json");
|
|
@@ -657,7 +827,6 @@ export class Daemon extends EventEmitter {
|
|
|
657
827
|
this.guardian.startWatching();
|
|
658
828
|
this.guardian.on("status_update", () => {
|
|
659
829
|
this.saveSessionId();
|
|
660
|
-
this.hangDetector?.recordStatuslineUpdate();
|
|
661
830
|
});
|
|
662
831
|
// Context rotation removed: all CLI backends have built-in auto-compact.
|
|
663
832
|
// Crash recovery (health check + respawn with snapshot) is retained below.
|
|
@@ -677,6 +846,40 @@ export class Daemon extends EventEmitter {
|
|
|
677
846
|
this.startInstanceStateMonitor();
|
|
678
847
|
this.logger.info(`${this.name} ready`);
|
|
679
848
|
}
|
|
849
|
+
/**
|
|
850
|
+
* Detect a CRASHED MCP server and report it once.
|
|
851
|
+
*
|
|
852
|
+
* The MCP server writes its pid to channel.mcp.pid at startup and unlinks it on
|
|
853
|
+
* a clean exit, so "file present + pid dead" specifically means it died without
|
|
854
|
+
* cleaning up (crash / OOM / SIGKILL). A missing file is either "not started
|
|
855
|
+
* yet" or "exited cleanly" — neither is an incident, and an orphan exit implies
|
|
856
|
+
* the CLI itself died, which the crash detector already covers.
|
|
857
|
+
*
|
|
858
|
+
* The daemon deliberately does NOT try to respawn it: MCP is a stdio protocol
|
|
859
|
+
* where the CLI spawns the server and owns its pipes, so a daemon-spawned
|
|
860
|
+
* process would have no client reading it. Only the CLI can restore its own
|
|
861
|
+
* tools — hence notify, and let the operator decide about restarting.
|
|
862
|
+
*/
|
|
863
|
+
checkMcpServerAlive() {
|
|
864
|
+
if (this.isPaused)
|
|
865
|
+
return;
|
|
866
|
+
const status = mcpServerState(this.instanceDir);
|
|
867
|
+
if (status.state === "unknown")
|
|
868
|
+
return;
|
|
869
|
+
if (status.state === "alive") {
|
|
870
|
+
this.mcpDeathNotifiedForPid = null; // the CLI respawned it — re-arm
|
|
871
|
+
return;
|
|
872
|
+
}
|
|
873
|
+
// Dead: report once per pid, and at most once per cooldown window.
|
|
874
|
+
if (this.mcpDeathNotifiedForPid === status.pid)
|
|
875
|
+
return;
|
|
876
|
+
if (Date.now() - this.lastMcpDeathNotifiedAt < Daemon.MCP_DEATH_COOLDOWN_MS)
|
|
877
|
+
return;
|
|
878
|
+
this.mcpDeathNotifiedForPid = status.pid;
|
|
879
|
+
this.lastMcpDeathNotifiedAt = Date.now();
|
|
880
|
+
this.logger.error({ pid: status.pid }, "MCP server process is gone — instance has no agend tools");
|
|
881
|
+
this.emit("mcp_died", { name: this.name, pid: status.pid });
|
|
882
|
+
}
|
|
680
883
|
startHealthCheck() {
|
|
681
884
|
if (this.runtimeMonitorsFrozen || this.healthCheckTimer)
|
|
682
885
|
return;
|
|
@@ -692,249 +895,285 @@ export class Daemon extends EventEmitter {
|
|
|
692
895
|
this.healthCheckTimer = null;
|
|
693
896
|
if (this.runtimeMonitorsFrozen)
|
|
694
897
|
return;
|
|
695
|
-
//
|
|
696
|
-
//
|
|
697
|
-
//
|
|
698
|
-
//
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
if (paneStatus === null) {
|
|
728
|
-
await new Promise(r => setTimeout(r, 1500));
|
|
729
|
-
paneStatus = await this.tmux.getPaneStatus();
|
|
898
|
+
// The whole tick is guarded: an unguarded throw in here (a tmux hiccup,
|
|
899
|
+
// ENOSPC on the crash-history write) became an unhandled rejection, which
|
|
900
|
+
// the CLI turned into stopAll() + exit(1) for the ENTIRE fleet, and it also
|
|
901
|
+
// ended this instance's health loop because scheduleNext() sat after the
|
|
902
|
+
// throwing code. Catch, log, and keep checking.
|
|
903
|
+
try {
|
|
904
|
+
// Instance directory removed externally (e.g. `rm -rf ~/.agend/instances/<name>`).
|
|
905
|
+
// Stop the loop permanently — otherwise every tick triggers a respawn, whose
|
|
906
|
+
// writeRotationSnapshot fails with ENOENT and gets caught as "Failed to respawn",
|
|
907
|
+
// spamming errors every ~30s forever.
|
|
908
|
+
if (!existsSync(this.instanceDir)) {
|
|
909
|
+
this.logger.warn({ instanceDir: this.instanceDir }, "Instance directory missing — stopping health check");
|
|
910
|
+
this.healthCheckPaused = true;
|
|
911
|
+
this.healthCheckTimer = null;
|
|
912
|
+
this.emitSupervisionEnded("instance directory was removed from disk", "Recreate the instance, or delete it from fleet.yaml.");
|
|
913
|
+
return;
|
|
914
|
+
}
|
|
915
|
+
if (!this.tmux || this.spawning || this.healthCheckPaused || Daemon.tmuxServerPaused) {
|
|
916
|
+
scheduleNext();
|
|
917
|
+
return;
|
|
918
|
+
}
|
|
919
|
+
// The CLI owns the MCP server process, so the daemon can only observe it.
|
|
920
|
+
this.checkMcpServerAlive();
|
|
921
|
+
// Human-readable backend label for logs (e.g. "claude", "kiro-cli")
|
|
922
|
+
const cliLabel = this.backend?.binaryName ?? "CLI";
|
|
923
|
+
let paneStatus = await this.tmux.getPaneStatus();
|
|
924
|
+
// Auto-pause intentionally exits the pane process. A health tick that
|
|
925
|
+
// began just before pause must not classify that exit as a crash.
|
|
926
|
+
if (this.isPaused || this.pauseWakeState === "waking") {
|
|
927
|
+
scheduleNext();
|
|
928
|
+
return;
|
|
929
|
+
}
|
|
730
930
|
if (paneStatus?.alive) {
|
|
731
|
-
|
|
931
|
+
// Instance output.log is fed by tmux pipe-pane and was previously never
|
|
932
|
+
// rotated (only fleet.log / daemon.log were). Cap growth every tick.
|
|
933
|
+
if (!this.config.lightweight) {
|
|
934
|
+
rotateLogIfNeeded(join(this.instanceDir, "output.log"));
|
|
935
|
+
}
|
|
732
936
|
scheduleNext();
|
|
733
937
|
return;
|
|
734
938
|
}
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
this.healthCheckPaused = true;
|
|
746
|
-
return;
|
|
747
|
-
}
|
|
748
|
-
this.setProcessStatus("crashed");
|
|
749
|
-
// Distinguish tmux server crash from single window crash.
|
|
750
|
-
// nullReason records *why* getPaneStatus returned null (for diagnosing
|
|
751
|
-
// whether this was a real window loss or a transient query failure).
|
|
752
|
-
let crashType = "window";
|
|
753
|
-
let nullReason;
|
|
754
|
-
if (!paneStatus) {
|
|
755
|
-
const serverAlive = await TmuxManager.sessionExists(this.tmuxSessionName);
|
|
756
|
-
if (!serverAlive) {
|
|
757
|
-
crashType = "server";
|
|
758
|
-
nullReason = "server_gone";
|
|
759
|
-
this.logger.error(`tmux server died — all ${cliLabel} windows lost`);
|
|
760
|
-
// Fleet-level circuit breaker: pause all instances on repeated tmux server crashes
|
|
761
|
-
Daemon.tmuxServerCrashTimestamps.push(Date.now());
|
|
762
|
-
const cutoff = Date.now() - 5 * 60_000;
|
|
763
|
-
Daemon.tmuxServerCrashTimestamps = Daemon.tmuxServerCrashTimestamps.filter(t => t > cutoff);
|
|
764
|
-
if (Daemon.tmuxServerCrashTimestamps.length >= 2 && !Daemon.tmuxServerPaused) {
|
|
765
|
-
Daemon.tmuxServerPaused = true;
|
|
766
|
-
this.logger.error("Fleet-level tmux server circuit breaker triggered — pausing all respawns for 30s");
|
|
767
|
-
this.emit("tmux_server_crash", this.name);
|
|
768
|
-
if (!Daemon.tmuxServerRecoveryTimer) {
|
|
769
|
-
Daemon.tmuxServerRecoveryTimer = setTimeout(() => {
|
|
770
|
-
Daemon.tmuxServerRecoveryTimer = null;
|
|
771
|
-
Daemon.tmuxServerPaused = false;
|
|
772
|
-
}, 30_000);
|
|
773
|
-
}
|
|
939
|
+
// A null status is ambiguous: it can be a transient `tmux list-panes`
|
|
940
|
+
// failure (e.g. tmux busy during a fleet-restart storm) rather than a
|
|
941
|
+
// real exit. Re-confirm once after a short delay before treating it as
|
|
942
|
+
// a crash. A non-null {alive:false} is a definite dead pane (real exit)
|
|
943
|
+
// and needs no recheck.
|
|
944
|
+
if (paneStatus === null) {
|
|
945
|
+
await new Promise(r => setTimeout(r, 1500));
|
|
946
|
+
paneStatus = await this.tmux.getPaneStatus();
|
|
947
|
+
if (paneStatus?.alive) {
|
|
948
|
+
this.logger.debug(`[health] ${cliLabel} pane reported gone then alive on recheck — transient query failure, ignoring`);
|
|
774
949
|
scheduleNext();
|
|
775
950
|
return;
|
|
776
951
|
}
|
|
777
|
-
await new Promise(r => setTimeout(r, 2_000)); // let session stabilize
|
|
778
952
|
}
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
953
|
+
// paneStatus === null → window gone entirely (e.g. tmux server crash)
|
|
954
|
+
// paneStatus.alive === false → pane dead, exit code available
|
|
955
|
+
const exitCode = paneStatus?.exitCode;
|
|
956
|
+
this.logger.debug({ exitCode }, `[health] pane exited with code: ${exitCode}`);
|
|
957
|
+
// Normal exit (e.g. user Ctrl+C or /exit) — no crash, no respawn
|
|
958
|
+
if (paneStatus && exitCode === 0) {
|
|
959
|
+
this.setProcessStatus("stopped");
|
|
960
|
+
this.logger.info("CLI exited normally (code 0) — pausing health check");
|
|
961
|
+
await this.tmux.killWindow();
|
|
962
|
+
this.healthCheckPaused = true;
|
|
963
|
+
this.emitSupervisionEnded("the CLI exited normally (code 0)", "Nothing crashed — start it again when you need it.");
|
|
964
|
+
return;
|
|
965
|
+
}
|
|
966
|
+
this.setProcessStatus("crashed");
|
|
967
|
+
// Distinguish tmux server crash from single window crash.
|
|
968
|
+
// nullReason records *why* getPaneStatus returned null (for diagnosing
|
|
969
|
+
// whether this was a real window loss or a transient query failure).
|
|
970
|
+
let crashType = "window";
|
|
971
|
+
let nullReason;
|
|
972
|
+
if (!paneStatus) {
|
|
973
|
+
const serverAlive = await TmuxManager.sessionExists(this.tmuxSessionName);
|
|
974
|
+
if (!serverAlive) {
|
|
975
|
+
crashType = "server";
|
|
976
|
+
nullReason = "server_gone";
|
|
977
|
+
this.logger.error(`tmux server died — all ${cliLabel} windows lost`);
|
|
978
|
+
// Fleet-level circuit breaker: pause all instances on repeated tmux server crashes
|
|
979
|
+
Daemon.tmuxServerCrashTimestamps.push(Date.now());
|
|
980
|
+
const cutoff = Date.now() - 5 * 60_000;
|
|
981
|
+
Daemon.tmuxServerCrashTimestamps = Daemon.tmuxServerCrashTimestamps.filter(t => t > cutoff);
|
|
982
|
+
if (Daemon.tmuxServerCrashTimestamps.length >= 2 && !Daemon.tmuxServerPaused) {
|
|
983
|
+
Daemon.tmuxServerPaused = true;
|
|
984
|
+
this.logger.error("Fleet-level tmux server circuit breaker triggered — pausing all respawns for 30s");
|
|
985
|
+
this.emit("tmux_server_crash", this.name);
|
|
986
|
+
if (!Daemon.tmuxServerRecoveryTimer) {
|
|
987
|
+
Daemon.tmuxServerRecoveryTimer = setTimeout(() => {
|
|
988
|
+
Daemon.tmuxServerRecoveryTimer = null;
|
|
989
|
+
Daemon.tmuxServerPaused = false;
|
|
990
|
+
}, 30_000);
|
|
991
|
+
}
|
|
992
|
+
scheduleNext();
|
|
993
|
+
return;
|
|
994
|
+
}
|
|
995
|
+
await new Promise(r => setTimeout(r, 2_000)); // let session stabilize
|
|
787
996
|
}
|
|
788
|
-
|
|
789
|
-
|
|
997
|
+
else {
|
|
998
|
+
// null but server alive: window-level disappearance. Probe whether
|
|
999
|
+
// the window truly no longer exists vs a transient query glitch.
|
|
1000
|
+
nullReason = "no_window";
|
|
1001
|
+
try {
|
|
1002
|
+
const windows = await TmuxManager.listWindows(this.tmuxSessionName);
|
|
1003
|
+
if (windows.some(w => w.name === this.name))
|
|
1004
|
+
nullReason = "window_present_query_glitch";
|
|
1005
|
+
}
|
|
1006
|
+
catch {
|
|
1007
|
+
nullReason = "query_error";
|
|
1008
|
+
}
|
|
1009
|
+
this.logger.warn({ exitCode, nullReason }, `${cliLabel} window not found (tmux server alive)`);
|
|
790
1010
|
}
|
|
791
|
-
this.logger.warn({ exitCode, nullReason }, `${cliLabel} window not found (tmux server alive)`);
|
|
792
1011
|
}
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
1012
|
+
else {
|
|
1013
|
+
this.logger.warn({ exitCode }, `${cliLabel} process exited`);
|
|
1014
|
+
}
|
|
1015
|
+
// Capture last output before killing. Best-effort even when the pane is
|
|
1016
|
+
// gone (paneStatus null) — gives the crash record something to diagnose
|
|
1017
|
+
// from instead of an empty lastOutput.
|
|
1018
|
+
let lastOutput;
|
|
1019
|
+
try {
|
|
1020
|
+
const raw = await this.tmux.capturePaneWithHistory(50);
|
|
1021
|
+
// Strip ANSI escape codes for readability
|
|
1022
|
+
const cleaned = raw.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
|
|
1023
|
+
lastOutput = cleaned.trimEnd() || undefined;
|
|
1024
|
+
}
|
|
1025
|
+
catch { /* best effort — pane may already be gone */ }
|
|
1026
|
+
// Kill the dead window (remain-on-exit keeps it around) before respawn
|
|
1027
|
+
if (paneStatus) {
|
|
1028
|
+
await this.tmux.killWindow();
|
|
1029
|
+
}
|
|
1030
|
+
// Detect claude-code background session conflict — recover without counting as crash
|
|
1031
|
+
if (lastOutput && (lastOutput.includes("background agent") || lastOutput.includes("Session is currently running"))) {
|
|
1032
|
+
if (!this.backgroundSessionRecoveryAttempted) {
|
|
1033
|
+
this.backgroundSessionRecoveryAttempted = true;
|
|
1034
|
+
this.logger.warn("Detected lingering background agent session — starting fresh (no resume)");
|
|
1035
|
+
const sidFile = join(this.instanceDir, "session-id");
|
|
1036
|
+
try {
|
|
1037
|
+
unlinkSync(sidFile);
|
|
1038
|
+
}
|
|
1039
|
+
catch { }
|
|
1040
|
+
this.skipResume = true;
|
|
1041
|
+
await new Promise(r => setTimeout(r, 2_000));
|
|
1042
|
+
try {
|
|
1043
|
+
await this.spawnClaudeWindow();
|
|
1044
|
+
this.setProcessStatus("running");
|
|
1045
|
+
this.logger.info("Recovered from background session conflict");
|
|
1046
|
+
this.emit("crash_respawn", this.name);
|
|
1047
|
+
}
|
|
1048
|
+
catch (err) {
|
|
1049
|
+
this.logger.error({ err: err.message }, "Recovery from background session conflict failed");
|
|
1050
|
+
}
|
|
1051
|
+
return; // Don't count as crash
|
|
1052
|
+
}
|
|
1053
|
+
// Already attempted recovery — fall through to normal crash handling
|
|
1054
|
+
}
|
|
1055
|
+
// Detect a --continue/--resume failure (no conversation to resume). The
|
|
1056
|
+
// session-id file persists across the crash, so a blind respawn would add
|
|
1057
|
+
// --continue again and crash in the same way → loop. Clear the session id
|
|
1058
|
+
// and skip resume so the next spawn starts fresh. (skipResume also stops
|
|
1059
|
+
// saveSessionId below from resurrecting the id from statusline.json.)
|
|
1060
|
+
if (lastOutput && /no conversation found|no conversation to (continue|resume)|no previous (session|conversation)|--continue/i.test(lastOutput)) {
|
|
1061
|
+
this.logger.warn("Detected --continue/resume failure — clearing session-id; next spawn starts fresh");
|
|
818
1062
|
try {
|
|
819
|
-
unlinkSync(
|
|
1063
|
+
unlinkSync(join(this.instanceDir, "session-id"));
|
|
820
1064
|
}
|
|
821
|
-
catch { }
|
|
1065
|
+
catch { /* may not exist */ }
|
|
822
1066
|
this.skipResume = true;
|
|
823
|
-
|
|
1067
|
+
}
|
|
1068
|
+
// Append to crash history
|
|
1069
|
+
this.appendCrashHistory({ exitCode, lastOutput, crashType, reason: nullReason });
|
|
1070
|
+
if (max_retries <= 0) {
|
|
1071
|
+
this.healthCheckPaused = true;
|
|
1072
|
+
this.logger.warn(`${cliLabel} window died — automatic restart is disabled`);
|
|
1073
|
+
this.emitSupervisionEnded("the CLI died and automatic restart is disabled (restart_policy.max_retries is 0)", "Start it manually, or raise max_retries in fleet.yaml.");
|
|
1074
|
+
return;
|
|
1075
|
+
}
|
|
1076
|
+
// Detect rapid crash: sliding window — 3+ crashes in 5 minutes
|
|
1077
|
+
this.crashTimestamps.push(Date.now());
|
|
1078
|
+
const crashWindowMs = 5 * 60_000;
|
|
1079
|
+
this.crashTimestamps = this.crashTimestamps.filter(t => t > Date.now() - crashWindowMs);
|
|
1080
|
+
if (this.crashTimestamps.length >= 3) {
|
|
1081
|
+
this.healthCheckPaused = true;
|
|
1082
|
+
this.logger.error({ crashesInWindow: this.crashTimestamps.length }, "3+ crashes in 5 minutes — pausing respawn");
|
|
1083
|
+
// P1: Persist crash state so next process restart skips resume
|
|
824
1084
|
try {
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
1085
|
+
writeFileSync(join(this.instanceDir, "crash-state.json"), JSON.stringify({
|
|
1086
|
+
crashesInWindow: this.crashTimestamps.length,
|
|
1087
|
+
lastCrashAt: Date.now(),
|
|
1088
|
+
resumeDisabled: true,
|
|
1089
|
+
}));
|
|
829
1090
|
}
|
|
830
|
-
catch
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
return; // Don't count as crash
|
|
1091
|
+
catch { /* best effort */ }
|
|
1092
|
+
this.emit("crash_loop", this.name);
|
|
1093
|
+
return; // don't schedule next — paused
|
|
834
1094
|
}
|
|
835
|
-
//
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
// session-id file persists across the crash, so a blind respawn would add
|
|
839
|
-
// --continue again and crash in the same way → loop. Clear the session id
|
|
840
|
-
// and skip resume so the next spawn starts fresh. (skipResume also stops
|
|
841
|
-
// saveSessionId below from resurrecting the id from statusline.json.)
|
|
842
|
-
if (lastOutput && /no conversation found|no conversation to (continue|resume)|no previous (session|conversation)|--continue/i.test(lastOutput)) {
|
|
843
|
-
this.logger.warn("Detected --continue/resume failure — clearing session-id; next spawn starts fresh");
|
|
844
|
-
try {
|
|
845
|
-
unlinkSync(join(this.instanceDir, "session-id"));
|
|
1095
|
+
// Reset crash count if enough time has passed
|
|
1096
|
+
if (reset_after > 0 && Date.now() - this.lastCrashAt > reset_after) {
|
|
1097
|
+
this.crashCount = 0;
|
|
846
1098
|
}
|
|
847
|
-
|
|
848
|
-
this.
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
this.logger.warn(`${cliLabel} window died — automatic restart is disabled`);
|
|
855
|
-
return;
|
|
856
|
-
}
|
|
857
|
-
// Detect rapid crash: sliding window — 3+ crashes in 5 minutes
|
|
858
|
-
this.crashTimestamps.push(Date.now());
|
|
859
|
-
const crashWindowMs = 5 * 60_000;
|
|
860
|
-
this.crashTimestamps = this.crashTimestamps.filter(t => t > Date.now() - crashWindowMs);
|
|
861
|
-
if (this.crashTimestamps.length >= 3) {
|
|
862
|
-
this.healthCheckPaused = true;
|
|
863
|
-
this.logger.error({ crashesInWindow: this.crashTimestamps.length }, "3+ crashes in 5 minutes — pausing respawn");
|
|
864
|
-
// P1: Persist crash state so next process restart skips resume
|
|
865
|
-
try {
|
|
866
|
-
writeFileSync(join(this.instanceDir, "crash-state.json"), JSON.stringify({
|
|
867
|
-
crashesInWindow: this.crashTimestamps.length,
|
|
868
|
-
lastCrashAt: Date.now(),
|
|
869
|
-
resumeDisabled: true,
|
|
870
|
-
}));
|
|
1099
|
+
this.crashCount++;
|
|
1100
|
+
this.lastCrashAt = Date.now();
|
|
1101
|
+
if (this.crashCount > max_retries) {
|
|
1102
|
+
this.logger.error({ crashCount: this.crashCount, maxRetries: max_retries }, "Max crash retries exceeded — not respawning");
|
|
1103
|
+
this.healthCheckPaused = true;
|
|
1104
|
+
this.emitSupervisionEnded(`it crashed ${this.crashCount} times, exceeding restart_policy.max_retries (${max_retries})`, "Check the logs for the cause, then restart it.");
|
|
1105
|
+
return; // don't schedule next — given up
|
|
871
1106
|
}
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
this.crashCount = 0;
|
|
879
|
-
}
|
|
880
|
-
this.crashCount++;
|
|
881
|
-
this.lastCrashAt = Date.now();
|
|
882
|
-
if (this.crashCount > max_retries) {
|
|
883
|
-
this.logger.error({ crashCount: this.crashCount, maxRetries: max_retries }, "Max crash retries exceeded — not respawning");
|
|
884
|
-
return; // don't schedule next — given up
|
|
885
|
-
}
|
|
886
|
-
// Calculate backoff delay
|
|
887
|
-
const delay = backoff === "exponential"
|
|
888
|
-
? Math.min(1000 * Math.pow(2, this.crashCount - 1), 60_000)
|
|
889
|
-
: 1000 * this.crashCount;
|
|
890
|
-
this.logger.warn({ crashCount: this.crashCount, delay }, `${cliLabel} window died — respawning after backoff`);
|
|
891
|
-
await new Promise(r => setTimeout(r, delay));
|
|
892
|
-
try {
|
|
893
|
-
this.saveSessionId();
|
|
894
|
-
this.transcriptMonitor?.resetOffset();
|
|
895
|
-
// Kill orphan MCP server from the crashed CLI session.
|
|
896
|
-
// MCP server writes its PID to channel.mcp.pid on startup.
|
|
1107
|
+
// Calculate backoff delay
|
|
1108
|
+
const delay = backoff === "exponential"
|
|
1109
|
+
? Math.min(1000 * Math.pow(2, this.crashCount - 1), 60_000)
|
|
1110
|
+
: 1000 * this.crashCount;
|
|
1111
|
+
this.logger.warn({ crashCount: this.crashCount, delay }, `${cliLabel} window died — respawning after backoff`);
|
|
1112
|
+
await new Promise(r => setTimeout(r, delay));
|
|
897
1113
|
try {
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
try {
|
|
908
|
-
const windows = await TmuxManager.listWindows(this.tmuxSessionName);
|
|
909
|
-
for (const w of windows) {
|
|
910
|
-
if (w.name === this.name) {
|
|
911
|
-
const tm = new TmuxManager(this.tmuxSessionName, w.id);
|
|
912
|
-
await tm.killWindow();
|
|
913
|
-
}
|
|
1114
|
+
this.saveSessionId();
|
|
1115
|
+
this.transcriptMonitor?.resetOffset();
|
|
1116
|
+
// Kill orphan MCP server from the crashed CLI session.
|
|
1117
|
+
// MCP server writes its PID to channel.mcp.pid on startup.
|
|
1118
|
+
try {
|
|
1119
|
+
const pidFile = join(this.instanceDir, "channel.mcp.pid");
|
|
1120
|
+
const pid = parseInt(readFileSync(pidFile, "utf-8").trim(), 10);
|
|
1121
|
+
process.kill(pid, "SIGTERM");
|
|
1122
|
+
this.logger.info({ pid }, "Killed orphan MCP server");
|
|
914
1123
|
}
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
// Try --resume first; spawnClaudeWindow falls back to fresh session if resume fails
|
|
920
|
-
const resumed = await this.spawnClaudeWindow();
|
|
921
|
-
if (!resumed) {
|
|
922
|
-
// Resume failed → fresh session → inject snapshot for context
|
|
923
|
-
await this.injectSnapshotMessage();
|
|
924
|
-
}
|
|
925
|
-
else {
|
|
926
|
-
// Clean up stale snapshot — resume restored full context
|
|
1124
|
+
catch { /* no pid file or process already dead */ }
|
|
1125
|
+
// Kill any same-name windows before respawn to prevent orphans.
|
|
1126
|
+
// Wrapped in try-catch: if tmux server is dead, listWindows throws —
|
|
1127
|
+
// must not block spawnClaudeWindow (which calls ensureSession).
|
|
927
1128
|
try {
|
|
928
|
-
|
|
1129
|
+
const windows = await TmuxManager.listWindows(this.tmuxSessionName);
|
|
1130
|
+
for (const w of windows) {
|
|
1131
|
+
if (w.name === this.name) {
|
|
1132
|
+
const tm = new TmuxManager(this.tmuxSessionName, w.id);
|
|
1133
|
+
await tm.killWindow();
|
|
1134
|
+
}
|
|
1135
|
+
}
|
|
929
1136
|
}
|
|
930
|
-
catch { /* may
|
|
1137
|
+
catch { /* tmux server may be dead — ensureSession in trySpawn will recover */ }
|
|
1138
|
+
// Write snapshot before spawn — consumed only if resume fails
|
|
1139
|
+
this.writeRotationSnapshot("crash");
|
|
1140
|
+
// Try --resume first; spawnClaudeWindow falls back to fresh session if resume fails
|
|
1141
|
+
const resumed = await this.spawnClaudeWindow();
|
|
1142
|
+
if (!resumed) {
|
|
1143
|
+
// Resume failed → fresh session → inject snapshot for context
|
|
1144
|
+
await this.injectSnapshotMessage();
|
|
1145
|
+
}
|
|
1146
|
+
else {
|
|
1147
|
+
// Clean up stale snapshot — resume restored full context
|
|
1148
|
+
try {
|
|
1149
|
+
unlinkSync(join(this.instanceDir, "rotation-state.json"));
|
|
1150
|
+
}
|
|
1151
|
+
catch { /* may not exist */ }
|
|
1152
|
+
}
|
|
1153
|
+
this.setProcessStatus("running");
|
|
1154
|
+
this.logger.info({ resumed }, `Respawned ${cliLabel} window after crash`);
|
|
1155
|
+
this.emit("crash_respawn", this.name);
|
|
1156
|
+
}
|
|
1157
|
+
catch (err) {
|
|
1158
|
+
this.logger.error({ err }, `Failed to respawn ${cliLabel} window`);
|
|
931
1159
|
}
|
|
932
|
-
this.setProcessStatus("running");
|
|
933
|
-
this.logger.info({ resumed }, `Respawned ${cliLabel} window after crash`);
|
|
934
|
-
this.emit("crash_respawn", this.name);
|
|
935
1160
|
}
|
|
936
1161
|
catch (err) {
|
|
937
|
-
this.logger.error({ err },
|
|
1162
|
+
this.logger.error({ err }, "Health check tick failed — continuing");
|
|
1163
|
+
// Surface it to the operator, not just the log — a health check that
|
|
1164
|
+
// keeps throwing means this instance is no longer being supervised.
|
|
1165
|
+
// Throttled: the tick repeats every ~30s, so an unnotified persistent
|
|
1166
|
+
// fault would otherwise post twice a minute forever.
|
|
1167
|
+
const now = Date.now();
|
|
1168
|
+
if (now - this.lastHealthErrorNotifyAt > HEALTH_ERROR_NOTIFY_INTERVAL_MS) {
|
|
1169
|
+
this.lastHealthErrorNotifyAt = now;
|
|
1170
|
+
this.emit("health_check_error", {
|
|
1171
|
+
name: this.name,
|
|
1172
|
+
message: err instanceof Error ? err.message : String(err),
|
|
1173
|
+
});
|
|
1174
|
+
}
|
|
1175
|
+
scheduleNext();
|
|
1176
|
+
return;
|
|
938
1177
|
}
|
|
939
1178
|
scheduleNext();
|
|
940
1179
|
}, healthCheckIntervalMs);
|
|
@@ -946,6 +1185,21 @@ export class Daemon extends EventEmitter {
|
|
|
946
1185
|
* A dead remain-on-exit pane still contains the old ready prompt, so allowing
|
|
947
1186
|
* the pane monitor to capture it would re-create a false idle state.
|
|
948
1187
|
*/
|
|
1188
|
+
/**
|
|
1189
|
+
* Announce that this instance is no longer being supervised.
|
|
1190
|
+
*
|
|
1191
|
+
* Four health-check exits set `healthCheckPaused = true` and returned without
|
|
1192
|
+
* telling anyone: a clean CLI exit, `max_retries <= 0`, crash retries exhausted,
|
|
1193
|
+
* and the instance directory being deleted. Only the crash-LOOP case emitted an
|
|
1194
|
+
* event, so a fleet could quietly contain a dead instance that still looked fine
|
|
1195
|
+
* on the dashboard — while messages routed to it queued or failed with a bare ❌.
|
|
1196
|
+
*
|
|
1197
|
+
* `crash_loop` already had this treatment; this is the same bridge for the other
|
|
1198
|
+
* four, carrying a human-readable cause and the operator's next step.
|
|
1199
|
+
*/
|
|
1200
|
+
emitSupervisionEnded(reason, remedy) {
|
|
1201
|
+
this.emit("supervision_ended", { name: this.name, reason, remedy });
|
|
1202
|
+
}
|
|
949
1203
|
setProcessStatus(status) {
|
|
950
1204
|
if (this.processStatus === status)
|
|
951
1205
|
return;
|
|
@@ -991,6 +1245,7 @@ export class Daemon extends EventEmitter {
|
|
|
991
1245
|
if (!this.backend)
|
|
992
1246
|
return; // lightweight mode has no backend
|
|
993
1247
|
const readyPattern = this.backend.getReadyPattern();
|
|
1248
|
+
const busyPattern = this.backend.getBusyPattern?.() ?? null;
|
|
994
1249
|
this.errorMonitorTimer = setInterval(async () => {
|
|
995
1250
|
if (!this.tmux || this.spawning)
|
|
996
1251
|
return;
|
|
@@ -1003,20 +1258,30 @@ export class Daemon extends EventEmitter {
|
|
|
1003
1258
|
for (const dialog of dialogs) {
|
|
1004
1259
|
if (!dialog.pattern.test(pane))
|
|
1005
1260
|
continue;
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1261
|
+
// These keys go straight into the pane. Sent while a message delivery is
|
|
1262
|
+
// mid-transaction, an `Escape` wipes the pasted text and an `Enter`
|
|
1263
|
+
// submits it half-composed — the user sees a message that vanished. Skip
|
|
1264
|
+
// (not queue) when the pane is busy: this poller runs every 5s, and the
|
|
1265
|
+
// dialog will still be on screen next tick.
|
|
1266
|
+
const dismissed = await this.paneWriteLock.tryRun(async () => {
|
|
1267
|
+
this.logger.info(`Auto-dismissing runtime dialog: ${dialog.description}`);
|
|
1268
|
+
const SPECIAL_KEYS = new Set(["Up", "Down", "Enter", "Escape", "Right", "Left"]);
|
|
1269
|
+
for (const key of dialog.keys) {
|
|
1270
|
+
if (SPECIAL_KEYS.has(key)) {
|
|
1271
|
+
await this.tmux.sendSpecialKey(key);
|
|
1272
|
+
}
|
|
1273
|
+
else {
|
|
1274
|
+
await this.tmux.pasteText(key, this.systemPasteOptions());
|
|
1275
|
+
}
|
|
1276
|
+
await new Promise(r => setTimeout(r, 200));
|
|
1014
1277
|
}
|
|
1015
|
-
|
|
1278
|
+
});
|
|
1279
|
+
if (!dismissed) {
|
|
1280
|
+
this.logger.debug({ dialog: dialog.description }, "Dialog dismissal deferred — pane write in flight");
|
|
1016
1281
|
}
|
|
1017
|
-
return; // Dialog
|
|
1282
|
+
return; // Dialog handled (or deliberately deferred): skip error checks this cycle
|
|
1018
1283
|
}
|
|
1019
|
-
this.evaluateErrorPatterns(pane, patterns, readyPattern);
|
|
1284
|
+
this.evaluateErrorPatterns(pane, patterns, readyPattern, Date.now(), busyPattern);
|
|
1020
1285
|
}
|
|
1021
1286
|
catch {
|
|
1022
1287
|
// capturePane can fail if window is transitioning — ignore
|
|
@@ -1024,7 +1289,7 @@ export class Daemon extends EventEmitter {
|
|
|
1024
1289
|
}, 5_000); // Check every 5 seconds (runtime dialogs need fast response)
|
|
1025
1290
|
}
|
|
1026
1291
|
/** Evaluate one pane snapshot. Kept synchronous so state-machine edges are unit-testable. */
|
|
1027
|
-
evaluateErrorPatterns(pane, patterns, readyPattern, now = Date.now()) {
|
|
1292
|
+
evaluateErrorPatterns(pane, patterns, readyPattern, now = Date.now(), busyPattern = null) {
|
|
1028
1293
|
// Count occurrences across the WHOLE pane (not just text after the last
|
|
1029
1294
|
// ready prompt). Clone with `g` so stateful backend regexes cannot leak
|
|
1030
1295
|
// lastIndex between monitor cycles.
|
|
@@ -1032,10 +1297,15 @@ export class Daemon extends EventEmitter {
|
|
|
1032
1297
|
const flags = pattern.flags.includes("g") ? pattern.flags : pattern.flags + "g";
|
|
1033
1298
|
return (pane.match(new RegExp(pattern.source, flags)) || []).length;
|
|
1034
1299
|
};
|
|
1300
|
+
// Same veto as the state machine: a backend whose ready marker is permanently
|
|
1301
|
+
// on screen would otherwise "recover" on the very first tick after the error,
|
|
1302
|
+
// rebaselining the occurrence count while the error is still displayed — one
|
|
1303
|
+
// notification, a false recovery log, then silence.
|
|
1304
|
+
const looksReady = () => !busyPattern?.test(pane) && readyPattern.test(pane);
|
|
1035
1305
|
// State: waiting for recovery. A missing/outdated ready pattern must not
|
|
1036
1306
|
// suppress every future error forever, so the gate has a hard deadline.
|
|
1037
1307
|
if (this.errorWaitingForRecovery) {
|
|
1038
|
-
if (
|
|
1308
|
+
if (looksReady()) {
|
|
1039
1309
|
const downtime = Math.round((now - this.errorDetectedAt) / 1000);
|
|
1040
1310
|
const active = patterns.find(ep => Daemon.errorPatternKey(ep) === this.activeErrorPatternKey);
|
|
1041
1311
|
if (active) {
|
|
@@ -1101,20 +1371,74 @@ export class Daemon extends EventEmitter {
|
|
|
1101
1371
|
this.lastErrorNotifiedAt.set(key, now);
|
|
1102
1372
|
if (ep.action === "failover")
|
|
1103
1373
|
this.lastFailoverAt = now;
|
|
1104
|
-
|
|
1105
|
-
this.
|
|
1374
|
+
const message = this.resolveErrorMessage(pane, ep);
|
|
1375
|
+
this.logger.warn({ errorType: ep.type, action: ep.action }, `PTY error detected: ${message}`);
|
|
1376
|
+
this.emit("pty_error", { name: this.name, ...ep, message });
|
|
1106
1377
|
break; // Only handle first unsuppressed new error per scan
|
|
1107
1378
|
}
|
|
1108
1379
|
}
|
|
1380
|
+
/**
|
|
1381
|
+
* Notification text for a detected pattern. `formatMessage` patterns build it
|
|
1382
|
+
* from the match so the user gets the specifics (e.g. which period and what
|
|
1383
|
+
* percentage remains) instead of a generic "running low".
|
|
1384
|
+
*
|
|
1385
|
+
* Uses the LAST match in the pane: an older warning may still be scrolled up
|
|
1386
|
+
* (10% earlier, 5% now), and the newest one is the one worth reporting.
|
|
1387
|
+
*/
|
|
1388
|
+
resolveErrorMessage(pane, ep) {
|
|
1389
|
+
if (!ep.formatMessage)
|
|
1390
|
+
return ep.message;
|
|
1391
|
+
try {
|
|
1392
|
+
const flags = ep.pattern.flags.includes("g") ? ep.pattern.flags : ep.pattern.flags + "g";
|
|
1393
|
+
const matches = [...pane.matchAll(new RegExp(ep.pattern.source, flags))];
|
|
1394
|
+
const last = matches[matches.length - 1];
|
|
1395
|
+
return last ? ep.formatMessage(last) : ep.message;
|
|
1396
|
+
}
|
|
1397
|
+
catch (err) {
|
|
1398
|
+
// A bad formatter must not swallow the notification entirely.
|
|
1399
|
+
this.logger.debug({ err, errorType: ep.type }, "formatMessage failed — using static message");
|
|
1400
|
+
return ep.message;
|
|
1401
|
+
}
|
|
1402
|
+
}
|
|
1109
1403
|
/**
|
|
1110
1404
|
* Interrupt the CLI's current generation (cancel button / `/cancel`).
|
|
1111
1405
|
* Direct tmux key event (not a paste) so it registers as the interrupt key.
|
|
1112
1406
|
* kiro-cli interrupts on Ctrl+C; the others (claude-code, codex, …) on Escape.
|
|
1407
|
+
*
|
|
1408
|
+
* Deliberately NOT taken through `paneWriteLock`. Cancel is the user's way out
|
|
1409
|
+
* of a pane that is busy or wedged — queueing it behind the very delivery chain
|
|
1410
|
+
* it exists to unblock would make the button useless exactly when it is needed.
|
|
1411
|
+
* The cost is accepted: an Escape landing between a paste and its Enter discards
|
|
1412
|
+
* that message, which is what the user asked for anyway.
|
|
1113
1413
|
*/
|
|
1114
1414
|
async sendEscape() {
|
|
1115
1415
|
const cancelKey = this.backend?.getCancelKey() ?? "Escape";
|
|
1116
1416
|
await this.tmux?.sendSpecialKey(cancelKey);
|
|
1117
1417
|
}
|
|
1418
|
+
/** Send the backend-specific graceful quit command/key sequence. */
|
|
1419
|
+
async sendQuitSequence() {
|
|
1420
|
+
if (!this.tmux || !this.backend)
|
|
1421
|
+
return false;
|
|
1422
|
+
const quitCmd = this.backend.getQuitCommand();
|
|
1423
|
+
const quitKey = this.backend.getQuitKey?.();
|
|
1424
|
+
if (quitCmd) {
|
|
1425
|
+
await this.tmux.sendKeys(quitCmd);
|
|
1426
|
+
// Delay before Enter to prevent tmux server races when instances stop in
|
|
1427
|
+
// parallel (same pattern as pasteText).
|
|
1428
|
+
await new Promise(r => setTimeout(r, 150));
|
|
1429
|
+
await this.tmux.sendSpecialKey("Enter");
|
|
1430
|
+
return true;
|
|
1431
|
+
}
|
|
1432
|
+
if (!quitKey)
|
|
1433
|
+
return false;
|
|
1434
|
+
const presses = Math.max(1, Math.floor(this.backend.getQuitKeyPresses?.() ?? 1));
|
|
1435
|
+
for (let i = 0; i < presses; i++) {
|
|
1436
|
+
await this.tmux.sendSpecialKey(quitKey);
|
|
1437
|
+
if (i + 1 < presses)
|
|
1438
|
+
await new Promise(r => setTimeout(r, 250));
|
|
1439
|
+
}
|
|
1440
|
+
return true;
|
|
1441
|
+
}
|
|
1118
1442
|
async stop() {
|
|
1119
1443
|
this.logger.info("Stopping daemon instance");
|
|
1120
1444
|
this.freezeRuntimeMonitors();
|
|
@@ -1123,7 +1447,6 @@ export class Daemon extends EventEmitter {
|
|
|
1123
1447
|
this.toolStatusDebounce = null;
|
|
1124
1448
|
}
|
|
1125
1449
|
this.pendingIpcRequests.clear();
|
|
1126
|
-
this.hangDetector?.stop();
|
|
1127
1450
|
if (this.adapter)
|
|
1128
1451
|
await this.adapter.stop();
|
|
1129
1452
|
// Notify MCP servers of graceful shutdown (prevents reconnect attempts)
|
|
@@ -1134,20 +1457,8 @@ export class Daemon extends EventEmitter {
|
|
|
1134
1457
|
this.saveSessionId();
|
|
1135
1458
|
this.healthCheckPaused = true;
|
|
1136
1459
|
let killed = false;
|
|
1137
|
-
const
|
|
1138
|
-
|
|
1139
|
-
if (quitCmd) {
|
|
1140
|
-
await this.tmux.sendKeys(quitCmd);
|
|
1141
|
-
// Delay before Enter to prevent tmux server race when multiple
|
|
1142
|
-
// instances stop in parallel (same pattern as pasteText).
|
|
1143
|
-
await new Promise(r => setTimeout(r, 150));
|
|
1144
|
-
await this.tmux.sendSpecialKey("Enter");
|
|
1145
|
-
}
|
|
1146
|
-
else if (quitKey) {
|
|
1147
|
-
// Some CLIs quit via a key chord (e.g. grok Ctrl+Q), not a typed command.
|
|
1148
|
-
await this.tmux.sendSpecialKey(quitKey);
|
|
1149
|
-
}
|
|
1150
|
-
if (quitCmd || quitKey) {
|
|
1460
|
+
const quitSent = await this.sendQuitSequence();
|
|
1461
|
+
if (quitSent) {
|
|
1151
1462
|
// Wait up to 3s for graceful exit, polling every 200ms. A healthy CLI
|
|
1152
1463
|
// exits within ~1s; a longer wait just delays the force-kill fallback.
|
|
1153
1464
|
for (let i = 0; i < 15; i++) {
|
|
@@ -1162,7 +1473,12 @@ export class Daemon extends EventEmitter {
|
|
|
1162
1473
|
if (!killed)
|
|
1163
1474
|
this.logger.warn("CLI did not exit gracefully within 3s, force killing window");
|
|
1164
1475
|
// Always kill window — remain-on-exit keeps dead panes around after CLI exits
|
|
1476
|
+
const stoppedWindowId = this.tmux.getWindowId();
|
|
1165
1477
|
await this.tmux.killWindow();
|
|
1478
|
+
// The control client is shared across the fleet and outlives this daemon, so
|
|
1479
|
+
// a registration left behind here is re-resolved on every reconnect forever.
|
|
1480
|
+
if (stoppedWindowId)
|
|
1481
|
+
this.controlClient?.unregisterWindow(stoppedWindowId);
|
|
1166
1482
|
const windowIdFile = join(this.instanceDir, "window-id");
|
|
1167
1483
|
try {
|
|
1168
1484
|
unlinkSync(windowIdFile);
|
|
@@ -1205,6 +1521,16 @@ export class Daemon extends EventEmitter {
|
|
|
1205
1521
|
};
|
|
1206
1522
|
}
|
|
1207
1523
|
/** Gracefully stop the CLI while keeping its remain-on-exit tmux window. */
|
|
1524
|
+
/**
|
|
1525
|
+
* Mark the instance to pause as soon as its pane is idle. Used for auth
|
|
1526
|
+
* failures: pause() alone no-ops while the CLI is busy/stuck, which is the
|
|
1527
|
+
* usual state when the error surfaces. Cleared by a successful pause or wake.
|
|
1528
|
+
*/
|
|
1529
|
+
requestPauseWhenIdle() {
|
|
1530
|
+
if (this.pauseWakeState === "paused")
|
|
1531
|
+
return;
|
|
1532
|
+
this.pausePending = true;
|
|
1533
|
+
}
|
|
1208
1534
|
async pause() {
|
|
1209
1535
|
if (this.pauseWakeState === "paused")
|
|
1210
1536
|
return;
|
|
@@ -1223,19 +1549,7 @@ export class Daemon extends EventEmitter {
|
|
|
1223
1549
|
const transition = (async () => {
|
|
1224
1550
|
try {
|
|
1225
1551
|
this.saveSessionId();
|
|
1226
|
-
|
|
1227
|
-
const quitKey = this.backend?.getQuitKey?.();
|
|
1228
|
-
if (this.tmux) {
|
|
1229
|
-
if (quitCmd) {
|
|
1230
|
-
await this.tmux.sendKeys(quitCmd);
|
|
1231
|
-
await new Promise(r => setTimeout(r, 150));
|
|
1232
|
-
await this.tmux.sendSpecialKey("Enter");
|
|
1233
|
-
}
|
|
1234
|
-
else if (quitKey) {
|
|
1235
|
-
// Key-chord quit (e.g. grok Ctrl+Q) — no typed command to send.
|
|
1236
|
-
await this.tmux.sendSpecialKey(quitKey);
|
|
1237
|
-
}
|
|
1238
|
-
}
|
|
1552
|
+
await this.sendQuitSequence();
|
|
1239
1553
|
let exited = false;
|
|
1240
1554
|
for (let i = 0; i < 15; i++) {
|
|
1241
1555
|
await new Promise(r => setTimeout(r, 200));
|
|
@@ -1286,6 +1600,9 @@ export class Daemon extends EventEmitter {
|
|
|
1286
1600
|
}
|
|
1287
1601
|
/** Respawn the CLI in the preserved window and block until its prompt is ready. */
|
|
1288
1602
|
async wake(timeoutMs = 30_000) {
|
|
1603
|
+
// An explicit wake (e.g. after the user re-logs in) cancels a deferred
|
|
1604
|
+
// auth pause — otherwise the instance would pause again the moment it idles.
|
|
1605
|
+
this.pausePending = false;
|
|
1289
1606
|
if (this.pauseWakeState === "active")
|
|
1290
1607
|
return;
|
|
1291
1608
|
if (this.pauseWakeState === "pausing")
|
|
@@ -1297,7 +1614,7 @@ export class Daemon extends EventEmitter {
|
|
|
1297
1614
|
return;
|
|
1298
1615
|
}
|
|
1299
1616
|
this.pauseWakeState = "waking";
|
|
1300
|
-
this.
|
|
1617
|
+
this.beginSpawn();
|
|
1301
1618
|
const transition = this.autoPauseController.wakeOnDeliver(async () => {
|
|
1302
1619
|
let timeout;
|
|
1303
1620
|
try {
|
|
@@ -1338,7 +1655,7 @@ export class Daemon extends EventEmitter {
|
|
|
1338
1655
|
throw err;
|
|
1339
1656
|
}
|
|
1340
1657
|
finally {
|
|
1341
|
-
this.
|
|
1658
|
+
this.endSpawn();
|
|
1342
1659
|
if (this.pauseWakeTransition === transition)
|
|
1343
1660
|
this.pauseWakeTransition = null;
|
|
1344
1661
|
}
|
|
@@ -1350,6 +1667,10 @@ export class Daemon extends EventEmitter {
|
|
|
1350
1667
|
// observations between enqueue and paste must not clear a newer inbound.
|
|
1351
1668
|
if (snapshot.state === "idle" && previous !== "idle") {
|
|
1352
1669
|
this.pendingWork.recordIdle(snapshot.observedAt);
|
|
1670
|
+
// The turn is over. A transcript can end on a tool_use with no matching
|
|
1671
|
+
// tool_result (interrupted, crashed, cancelled), which would otherwise leave
|
|
1672
|
+
// the last tool pinned to the progress line for the rest of the session.
|
|
1673
|
+
this.publishActivity(null);
|
|
1353
1674
|
}
|
|
1354
1675
|
if (snapshot.state !== previous) {
|
|
1355
1676
|
this.logger.info({
|
|
@@ -1365,6 +1686,12 @@ export class Daemon extends EventEmitter {
|
|
|
1365
1686
|
}
|
|
1366
1687
|
if (snapshot.state !== "idle")
|
|
1367
1688
|
this.pauseRequested = false;
|
|
1689
|
+
// A pause deferred by an auth failure: retry the moment the pane settles.
|
|
1690
|
+
if (this.pausePending && snapshot.state === "idle" && this.pasteQueueDepth === 0) {
|
|
1691
|
+
this.pausePending = false;
|
|
1692
|
+
this.emit("auto_pause_requested", { name: this.name, idleSince: snapshot.stateChangedAt });
|
|
1693
|
+
return;
|
|
1694
|
+
}
|
|
1368
1695
|
if (!this.pauseRequested && this.pasteQueueDepth === 0 && this.autoPauseController.observe(snapshot.state)) {
|
|
1369
1696
|
this.pauseRequested = true;
|
|
1370
1697
|
this.emit("auto_pause_requested", { name: this.name, idleSince: snapshot.stateChangedAt });
|
|
@@ -1434,6 +1761,13 @@ export class Daemon extends EventEmitter {
|
|
|
1434
1761
|
this.instanceStateMachine.observe(pane, observedChangeAt);
|
|
1435
1762
|
const snapshot = this.instanceStateMachine.observe(pane, Date.now());
|
|
1436
1763
|
this.applyInstanceStateSnapshot(snapshot, pane);
|
|
1764
|
+
// Backends without a transcript feed can still say what they are doing, if
|
|
1765
|
+
// their TUI names it. Free-riding on a capture that already happened: no
|
|
1766
|
+
// extra tmux call. Cadence is the capture cadence (idle debounce + the 60s
|
|
1767
|
+
// safety sweep), which matches the progress ticker's own 60s interval.
|
|
1768
|
+
if (this.backend?.getPaneActivity) {
|
|
1769
|
+
this.publishActivity(snapshot.state === "idle" ? null : this.backend.getPaneActivity(pane));
|
|
1770
|
+
}
|
|
1437
1771
|
if (snapshot.state === "idle") {
|
|
1438
1772
|
this.clearInstanceStateStuckTimer();
|
|
1439
1773
|
}
|
|
@@ -1488,7 +1822,8 @@ export class Daemon extends EventEmitter {
|
|
|
1488
1822
|
? rawConfig.idle_debounce_ms
|
|
1489
1823
|
: DEFAULT_STATE_IDLE_DEBOUNCE_MS;
|
|
1490
1824
|
this.instanceStateReadyPattern = this.backend.getReadyPattern();
|
|
1491
|
-
this.
|
|
1825
|
+
this.instanceStateBusyPattern = this.backend.getBusyPattern?.() ?? null;
|
|
1826
|
+
this.instanceStateMachine = new PaneStateMachine(this.instanceStateReadyPattern, this.instanceStateStuckTimeoutMs, Date.now(), this.instanceStateBusyPattern);
|
|
1492
1827
|
this.instanceStateLastOutputAt = 0;
|
|
1493
1828
|
this.instanceStateMonitorActive = true;
|
|
1494
1829
|
if (this.controlClient) {
|
|
@@ -1529,6 +1864,12 @@ export class Daemon extends EventEmitter {
|
|
|
1529
1864
|
paneTail: sanitizePaneTail(pane),
|
|
1530
1865
|
readyPattern: readyPattern.toString(),
|
|
1531
1866
|
readyMatched: deterministicReadyPattern.test(pane),
|
|
1867
|
+
// A pane that is stuck *while* the ready marker matches means the busy
|
|
1868
|
+
// pattern is what held it back — worth seeing when tuning either regex.
|
|
1869
|
+
busyPattern: this.instanceStateBusyPattern?.toString(),
|
|
1870
|
+
busyMatched: this.instanceStateBusyPattern
|
|
1871
|
+
? new RegExp(this.instanceStateBusyPattern.source, this.instanceStateBusyPattern.flags.replace(/[gy]/g, "")).test(pane)
|
|
1872
|
+
: undefined,
|
|
1532
1873
|
unchangedForMs: snapshot.unchangedForMs,
|
|
1533
1874
|
pendingWork: this.pendingWork.hasPendingWork(),
|
|
1534
1875
|
};
|
|
@@ -1571,6 +1912,40 @@ export class Daemon extends EventEmitter {
|
|
|
1571
1912
|
return this.messageBus;
|
|
1572
1913
|
}
|
|
1573
1914
|
// ── Tool status tracking ──────────────────────────────────────
|
|
1915
|
+
/**
|
|
1916
|
+
* Tell the fleet manager what this instance is doing right now, for the live
|
|
1917
|
+
* progress line on the cancel button.
|
|
1918
|
+
*
|
|
1919
|
+
* Purely cosmetic — nothing decides anything from it, so it is fine that only
|
|
1920
|
+
* backends with a transcript feed report at all (claude-code today). For the
|
|
1921
|
+
* rest the progress line keeps showing just elapsed time.
|
|
1922
|
+
*
|
|
1923
|
+
* Repeats are dropped: the ticker only edits the channel message when the text
|
|
1924
|
+
* changes, and a stream of identical broadcasts would defeat that.
|
|
1925
|
+
*/
|
|
1926
|
+
publishActivity(activity) {
|
|
1927
|
+
const next = activity && activity.trim() ? activity.trim() : null;
|
|
1928
|
+
if (next === this.currentActivity)
|
|
1929
|
+
return;
|
|
1930
|
+
this.currentActivity = next;
|
|
1931
|
+
this.ipcServer?.broadcast({
|
|
1932
|
+
type: "instance_activity",
|
|
1933
|
+
instanceName: this.name,
|
|
1934
|
+
activity: next,
|
|
1935
|
+
});
|
|
1936
|
+
}
|
|
1937
|
+
/**
|
|
1938
|
+
* Options for every system-initiated paste (startup notice, session snapshot,
|
|
1939
|
+
* runtime-dialog keys).
|
|
1940
|
+
*
|
|
1941
|
+
* One place, so the three call sites cannot drift apart on it. Backends with a
|
|
1942
|
+
* native input queue do not get the retry Enter: on those, a second bare Enter
|
|
1943
|
+
* is not the no-op the retry assumed but a queue mutation — the same reason
|
|
1944
|
+
* `deliverMessage` refuses to probe for busy with one.
|
|
1945
|
+
*/
|
|
1946
|
+
systemPasteOptions() {
|
|
1947
|
+
return { retryEnter: this.backend?.supportsQueuedInput?.() !== true };
|
|
1948
|
+
}
|
|
1574
1949
|
summarizeTool(name, input) {
|
|
1575
1950
|
const inp = input;
|
|
1576
1951
|
if (!inp)
|
|
@@ -1660,7 +2035,6 @@ export class Daemon extends EventEmitter {
|
|
|
1660
2035
|
writeFileSync(join(this.instanceDir, "prev-instructions"), this.pendingInstructionsUpdate);
|
|
1661
2036
|
this.pendingInstructionsUpdate = undefined;
|
|
1662
2037
|
}
|
|
1663
|
-
this.hangDetector?.recordInbound();
|
|
1664
2038
|
this.pendingWork.recordInbound();
|
|
1665
2039
|
// v3: record user messages for rotation snapshot
|
|
1666
2040
|
this.recordRecentUserMessage(content, meta);
|
|
@@ -1683,12 +2057,26 @@ export class Daemon extends EventEmitter {
|
|
|
1683
2057
|
// #77: show the sender's display name for readability, keeping the machine
|
|
1684
2058
|
// instance name in parens so the recipient's send_to_instance target is valid.
|
|
1685
2059
|
const fromLabel = meta.from_display ? `${meta.from_display} (${fromInstance})` : fromInstance;
|
|
1686
|
-
formatted = `[from:${fromLabel}] ${content}
|
|
2060
|
+
formatted = `[from:${fromLabel}] ${content}`;
|
|
2061
|
+
formatted += renderHandoffMetadata(meta);
|
|
2062
|
+
// A delegated task that requires a reply must not read like a chatty FYI —
|
|
2063
|
+
// the "you may stay silent" line is for the latter only.
|
|
2064
|
+
formatted += meta.requires_reply === "true"
|
|
2065
|
+
? "\n(A reply IS required: use report_result with the correlation_id above — or send_to_instance. Not direct text.)"
|
|
2066
|
+
: "\n(If you need to reply, use send_to_instance tool, NOT direct text. If there is nothing to add, you may stay silent.)";
|
|
1687
2067
|
}
|
|
1688
2068
|
else {
|
|
1689
2069
|
const via = meta.source ? ` via ${meta.source}` : "";
|
|
1690
2070
|
const idTag = meta.user_id ? `, id:${meta.user_id}` : "";
|
|
1691
|
-
formatted = `[user:${user}${via}${idTag}] ${content}
|
|
2071
|
+
formatted = `[user:${user}${via}${idTag}] ${content}`;
|
|
2072
|
+
// Reactions queued since the last real message (#432). One leading line of
|
|
2073
|
+
// context, present only when something is pending — a reaction no longer
|
|
2074
|
+
// costs a turn of its own.
|
|
2075
|
+
if (meta.pending_reactions) {
|
|
2076
|
+
formatted = `[Recent reactions: ${meta.pending_reactions}]\n${formatted}`;
|
|
2077
|
+
}
|
|
2078
|
+
formatted += renderHandoffMetadata(meta);
|
|
2079
|
+
formatted += "\n(Reply using the reply tool — do NOT respond with direct text)";
|
|
1692
2080
|
}
|
|
1693
2081
|
if (meta.reply_to_text) {
|
|
1694
2082
|
formatted += `\n(reply_to: "${meta.reply_to_text}")`;
|
|
@@ -1731,18 +2119,19 @@ export class Daemon extends EventEmitter {
|
|
|
1731
2119
|
}
|
|
1732
2120
|
/**
|
|
1733
2121
|
* Deliver a single message and drive its status reactions:
|
|
1734
|
-
* ⏳ message_queued — CLI busy; queued
|
|
2122
|
+
* ⏳ message_queued — CLI busy; queued locally or by the backend
|
|
1735
2123
|
* 👀 message_delivered — pasted + Enter sent; agent now has it
|
|
1736
|
-
* ✅ message_confirmed —
|
|
2124
|
+
* ✅ message_confirmed — processing observed, or native queue handoff succeeded
|
|
1737
2125
|
* ❌ message_failed — tmux window gone, paste retries exhausted
|
|
1738
2126
|
* Returns true once the message is in the CLI, false only on real delivery failure.
|
|
1739
2127
|
*
|
|
1740
2128
|
* Bug A (silent message loss): paste failures retry with backoff (window recovery)
|
|
1741
2129
|
* and emit `message_failed` if all attempts fail.
|
|
1742
|
-
* Busy handling (UX):
|
|
1743
|
-
*
|
|
1744
|
-
*
|
|
1745
|
-
*
|
|
2130
|
+
* Busy handling (UX): backends without a native input queue show ⏳ and wait for
|
|
2131
|
+
* idle indefinitely (a genuinely hung CLI is the hang detector's job). Backends
|
|
2132
|
+
* that explicitly support queued input receive one complete paste+Enter transaction
|
|
2133
|
+
* immediately and own the application-level queue themselves. The pasteLock remains
|
|
2134
|
+
* serial in both cases so separate PTY writes can never overlap.
|
|
1746
2135
|
*/
|
|
1747
2136
|
async deliverMessage(formatted, status) {
|
|
1748
2137
|
// Sanitize unclosed code fences — they cause CLI to wait for closure on Enter
|
|
@@ -1751,14 +2140,50 @@ export class Daemon extends EventEmitter {
|
|
|
1751
2140
|
// Odd number of fences = unclosed. Remove all code fences from the message.
|
|
1752
2141
|
formatted = formatted.replace(/```/g, "");
|
|
1753
2142
|
}
|
|
2143
|
+
// Before anything reads the window id: a spawn in progress is about to change it.
|
|
2144
|
+
await this.waitForSpawnToSettle();
|
|
1754
2145
|
let windowId = this.getWindowId();
|
|
1755
|
-
|
|
2146
|
+
const supportsQueuedInput = this.backend?.supportsQueuedInput?.() === true;
|
|
2147
|
+
let handingOffToNativeQueue = false;
|
|
2148
|
+
// If the CLI is busy, either hand the complete submission to its native input
|
|
2149
|
+
// queue or wait for idle. Native queue support is an explicit backend capability:
|
|
2150
|
+
// normal Enter input has steering/interrupt semantics in several other CLIs.
|
|
1756
2151
|
if (windowId && this.controlClient && !this.controlClient.isIdle(windowId)) {
|
|
1757
2152
|
if (status)
|
|
1758
2153
|
this.emit("message_queued", status);
|
|
1759
|
-
|
|
1760
|
-
|
|
2154
|
+
if (supportsQueuedInput) {
|
|
2155
|
+
handingOffToNativeQueue = true;
|
|
2156
|
+
this.logger.debug("CLI busy — handing message to backend-native input queue");
|
|
2157
|
+
}
|
|
2158
|
+
else {
|
|
2159
|
+
this.logger.debug("CLI busy — queuing message until idle");
|
|
2160
|
+
const becameIdle = await this.controlClient.waitUntilIdle(windowId);
|
|
2161
|
+
if (!becameIdle) {
|
|
2162
|
+
// The pane never freed up. Report the failure instead of pasting into a
|
|
2163
|
+
// wedged CLI (where the text would sit unsubmitted and the next message
|
|
2164
|
+
// would land on top of it) — and instead of holding the queue silently.
|
|
2165
|
+
this.logger.error("Pane still busy after the idle wait — reporting delivery failure");
|
|
2166
|
+
if (status)
|
|
2167
|
+
this.emit("message_failed", status); // ❌
|
|
2168
|
+
return false;
|
|
2169
|
+
}
|
|
2170
|
+
}
|
|
1761
2171
|
}
|
|
2172
|
+
// Everything above is *waiting*; everything below *writes*. Only the write is
|
|
2173
|
+
// held under the pane lock — holding it across the idle wait (up to 30 min)
|
|
2174
|
+
// would starve the runtime-dialog dismisser, which is often the very thing
|
|
2175
|
+
// that would let the pane go idle again.
|
|
2176
|
+
return this.paneWriteLock.run(() => this.writeMessageToPane(formatted, windowId, handingOffToNativeQueue, status));
|
|
2177
|
+
}
|
|
2178
|
+
/**
|
|
2179
|
+
* The write half of a delivery: paste → Enter → confirm, with retries.
|
|
2180
|
+
*
|
|
2181
|
+
* Always called under {@link paneWriteLock}. Split out from `deliverMessage`
|
|
2182
|
+
* precisely so the lock's scope is visible at the call site rather than being
|
|
2183
|
+
* an invariant maintained by comments.
|
|
2184
|
+
*/
|
|
2185
|
+
async writeMessageToPane(formatted, initialWindowId, handingOffToNativeQueue, status) {
|
|
2186
|
+
let windowId = initialWindowId;
|
|
1762
2187
|
// Bug A: paste with backoff. Transient failures are usually a stale window id
|
|
1763
2188
|
// after a crash/respawn — recover by name and retry (max 3 attempts, 2s apart).
|
|
1764
2189
|
const maxAttempts = 3;
|
|
@@ -1783,9 +2208,56 @@ export class Daemon extends EventEmitter {
|
|
|
1783
2208
|
await this.tmux.sendSpecialKey("Enter");
|
|
1784
2209
|
if (status)
|
|
1785
2210
|
this.emit("message_delivered", status); // 👀
|
|
1786
|
-
//
|
|
1787
|
-
//
|
|
1788
|
-
// the
|
|
2211
|
+
// Busy queue-capable CLIs (codex) may accept paste without an idle→busy
|
|
2212
|
+
// transition. Do NOT probe for busy — a second bare Enter can mutate the
|
|
2213
|
+
// queue. Instead verify the paste actually landed in the pane (TUI redraw
|
|
2214
|
+
// can silently swallow it). Idle submissions keep the swallowed-Enter path.
|
|
2215
|
+
if (handingOffToNativeQueue) {
|
|
2216
|
+
await new Promise(r => setTimeout(r, NATIVE_QUEUE_PASTE_VERIFY_MS));
|
|
2217
|
+
if (await this.nativeQueuePasteVisible(formatted)) {
|
|
2218
|
+
if (status)
|
|
2219
|
+
this.emit("message_confirmed", status); // ✅ native queue accepted
|
|
2220
|
+
return true;
|
|
2221
|
+
}
|
|
2222
|
+
// Silent loss: fall back once to the normal idle-gated path.
|
|
2223
|
+
this.logger.warn("Native-queue paste not visible in pane — retrying via idle-gated delivery");
|
|
2224
|
+
if (windowId && this.controlClient) {
|
|
2225
|
+
await this.controlClient.waitUntilIdle(windowId);
|
|
2226
|
+
}
|
|
2227
|
+
const repasted = await this.tmux.pasteBuffer(formatted);
|
|
2228
|
+
if (!repasted) {
|
|
2229
|
+
this.logger.error("Idle-gated redelivery paste failed after native-queue silent loss");
|
|
2230
|
+
if (status)
|
|
2231
|
+
this.emit("message_failed", status); // ❌
|
|
2232
|
+
return false;
|
|
2233
|
+
}
|
|
2234
|
+
await new Promise(r => setTimeout(r, NORMAL_ENTER_SETTLE_MS));
|
|
2235
|
+
const retryAt = Date.now();
|
|
2236
|
+
await this.tmux.sendSpecialKey("Enter");
|
|
2237
|
+
if (windowId && this.controlClient) {
|
|
2238
|
+
let becameBusy = await this.confirmBusyAfterEnter(windowId, retryAt);
|
|
2239
|
+
if (!becameBusy) {
|
|
2240
|
+
this.logger.warn("No idle→busy after idle-gated redelivery — re-sending Enter once");
|
|
2241
|
+
const retry2At = Date.now();
|
|
2242
|
+
await this.tmux.sendSpecialKey("Enter");
|
|
2243
|
+
becameBusy = await this.confirmBusyAfterEnter(windowId, retry2At);
|
|
2244
|
+
}
|
|
2245
|
+
if (becameBusy) {
|
|
2246
|
+
if (status)
|
|
2247
|
+
this.emit("message_confirmed", status); // ✅
|
|
2248
|
+
return true;
|
|
2249
|
+
}
|
|
2250
|
+
}
|
|
2251
|
+
else if (await this.nativeQueuePasteVisible(formatted)) {
|
|
2252
|
+
if (status)
|
|
2253
|
+
this.emit("message_confirmed", status); // ✅
|
|
2254
|
+
return true;
|
|
2255
|
+
}
|
|
2256
|
+
this.logger.error("Idle-gated redelivery also failed after native-queue silent loss");
|
|
2257
|
+
if (status)
|
|
2258
|
+
this.emit("message_failed", status); // ❌
|
|
2259
|
+
return false;
|
|
2260
|
+
}
|
|
1789
2261
|
if (windowId && this.controlClient) {
|
|
1790
2262
|
let becameBusy = await this.confirmBusyAfterEnter(windowId, enterAt);
|
|
1791
2263
|
if (!becameBusy) {
|
|
@@ -1794,8 +2266,20 @@ export class Daemon extends EventEmitter {
|
|
|
1794
2266
|
await this.tmux.sendSpecialKey("Enter");
|
|
1795
2267
|
becameBusy = await this.confirmBusyAfterEnter(windowId, retryAt);
|
|
1796
2268
|
}
|
|
1797
|
-
if (becameBusy
|
|
1798
|
-
|
|
2269
|
+
if (becameBusy) {
|
|
2270
|
+
if (status)
|
|
2271
|
+
this.emit("message_confirmed", status); // ✅
|
|
2272
|
+
}
|
|
2273
|
+
else {
|
|
2274
|
+
// Both Enters were swallowed: the text is sitting UNSUBMITTED in the
|
|
2275
|
+
// CLI's input box. This used to return true, so the reaction stayed at 👀
|
|
2276
|
+
// forever and the next delivery pasted on top — submitting two messages
|
|
2277
|
+
// as one. Say so instead.
|
|
2278
|
+
this.logger.error("Message pasted but never submitted (no idle→busy after two Enters)");
|
|
2279
|
+
if (status)
|
|
2280
|
+
this.emit("message_failed", status); // ❌
|
|
2281
|
+
return false;
|
|
2282
|
+
}
|
|
1799
2283
|
}
|
|
1800
2284
|
else {
|
|
1801
2285
|
// No control client to observe output: fall back to the legacy double-Enter.
|
|
@@ -1811,15 +2295,49 @@ export class Daemon extends EventEmitter {
|
|
|
1811
2295
|
this.emit("message_failed", status); // ❌
|
|
1812
2296
|
return false;
|
|
1813
2297
|
}
|
|
2298
|
+
/**
|
|
2299
|
+
* True when a busy native-queue paste appears to have landed: Codex shows a
|
|
2300
|
+
* `↳` queue marker, or a distinctive slice of the pasted text is on screen.
|
|
2301
|
+
* Used only to detect silent paste loss — not as a general ready check.
|
|
2302
|
+
*/
|
|
2303
|
+
async nativeQueuePasteVisible(formatted) {
|
|
2304
|
+
if (!this.tmux)
|
|
2305
|
+
return false;
|
|
2306
|
+
try {
|
|
2307
|
+
const pane = await this.tmux.capturePane();
|
|
2308
|
+
if (pane.includes("↳"))
|
|
2309
|
+
return true;
|
|
2310
|
+
for (const line of formatted.split(/\r?\n/)) {
|
|
2311
|
+
const t = line.trim();
|
|
2312
|
+
if (t.length >= 8 && pane.includes(t))
|
|
2313
|
+
return true;
|
|
2314
|
+
}
|
|
2315
|
+
const compact = formatted.replace(/\s+/g, " ").trim();
|
|
2316
|
+
if (compact.length >= 8 && pane.includes(compact.slice(0, Math.min(80, compact.length)))) {
|
|
2317
|
+
return true;
|
|
2318
|
+
}
|
|
2319
|
+
return false;
|
|
2320
|
+
}
|
|
2321
|
+
catch {
|
|
2322
|
+
return false;
|
|
2323
|
+
}
|
|
2324
|
+
}
|
|
1814
2325
|
/** Re-resolve this instance's tmux window by name (stale id after crash/respawn). */
|
|
1815
2326
|
async recoverWindow() {
|
|
2327
|
+
const previousWindowId = this.tmux?.getWindowId();
|
|
1816
2328
|
try {
|
|
1817
2329
|
const windows = await TmuxManager.listWindows(this.tmuxSessionName);
|
|
1818
2330
|
const match = windows.find(w => w.name === this.name);
|
|
1819
2331
|
if (!match)
|
|
1820
2332
|
return undefined;
|
|
1821
|
-
this.tmux = new TmuxManager(this.tmuxSessionName, match.id);
|
|
2333
|
+
this.tmux = new TmuxManager(this.tmuxSessionName, match.id, resolveTmuxLogicalSize(this.config.terminal));
|
|
1822
2334
|
writeFileSync(join(this.instanceDir, "window-id"), match.id);
|
|
2335
|
+
// The window we were talking to is gone; leaving it registered means the
|
|
2336
|
+
// control client re-resolves a dead id — one tmux subprocess — on every
|
|
2337
|
+
// reconnect, for the life of the fleet process.
|
|
2338
|
+
if (previousWindowId && previousWindowId !== match.id) {
|
|
2339
|
+
this.controlClient?.unregisterWindow(previousWindowId);
|
|
2340
|
+
}
|
|
1823
2341
|
await this.controlClient?.registerWindow(match.id);
|
|
1824
2342
|
this.bindInstanceStateOutputListener(match.id);
|
|
1825
2343
|
this.logger.info({ windowId: match.id }, "Recovered window ID for message delivery");
|
|
@@ -1830,12 +2348,38 @@ export class Daemon extends EventEmitter {
|
|
|
1830
2348
|
return undefined;
|
|
1831
2349
|
}
|
|
1832
2350
|
}
|
|
1833
|
-
/**
|
|
2351
|
+
/**
|
|
2352
|
+
* Poll up to ~2s (200ms × 10) for the pane to emit output after `since`.
|
|
2353
|
+
*
|
|
2354
|
+
* A `false` here is not "no output" — it is read as **"the message was pasted but
|
|
2355
|
+
* never submitted"**, which re-sends Enter and then reports ❌ to the user. So a
|
|
2356
|
+
* false negative costs a possible double submit and a failure notice for a
|
|
2357
|
+
* message that actually arrived.
|
|
2358
|
+
*
|
|
2359
|
+
* That is exactly what a control-mode reconnect used to produce. `connect()`
|
|
2360
|
+
* drops every output timestamp, so `hasOutputSince` answers `false` for a pane
|
|
2361
|
+
* that did react — the evidence was thrown away, not absent. Blind time is
|
|
2362
|
+
* therefore not counted against the budget: after a reset the poll restarts from
|
|
2363
|
+
* the moment observation resumed. A hard wall-clock cap keeps a reconnect loop
|
|
2364
|
+
* from extending this forever.
|
|
2365
|
+
*/
|
|
1834
2366
|
async confirmBusyAfterEnter(windowId, since) {
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
2367
|
+
const client = this.controlClient;
|
|
2368
|
+
const hardDeadline = Date.now() + CONFIRM_BUSY_MAX_WAIT_MS;
|
|
2369
|
+
let observedFrom = since;
|
|
2370
|
+
let polls = 0;
|
|
2371
|
+
while (polls < CONFIRM_BUSY_POLLS) {
|
|
2372
|
+
await new Promise(r => setTimeout(r, CONFIRM_BUSY_POLL_MS));
|
|
2373
|
+
if (client.hasOutputSince(windowId, observedFrom))
|
|
1838
2374
|
return true;
|
|
2375
|
+
const resetAt = client.getObservationResetAt();
|
|
2376
|
+
if (resetAt > observedFrom && Date.now() < hardDeadline) {
|
|
2377
|
+
this.logger.debug({ windowId, resetAt }, "Control mode reconnected mid-confirmation — restarting the Enter check from when we could see again");
|
|
2378
|
+
observedFrom = resetAt;
|
|
2379
|
+
polls = 0;
|
|
2380
|
+
continue;
|
|
2381
|
+
}
|
|
2382
|
+
polls++;
|
|
1839
2383
|
}
|
|
1840
2384
|
return false;
|
|
1841
2385
|
}
|
|
@@ -1906,8 +2450,8 @@ export class Daemon extends EventEmitter {
|
|
|
1906
2450
|
});
|
|
1907
2451
|
const timeout = setTimeout(() => {
|
|
1908
2452
|
this.pendingIpcRequests.delete(fleetReqId);
|
|
1909
|
-
respond(null,
|
|
1910
|
-
},
|
|
2453
|
+
respond(null, `Task operation timed out after ${daemonBudgetMs(tool) / 1000}s`);
|
|
2454
|
+
}, daemonBudgetMs(tool));
|
|
1911
2455
|
this.pendingIpcRequests.set(fleetReqId, (respMsg) => {
|
|
1912
2456
|
clearTimeout(timeout);
|
|
1913
2457
|
respond(respMsg.result, respMsg.error);
|
|
@@ -1929,8 +2473,8 @@ export class Daemon extends EventEmitter {
|
|
|
1929
2473
|
});
|
|
1930
2474
|
const timeout = setTimeout(() => {
|
|
1931
2475
|
this.pendingIpcRequests.delete(fleetReqId);
|
|
1932
|
-
respond(null,
|
|
1933
|
-
},
|
|
2476
|
+
respond(null, `Decision operation timed out after ${daemonBudgetMs(tool) / 1000}s`);
|
|
2477
|
+
}, daemonBudgetMs(tool));
|
|
1934
2478
|
this.pendingIpcRequests.set(fleetReqId, (respMsg) => {
|
|
1935
2479
|
clearTimeout(timeout);
|
|
1936
2480
|
respond(respMsg.result, respMsg.error);
|
|
@@ -1956,8 +2500,8 @@ export class Daemon extends EventEmitter {
|
|
|
1956
2500
|
// Wait for fleet_schedule_response via pending request map
|
|
1957
2501
|
const timeout = setTimeout(() => {
|
|
1958
2502
|
this.pendingIpcRequests.delete(fleetReqId);
|
|
1959
|
-
respond(null,
|
|
1960
|
-
},
|
|
2503
|
+
respond(null, `Schedule operation timed out after ${daemonBudgetMs(tool) / 1000}s`);
|
|
2504
|
+
}, daemonBudgetMs(tool));
|
|
1961
2505
|
this.pendingIpcRequests.set(fleetReqId, (respMsg) => {
|
|
1962
2506
|
clearTimeout(timeout);
|
|
1963
2507
|
respond(respMsg.result, respMsg.error);
|
|
@@ -1978,7 +2522,9 @@ export class Daemon extends EventEmitter {
|
|
|
1978
2522
|
fleetRequestId: fleetReqId,
|
|
1979
2523
|
senderSessionName,
|
|
1980
2524
|
});
|
|
1981
|
-
|
|
2525
|
+
// Shared table, so delete_instance and the deployment tools get the same
|
|
2526
|
+
// budget here as the MCP client expects (the ad-hoc list omitted them).
|
|
2527
|
+
const crossTimeoutMs = daemonBudgetMs(tool);
|
|
1982
2528
|
const timeout = setTimeout(() => {
|
|
1983
2529
|
this.pendingIpcRequests.delete(fleetReqId);
|
|
1984
2530
|
respond(null, `Cross-instance operation timed out after ${crossTimeoutMs / 1000}s`);
|
|
@@ -2021,8 +2567,8 @@ export class Daemon extends EventEmitter {
|
|
|
2021
2567
|
this.ipcServer?.broadcast({ type: "fleet_outbound", tool, args, fleetRequestId: fleetReqId });
|
|
2022
2568
|
const timeout = setTimeout(() => {
|
|
2023
2569
|
this.pendingIpcRequests.delete(outboundKey);
|
|
2024
|
-
respond(null,
|
|
2025
|
-
},
|
|
2570
|
+
respond(null, `Fleet outbound timed out after ${daemonBudgetMs(tool) / 1000}s`);
|
|
2571
|
+
}, daemonBudgetMs(tool));
|
|
2026
2572
|
this.pendingIpcRequests.set(outboundKey, (respMsg) => {
|
|
2027
2573
|
clearTimeout(timeout);
|
|
2028
2574
|
respond(respMsg.result, respMsg.error);
|
|
@@ -2086,7 +2632,14 @@ export class Daemon extends EventEmitter {
|
|
|
2086
2632
|
try {
|
|
2087
2633
|
const all = JSON.parse(process.env.AGEND_DECISIONS);
|
|
2088
2634
|
const workDir = this.config.working_directory;
|
|
2089
|
-
|
|
2635
|
+
// `scope: "fleet"` used to bypass the project check entirely, so a
|
|
2636
|
+
// single-project worker inherited every other project's playbook (8 of
|
|
2637
|
+
// 14 injected here were foreign) — resent on every API call. Now a
|
|
2638
|
+
// fleet decision must also be RELEVANT: global (no project_root), the
|
|
2639
|
+
// same project, or the same project family (worktrees/checkouts of one
|
|
2640
|
+
// repo). A general is exempt: it dispatches across all projects, so
|
|
2641
|
+
// cross-project routing rules are exactly what it needs.
|
|
2642
|
+
decisions = selectRelevantDecisions(all, workDir, this.config.general_topic === true);
|
|
2090
2643
|
// Stable ordering so identical decision sets always build byte-identical
|
|
2091
2644
|
// instructions — otherwise source ordering jitter flips the warmup hash.
|
|
2092
2645
|
decisions.sort((a, b) => a.title.localeCompare(b.title));
|
|
@@ -2100,9 +2653,18 @@ export class Daemon extends EventEmitter {
|
|
|
2100
2653
|
AGEND_SOCKET_PATH: sockPath,
|
|
2101
2654
|
AGEND_INSTANCE_NAME: this.name,
|
|
2102
2655
|
AGEND_WORKING_DIR: this.config.working_directory,
|
|
2656
|
+
AGEND_INSTANCE_KIND: this.runtimeIdentity?.kind ?? "fleet-topic",
|
|
2657
|
+
AGEND_BACKEND: this.runtimeIdentity?.backend ?? this.config.backend ?? this.backend?.binaryName ?? "unknown",
|
|
2658
|
+
AGEND_MODEL: this.runtimeIdentity?.model ?? this.config.model ?? "default",
|
|
2103
2659
|
};
|
|
2104
|
-
|
|
2105
|
-
|
|
2660
|
+
// A general is a dispatcher, not a worker: default it to the `general` tool
|
|
2661
|
+
// profile instead of `full`. Every tool's schema is resent on every API call,
|
|
2662
|
+
// so full (44 tools) costs a general ~22k chars per turn — the single largest
|
|
2663
|
+
// slice of its prompt. Explicit config still wins.
|
|
2664
|
+
const defaultToolSet = this.config.general_topic ? "general" : undefined;
|
|
2665
|
+
const toolSet = this.config.tool_set ?? defaultToolSet;
|
|
2666
|
+
if (toolSet)
|
|
2667
|
+
mcpEnv.AGEND_TOOL_SET = toolSet;
|
|
2106
2668
|
if (this.config.display_name)
|
|
2107
2669
|
mcpEnv.AGEND_DISPLAY_NAME = this.config.display_name;
|
|
2108
2670
|
if (this.config.description)
|
|
@@ -2135,6 +2697,7 @@ export class Daemon extends EventEmitter {
|
|
|
2135
2697
|
instructions = buildFleetInstructions({
|
|
2136
2698
|
instanceName: this.name,
|
|
2137
2699
|
workingDirectory: this.config.working_directory,
|
|
2700
|
+
runtimeIdentity: this.runtimeIdentity,
|
|
2138
2701
|
displayName: this.config.display_name,
|
|
2139
2702
|
description: this.config.description,
|
|
2140
2703
|
customPrompt: resolvedCustomPrompt,
|
|
@@ -2147,6 +2710,7 @@ export class Daemon extends EventEmitter {
|
|
|
2147
2710
|
instructions = buildFleetInstructions({
|
|
2148
2711
|
instanceName: this.name,
|
|
2149
2712
|
workingDirectory: this.config.working_directory,
|
|
2713
|
+
runtimeIdentity: this.runtimeIdentity,
|
|
2150
2714
|
displayName: this.config.display_name,
|
|
2151
2715
|
description: this.config.description,
|
|
2152
2716
|
customPrompt: resolvedCustomPrompt,
|
|
@@ -2193,7 +2757,9 @@ export class Daemon extends EventEmitter {
|
|
|
2193
2757
|
// Small delay to let the CLI fully render its ready prompt
|
|
2194
2758
|
await new Promise(r => setTimeout(r, 1_000));
|
|
2195
2759
|
try {
|
|
2196
|
-
|
|
2760
|
+
// Messages can arrive during a restart and be queued on pasteLock before the
|
|
2761
|
+
// snapshot lands; both write to the pane, so both go through the same lock.
|
|
2762
|
+
await this.paneWriteLock.run(() => this.tmux.pasteText(`[system:session-snapshot]\n${snapshot}\n\nThis is a background context restore — do NOT reply to or acknowledge this message. Simply resume normal operation when the next user or instance message arrives.`, this.systemPasteOptions()));
|
|
2197
2763
|
this.logger.info("Injected session snapshot as first message");
|
|
2198
2764
|
this.emit("snapshot_injected", this.name);
|
|
2199
2765
|
}
|
|
@@ -2202,9 +2768,69 @@ export class Daemon extends EventEmitter {
|
|
|
2202
2768
|
this.emit("snapshot_failed", this.name);
|
|
2203
2769
|
}
|
|
2204
2770
|
}
|
|
2771
|
+
/**
|
|
2772
|
+
* Depth, not a flag: the wake path marks a spawn and then calls trySpawn, which
|
|
2773
|
+
* marks another. With a plain boolean the inner one's completion would clear the
|
|
2774
|
+
* latch while the outer spawn was still dismissing dialogs, releasing delivery
|
|
2775
|
+
* into exactly the window this exists to close.
|
|
2776
|
+
*/
|
|
2777
|
+
beginSpawn() {
|
|
2778
|
+
this.spawnDepth++;
|
|
2779
|
+
this.spawning = true;
|
|
2780
|
+
if (!this.spawnSettled) {
|
|
2781
|
+
this.spawnSettled = new Promise(resolve => { this.resolveSpawnSettled = resolve; });
|
|
2782
|
+
}
|
|
2783
|
+
}
|
|
2784
|
+
endSpawn() {
|
|
2785
|
+
this.spawnDepth = Math.max(0, this.spawnDepth - 1);
|
|
2786
|
+
if (this.spawnDepth > 0)
|
|
2787
|
+
return; // an enclosing spawn is still running
|
|
2788
|
+
this.spawning = false;
|
|
2789
|
+
const resolve = this.resolveSpawnSettled;
|
|
2790
|
+
this.spawnSettled = null;
|
|
2791
|
+
this.resolveSpawnSettled = null;
|
|
2792
|
+
resolve?.();
|
|
2793
|
+
}
|
|
2794
|
+
/**
|
|
2795
|
+
* Hold until the CLI has finished starting up.
|
|
2796
|
+
*
|
|
2797
|
+
* Startup is not a quiet period: `dismissDialogsUntilReady` is clicking through
|
|
2798
|
+
* trust prompts and session pickers. A pane showing a modal dialog produces no
|
|
2799
|
+
* output, so `waitUntilIdle` reports it idle and a queued message gets pasted
|
|
2800
|
+
* *into the dialog* — where the text is discarded and the Enter picks a menu
|
|
2801
|
+
* item. The paste and the Enter both "succeed", so this loses the message
|
|
2802
|
+
* without even a ❌.
|
|
2803
|
+
*
|
|
2804
|
+
* The pane write lock (#414) does not cover this: it serialises each key
|
|
2805
|
+
* sequence, but it is released between one dialog and the next.
|
|
2806
|
+
*
|
|
2807
|
+
* Bounded, and called BEFORE the pane lock is taken — waiting on the spawn while
|
|
2808
|
+
* holding the lock the spawn itself needs would deadlock.
|
|
2809
|
+
*/
|
|
2810
|
+
async waitForSpawnToSettle() {
|
|
2811
|
+
const settled = this.spawnSettled;
|
|
2812
|
+
if (!settled)
|
|
2813
|
+
return;
|
|
2814
|
+
this.logger.debug("Holding delivery until the CLI has finished starting up");
|
|
2815
|
+
let timer;
|
|
2816
|
+
const cap = new Promise(resolve => {
|
|
2817
|
+
timer = setTimeout(resolve, SPAWN_SETTLE_MAX_WAIT_MS);
|
|
2818
|
+
timer.unref?.();
|
|
2819
|
+
});
|
|
2820
|
+
try {
|
|
2821
|
+
await Promise.race([settled, cap]);
|
|
2822
|
+
}
|
|
2823
|
+
finally {
|
|
2824
|
+
if (timer)
|
|
2825
|
+
clearTimeout(timer);
|
|
2826
|
+
}
|
|
2827
|
+
if (this.spawning) {
|
|
2828
|
+
this.logger.warn("CLI still starting after the delivery hold — delivering anyway");
|
|
2829
|
+
}
|
|
2830
|
+
}
|
|
2205
2831
|
/** Spawn a CLI window. Returns true if --resume was used successfully. */
|
|
2206
2832
|
async spawnClaudeWindow() {
|
|
2207
|
-
this.
|
|
2833
|
+
this.beginSpawn();
|
|
2208
2834
|
let resumedSuccessfully = false;
|
|
2209
2835
|
try {
|
|
2210
2836
|
this.toolStatusLines = [];
|
|
@@ -2241,7 +2867,7 @@ export class Daemon extends EventEmitter {
|
|
|
2241
2867
|
this.backgroundSessionRecoveryAttempted = false;
|
|
2242
2868
|
}
|
|
2243
2869
|
finally {
|
|
2244
|
-
this.
|
|
2870
|
+
this.endSpawn();
|
|
2245
2871
|
}
|
|
2246
2872
|
return resumedSuccessfully;
|
|
2247
2873
|
}
|
|
@@ -2312,11 +2938,13 @@ export class Daemon extends EventEmitter {
|
|
|
2312
2938
|
// the global web token) from impersonating instances.
|
|
2313
2939
|
const agentTokenPath = join(this.instanceDir, "agent.token");
|
|
2314
2940
|
const agentToken = randomBytes(32).toString("hex");
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2941
|
+
const tokenWrite = writeSecretFile(agentTokenPath, agentToken);
|
|
2942
|
+
if (!tokenWrite.ok) {
|
|
2943
|
+
// Do not fail the spawn over it — an instance that cannot start is worse
|
|
2944
|
+
// than one whose token is readable. But say so, loudly: the whole point of
|
|
2945
|
+
// this token is that other local processes cannot use it.
|
|
2946
|
+
this.logger.error({ path: agentTokenPath, mode: tokenWrite.mode?.toString(8), reason: tokenWrite.reason }, "Agent token file is not owner-only — other local users can impersonate this instance");
|
|
2318
2947
|
}
|
|
2319
|
-
catch { }
|
|
2320
2948
|
// AGEND_HOME points the child's agent-cli at the same data dir the daemon
|
|
2321
2949
|
// is using, so it can locate <instanceDir>/agent.token.
|
|
2322
2950
|
const agendHome = join(this.instanceDir, "..", "..");
|
|
@@ -2334,7 +2962,12 @@ export class Daemon extends EventEmitter {
|
|
|
2334
2962
|
windowId = this.tmux.getWindowId();
|
|
2335
2963
|
}
|
|
2336
2964
|
else {
|
|
2965
|
+
// A crash respawn makes a brand new window. The dead one stays registered
|
|
2966
|
+
// unless we say so — the respawn branch above already does.
|
|
2967
|
+
const retired = this.tmux.getWindowId();
|
|
2337
2968
|
windowId = await this.tmux.createWindow(cmd, resolvedCwd, this.name);
|
|
2969
|
+
if (retired && retired !== windowId)
|
|
2970
|
+
this.controlClient?.unregisterWindow(retired);
|
|
2338
2971
|
}
|
|
2339
2972
|
writeFileSync(join(this.instanceDir, "window-id"), windowId);
|
|
2340
2973
|
// Enable remain-on-exit to capture exit codes on crash
|
|
@@ -2342,7 +2975,9 @@ export class Daemon extends EventEmitter {
|
|
|
2342
2975
|
this.logger.warn({ err }, "Failed to set remain-on-exit — exit codes will not be captured");
|
|
2343
2976
|
});
|
|
2344
2977
|
if (reuseWindow && !this.config.lightweight) {
|
|
2345
|
-
|
|
2978
|
+
const outputLog = join(this.instanceDir, "output.log");
|
|
2979
|
+
rotateLogIfNeeded(outputLog);
|
|
2980
|
+
await this.tmux.pipeOutput(outputLog).catch(err => {
|
|
2346
2981
|
this.logger.warn({ err }, "Failed to restore pipe-pane after wake");
|
|
2347
2982
|
});
|
|
2348
2983
|
}
|
|
@@ -2398,15 +3033,20 @@ export class Daemon extends EventEmitter {
|
|
|
2398
3033
|
for (const dialog of startupDialogs) {
|
|
2399
3034
|
if (dialog.pattern.test(pane)) {
|
|
2400
3035
|
this.logger.debug(`Dismissing startup dialog: ${dialog.description}`);
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
3036
|
+
// Restart is exactly when inbound messages pile up, and nothing gates
|
|
3037
|
+
// delivery on `spawning`. Take the pane lock for the key sequence so a
|
|
3038
|
+
// queued message cannot be pasted into a half-dismissed trust dialog.
|
|
3039
|
+
await this.paneWriteLock.run(async () => {
|
|
3040
|
+
for (const key of dialog.keys) {
|
|
3041
|
+
if (key === "Up" || key === "Down" || key === "Enter" || key === "Escape") {
|
|
3042
|
+
await this.tmux.sendSpecialKey(key);
|
|
3043
|
+
}
|
|
3044
|
+
else {
|
|
3045
|
+
await this.tmux.sendKeys(key);
|
|
3046
|
+
}
|
|
3047
|
+
await new Promise(r => setTimeout(r, 200));
|
|
2404
3048
|
}
|
|
2405
|
-
|
|
2406
|
-
await this.tmux.sendKeys(key);
|
|
2407
|
-
}
|
|
2408
|
-
await new Promise(r => setTimeout(r, 200));
|
|
2409
|
-
}
|
|
3049
|
+
});
|
|
2410
3050
|
// Wait for next screen to render
|
|
2411
3051
|
if (this.controlClient) {
|
|
2412
3052
|
const wid = readFileSync(join(this.instanceDir, "window-id"), "utf-8").trim();
|