@songsid/agend 2.1.4-beta.8 → 2.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/README.zh-TW.md +1 -1
- package/dist/agent-cli-instructions.md +2 -2
- package/dist/agent-endpoint.js +1 -1
- package/dist/agent-endpoint.js.map +1 -1
- package/dist/backend/antigravity.d.ts +12 -8
- package/dist/backend/antigravity.js +189 -42
- package/dist/backend/antigravity.js.map +1 -1
- package/dist/backend/claude-code.d.ts +87 -23
- package/dist/backend/claude-code.js +361 -46
- package/dist/backend/claude-code.js.map +1 -1
- package/dist/backend/codex.d.ts +18 -0
- package/dist/backend/codex.js +139 -5
- package/dist/backend/codex.js.map +1 -1
- package/dist/backend/gemini-cli.js +2 -2
- package/dist/backend/gemini-cli.js.map +1 -1
- package/dist/backend/grok.js +2 -2
- package/dist/backend/grok.js.map +1 -1
- package/dist/backend/kiro.d.ts +52 -2
- package/dist/backend/kiro.js +267 -12
- package/dist/backend/kiro.js.map +1 -1
- package/dist/backend/opencode.js +17 -4
- package/dist/backend/opencode.js.map +1 -1
- package/dist/backend/types.d.ts +122 -3
- package/dist/backend/types.js +5 -1
- package/dist/backend/types.js.map +1 -1
- package/dist/backend-outage.d.ts +61 -0
- package/dist/backend-outage.js +71 -0
- package/dist/backend-outage.js.map +1 -0
- package/dist/channel/adapters/discord.d.ts +44 -2
- package/dist/channel/adapters/discord.js +529 -89
- package/dist/channel/adapters/discord.js.map +1 -1
- package/dist/channel/adapters/telegram.d.ts +30 -0
- package/dist/channel/adapters/telegram.js +221 -19
- package/dist/channel/adapters/telegram.js.map +1 -1
- package/dist/channel/agy-mcp-launcher.d.ts +2 -0
- package/dist/channel/agy-mcp-launcher.js +28 -0
- package/dist/channel/agy-mcp-launcher.js.map +1 -0
- package/dist/channel/ipc-bridge.d.ts +2 -1
- package/dist/channel/ipc-bridge.js +33 -4
- package/dist/channel/ipc-bridge.js.map +1 -1
- package/dist/channel/markdown-chunk.d.ts +42 -0
- package/dist/channel/markdown-chunk.js +205 -0
- package/dist/channel/markdown-chunk.js.map +1 -0
- package/dist/channel/mcp-server.js +34 -5
- package/dist/channel/mcp-server.js.map +1 -1
- package/dist/channel/mcp-tools.js +5 -3
- package/dist/channel/mcp-tools.js.map +1 -1
- package/dist/channel/message-queue.js +64 -8
- package/dist/channel/message-queue.js.map +1 -1
- package/dist/channel/types.d.ts +31 -1
- package/dist/classic-channel-manager.d.ts +65 -0
- package/dist/classic-channel-manager.js +172 -3
- package/dist/classic-channel-manager.js.map +1 -1
- package/dist/cli.js +124 -83
- package/dist/cli.js.map +1 -1
- package/dist/config-validator.js +16 -0
- package/dist/config-validator.js.map +1 -1
- package/dist/config.d.ts +4 -0
- package/dist/config.js +12 -1
- package/dist/config.js.map +1 -1
- package/dist/cross-instance-envelope.d.ts +6 -0
- package/dist/cross-instance-envelope.js +30 -0
- package/dist/cross-instance-envelope.js.map +1 -0
- package/dist/daemon.d.ts +538 -11
- package/dist/daemon.js +2272 -288
- package/dist/daemon.js.map +1 -1
- package/dist/doctor.d.ts +41 -0
- package/dist/doctor.js +267 -0
- package/dist/doctor.js.map +1 -0
- package/dist/fleet-context.d.ts +63 -0
- package/dist/fleet-context.js.map +1 -1
- package/dist/fleet-manager.d.ts +424 -11
- package/dist/fleet-manager.js +2753 -444
- package/dist/fleet-manager.js.map +1 -1
- package/dist/fleet-yaml-slim.d.ts +10 -0
- package/dist/fleet-yaml-slim.js +59 -0
- package/dist/fleet-yaml-slim.js.map +1 -0
- package/dist/general-knowledge/skills/cross-instance-messaging/SKILL.md +1 -0
- package/dist/general-knowledge/skills/model-discovery/SKILL.md +63 -16
- package/dist/general-knowledge/skills/session-management/SKILL.md +171 -8
- package/dist/instance-lifecycle.d.ts +114 -1
- package/dist/instance-lifecycle.js +404 -14
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/instructions.js +42 -4
- package/dist/instructions.js.map +1 -1
- package/dist/locale.js +1024 -310
- package/dist/locale.js.map +1 -1
- package/dist/login-flows.d.ts +91 -0
- package/dist/login-flows.js +170 -0
- package/dist/login-flows.js.map +1 -0
- package/dist/login-manager.d.ts +63 -0
- package/dist/login-manager.js +134 -0
- package/dist/login-manager.js.map +1 -0
- package/dist/network-family.d.ts +18 -0
- package/dist/network-family.js +20 -0
- package/dist/network-family.js.map +1 -0
- package/dist/outbound-handlers.d.ts +16 -1
- package/dist/outbound-handlers.js +174 -40
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/outbound-schemas.d.ts +10 -3
- package/dist/outbound-schemas.js +10 -0
- package/dist/outbound-schemas.js.map +1 -1
- package/dist/pane-input-residue.d.ts +52 -0
- package/dist/pane-input-residue.js +107 -0
- package/dist/pane-input-residue.js.map +1 -0
- package/dist/process-memory.d.ts +6 -0
- package/dist/process-memory.js +45 -0
- package/dist/process-memory.js.map +1 -1
- package/dist/reply-dedup.d.ts +7 -8
- package/dist/reply-dedup.js +0 -0
- package/dist/reply-dedup.js.map +1 -1
- package/dist/restart-progress.d.ts +2 -0
- package/dist/restart-progress.js +3 -0
- package/dist/restart-progress.js.map +1 -1
- package/dist/scheduler/db.d.ts +12 -0
- package/dist/scheduler/db.js +59 -0
- package/dist/scheduler/db.js.map +1 -1
- package/dist/service-installer.d.ts +11 -0
- package/dist/service-installer.js +84 -18
- package/dist/service-installer.js.map +1 -1
- package/dist/settings-api.js +1 -1
- package/dist/settings-api.js.map +1 -1
- package/dist/setup-wizard.js +2 -2
- package/dist/setup-wizard.js.map +1 -1
- package/dist/spawn-gate.d.ts +30 -0
- package/dist/spawn-gate.js +79 -0
- package/dist/spawn-gate.js.map +1 -0
- package/dist/steer-capability.d.ts +1 -0
- package/dist/steer-capability.js +6 -0
- package/dist/steer-capability.js.map +1 -0
- package/dist/storm-window.d.ts +83 -0
- package/dist/storm-window.js +251 -0
- package/dist/storm-window.js.map +1 -0
- package/dist/tips.d.ts +44 -0
- package/dist/tips.js +355 -0
- package/dist/tips.js.map +1 -0
- package/dist/tmux-manager.d.ts +16 -0
- package/dist/tmux-manager.js +110 -25
- package/dist/tmux-manager.js.map +1 -1
- package/dist/tool-progress.d.ts +40 -0
- package/dist/tool-progress.js +289 -0
- package/dist/tool-progress.js.map +1 -0
- package/dist/topic-commands.d.ts +45 -6
- package/dist/topic-commands.js +394 -66
- package/dist/topic-commands.js.map +1 -1
- package/dist/transcript-monitor.d.ts +15 -2
- package/dist/transcript-monitor.js +63 -17
- package/dist/transcript-monitor.js.map +1 -1
- package/dist/transcript-sources.d.ts +131 -0
- package/dist/transcript-sources.js +580 -0
- package/dist/transcript-sources.js.map +1 -0
- package/dist/types.d.ts +11 -0
- package/dist/ui/dashboard.html +55 -32
- package/dist/ui/settings.html +200 -60
- package/dist/ui/view.html +147 -30
- package/dist/usage/format-rich.d.ts +1 -1
- package/dist/usage/format-rich.js +28 -24
- package/dist/usage/format-rich.js.map +1 -1
- package/dist/usage/i18n-keys.d.ts +7 -0
- package/dist/usage/i18n-keys.js +34 -0
- package/dist/usage/i18n-keys.js.map +1 -0
- package/dist/usage/i18n.d.ts +5 -0
- package/dist/usage/i18n.js +27 -0
- package/dist/usage/i18n.js.map +1 -0
- package/dist/usage/providers.d.ts +21 -0
- package/dist/usage/providers.js +153 -75
- package/dist/usage/providers.js.map +1 -1
- package/dist/usage/usage-api.d.ts +11 -3
- package/dist/usage/usage-api.js +61 -24
- package/dist/usage/usage-api.js.map +1 -1
- package/dist/workflow-templates/default.md +2 -1
- package/package.json +2 -2
package/dist/daemon.js
CHANGED
|
@@ -8,6 +8,9 @@ import { mcpServerState } from "./mcp-liveness.js";
|
|
|
8
8
|
import { clearPausedMarker, writePausedMarker } from "./pause-marker.js";
|
|
9
9
|
import { TmuxManager, resolveTmuxLogicalSize } from "./tmux-manager.js";
|
|
10
10
|
import { TranscriptMonitor } from "./transcript-monitor.js";
|
|
11
|
+
import { createTranscriptSource } from "./transcript-sources.js";
|
|
12
|
+
import { LOGIN_FLOWS } from "./login-flows.js";
|
|
13
|
+
import { ProgressAccumulator, summarizeProgress } from "./tool-progress.js";
|
|
11
14
|
import { ContextGuardian } from "./context-guardian.js";
|
|
12
15
|
import { IpcServer } from "./channel/ipc-bridge.js";
|
|
13
16
|
import { daemonBudgetMs } from "./channel/ipc-timeouts.js";
|
|
@@ -19,6 +22,8 @@ import { HangDetector } from "./hang-detector.js";
|
|
|
19
22
|
import { writeSecretFile } from "./secret-file.js";
|
|
20
23
|
import { PaneWriteLock } from "./pane-write-lock.js";
|
|
21
24
|
import { buildFleetInstructions } from "./instructions.js";
|
|
25
|
+
import { formatCrossInstanceInboundMessage, renderCrossInstanceHandoffMetadata, } from "./cross-instance-envelope.js";
|
|
26
|
+
import { bottomRowIsReady, pasteLeftInInput, strandedAgendMessageInInput } from "./pane-input-residue.js";
|
|
22
27
|
const __filename = fileURLToPath(import.meta.url);
|
|
23
28
|
const __dirname = dirname(__filename);
|
|
24
29
|
// Tool routing sets — module-level to avoid re-creation on every handleToolCall
|
|
@@ -45,6 +50,10 @@ export function buildInstructionReloadNotice(binaryName, instanceName, instanceD
|
|
|
45
50
|
export const DEFAULT_STUCK_TIMEOUT_MS = 10 * 60_000;
|
|
46
51
|
export const DEFAULT_STATE_IDLE_DEBOUNCE_MS = 2_000;
|
|
47
52
|
export const DEFAULT_STATE_SAFETY_SWEEP_MS = 60_000;
|
|
53
|
+
/** Coalesce one cosmetic-redraw burst before comparing the visible pane. */
|
|
54
|
+
export const PERIODIC_REDRAW_PROBE_MS = 25;
|
|
55
|
+
/** A foreground server/port-forward should hand control back or be acknowledged. */
|
|
56
|
+
export const DEFAULT_BLOCKING_PROCESS_GRACE_MS = 2 * 60_000;
|
|
48
57
|
const LAST_INBOUND_FILE = "last-inbound-at";
|
|
49
58
|
/** Minimum gap between "health check is failing" notifications for one instance. */
|
|
50
59
|
const HEALTH_ERROR_NOTIFY_INTERVAL_MS = 10 * 60_000;
|
|
@@ -131,19 +140,7 @@ export function writeLastInboundAt(instanceDir, timestamp) {
|
|
|
131
140
|
* message_id line.
|
|
132
141
|
*/
|
|
133
142
|
export function renderHandoffMetadata(meta) {
|
|
134
|
-
|
|
135
|
-
const add = (label, value) => {
|
|
136
|
-
if (value && value.trim())
|
|
137
|
-
rows.push(`${label}: ${value.trim()}`);
|
|
138
|
-
};
|
|
139
|
-
add("message_id", meta.message_id);
|
|
140
|
-
add("correlation_id", meta.correlation_id);
|
|
141
|
-
add("request_kind", meta.request_kind);
|
|
142
|
-
add("task_summary", meta.task_summary);
|
|
143
|
-
add("working_directory", meta.working_directory);
|
|
144
|
-
add("branch", meta.branch);
|
|
145
|
-
add("attachment_file_id", meta.attachment_file_id);
|
|
146
|
-
return rows.length ? `\n(${rows.join(" | ")})` : "";
|
|
143
|
+
return renderCrossInstanceHandoffMetadata(meta);
|
|
147
144
|
}
|
|
148
145
|
/** Headless inactivity timer used by the daemon and unit tests. */
|
|
149
146
|
export class AutoPauseController {
|
|
@@ -157,6 +154,11 @@ export class AutoPauseController {
|
|
|
157
154
|
recordActivity(now = Date.now()) {
|
|
158
155
|
this.lastActivityAt = now;
|
|
159
156
|
}
|
|
157
|
+
/** Apply a new idle threshold and start a fresh countdown from this update. */
|
|
158
|
+
reconfigure(thresholdMs, now = Date.now()) {
|
|
159
|
+
this.thresholdMs = Math.max(0, thresholdMs);
|
|
160
|
+
this.lastActivityAt = now;
|
|
161
|
+
}
|
|
160
162
|
observe(state, now = Date.now()) {
|
|
161
163
|
if (this.pausedAt !== null || this.thresholdMs <= 0)
|
|
162
164
|
return false;
|
|
@@ -279,9 +281,11 @@ export class PaneStateMachine {
|
|
|
279
281
|
* working" and a finished turn would not be seen as idle until the next 60s
|
|
280
282
|
* safety sweep. Output timestamps are the honest liveness signal here, and
|
|
281
283
|
* the caller has them.
|
|
284
|
+
* @param opts.forceBusy the backend has positively identified a still-running
|
|
285
|
+
* foreground tool even though its TUI keeps an old ready footer visible.
|
|
282
286
|
*/
|
|
283
287
|
observe(pane, now = Date.now(), opts = {}) {
|
|
284
|
-
const { settled = false, changeAt = now } = opts;
|
|
288
|
+
const { settled = false, changeAt = now, forceBusy = false } = opts;
|
|
285
289
|
const paneHash = createHash("sha256").update(pane).digest("hex");
|
|
286
290
|
const firstObservation = this.lastPaneHash === null;
|
|
287
291
|
const paneChanged = this.lastPaneHash !== paneHash;
|
|
@@ -290,7 +294,10 @@ export class PaneStateMachine {
|
|
|
290
294
|
this.lastPaneChangeAt = changeAt;
|
|
291
295
|
}
|
|
292
296
|
this.lastObservedAt = now;
|
|
293
|
-
|
|
297
|
+
// Some TUIs keep their ready footer visible while a foreground tool owns
|
|
298
|
+
// stdin. Backends can positively identify that tool from the pane; in that
|
|
299
|
+
// case the old prompt must not clear pending work or retire Cancel.
|
|
300
|
+
const ready = !forceBusy && this.isReady(pane);
|
|
294
301
|
const nextState = firstObservation
|
|
295
302
|
? ready ? "idle" : "working"
|
|
296
303
|
: paneChanged && !settled
|
|
@@ -352,6 +359,33 @@ export class PendingWorkTracker {
|
|
|
352
359
|
}
|
|
353
360
|
}
|
|
354
361
|
const NORMAL_ENTER_SETTLE_MS = 500;
|
|
362
|
+
/** Bottom-ready re-poll cadence for Enter-dropping TUIs once the pane is quiet. */
|
|
363
|
+
const BOTTOM_READY_POLL_MS = 250;
|
|
364
|
+
/** Consecutive unreadable pane probes tolerated by the delivery gate before the delivery is failed (≈10s at the poll cadence). */
|
|
365
|
+
const DIALOG_PROBE_UNKNOWN_MAX = 40;
|
|
366
|
+
/** Startup dialog scan: poll cadence and wall-clock budget (keys and waits included). */
|
|
367
|
+
const STARTUP_DIALOG_POLL_MS = 500;
|
|
368
|
+
const STARTUP_DIALOG_BUDGET_MS = 30_000;
|
|
369
|
+
/** A blocking dialog still on screen this long after first being seen is reported for a human. */
|
|
370
|
+
const DIALOG_PARKED_NOTIFY_MS = 60_000;
|
|
371
|
+
/** How many "dialog painted just before the write → wait → retry" rounds a delivery tolerates. */
|
|
372
|
+
const LATE_DIALOG_WRITE_ROUNDS = 3;
|
|
373
|
+
/**
|
|
374
|
+
* Startup failed because the CLI's backend is unreachable (see backend-outage.ts).
|
|
375
|
+
* The session-id is deliberately KEPT; the fleet schedules a delayed retry.
|
|
376
|
+
*/
|
|
377
|
+
export class BackendUnreachableStartupError extends Error {
|
|
378
|
+
backend;
|
|
379
|
+
constructor(backend) {
|
|
380
|
+
super(`CLI failed to start: the ${backend} backend is unreachable — session kept for a later retry`);
|
|
381
|
+
this.backend = backend;
|
|
382
|
+
this.name = "BackendUnreachableStartupError";
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
/** Bounded wait (under the pane lock) for the prompt to return before retrying a dropped Enter. */
|
|
386
|
+
const STRANDED_RETRY_READY_WAIT_MS = 30_000;
|
|
387
|
+
/** Max "stranded text → submit → wait for prompt → re-check" rounds per delivery; each may send one recovery Enter. */
|
|
388
|
+
const STRANDED_INPUT_MAX_ROUNDS = 3;
|
|
355
389
|
const FIRST_ENTER_SETTLE_MS = 1_750;
|
|
356
390
|
const FIRST_DELIVERY_WINDOW_MS = 5_000;
|
|
357
391
|
/** After busy native-queue paste+Enter, wait before checking the pane for silent loss. */
|
|
@@ -529,6 +563,66 @@ export function extractProxyReplyText(pane, opts = {}) {
|
|
|
529
563
|
// The end of the turn is the answer — when over budget, keep the tail.
|
|
530
564
|
return text.length > maxChars ? text.slice(text.length - maxChars) : text;
|
|
531
565
|
}
|
|
566
|
+
/**
|
|
567
|
+
* Find the narrow Claude XML failure signature reported in #648.
|
|
568
|
+
*
|
|
569
|
+
* This intentionally runs only at the daemon's busy→idle edge (the caller
|
|
570
|
+
* enforces that) and only scans output after the current inbound marker. A
|
|
571
|
+
* partial tool call while Claude is still streaming is therefore never acted
|
|
572
|
+
* on. The extracted argument goes through the same credential redaction as the
|
|
573
|
+
* dead-MCP pane proxy before it can leave the host.
|
|
574
|
+
*/
|
|
575
|
+
export function detectMalformedClaudeToolCall(pane, opts = {}) {
|
|
576
|
+
let clean = pane
|
|
577
|
+
.replace(/\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])/g, "")
|
|
578
|
+
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "");
|
|
579
|
+
const marker = opts.inboundMarker?.trim();
|
|
580
|
+
if (marker) {
|
|
581
|
+
const markerAt = clean.lastIndexOf(marker);
|
|
582
|
+
if (markerAt >= 0)
|
|
583
|
+
clean = clean.slice(markerAt + marker.length);
|
|
584
|
+
}
|
|
585
|
+
const lines = clean.split(/\r?\n/).slice(-(opts.maxLines ?? 120));
|
|
586
|
+
const tail = lines.join("\n");
|
|
587
|
+
const closingTags = [...tail.matchAll(/<\/(?:parameter|invoke|function_calls)>/gi)];
|
|
588
|
+
if (closingTags.length === 0)
|
|
589
|
+
return null;
|
|
590
|
+
const parameterOpen = /<parameter\b[^>]*\bname\s*=\s*["']text["'][^>]*>/gi;
|
|
591
|
+
const parameterOpens = [...tail.matchAll(parameterOpen)];
|
|
592
|
+
const open = parameterOpens.at(-1);
|
|
593
|
+
if (!open || open.index === undefined) {
|
|
594
|
+
// Base stale suppression on the XML artifact itself, not unrelated pane
|
|
595
|
+
// redraws appended after it. The occurrence counts still distinguish a
|
|
596
|
+
// genuinely new identical malformed call while both remain visible.
|
|
597
|
+
const signature = createHash("sha256")
|
|
598
|
+
.update(`0:${closingTags.length}:${closingTags.map(tag => tag[0].toLowerCase()).join(":")}`)
|
|
599
|
+
.digest("hex");
|
|
600
|
+
return { signature, text: null };
|
|
601
|
+
}
|
|
602
|
+
const contentAt = open.index + open[0].length;
|
|
603
|
+
const closeAt = tail.slice(contentAt).search(/<\/parameter>/i);
|
|
604
|
+
if (closeAt < 0) {
|
|
605
|
+
const signature = createHash("sha256")
|
|
606
|
+
.update(`${parameterOpens.length}:${closingTags.length}:${open[0]}`)
|
|
607
|
+
.digest("hex");
|
|
608
|
+
return { signature, text: null };
|
|
609
|
+
}
|
|
610
|
+
const fragment = tail.slice(contentAt, contentAt + closeAt);
|
|
611
|
+
// Fingerprint only the malformed XML artifact; never put its contents in logs.
|
|
612
|
+
const signature = createHash("sha256")
|
|
613
|
+
.update(`${parameterOpens.length}:${closingTags.length}:${open[0]}${fragment}</parameter>`)
|
|
614
|
+
.digest("hex");
|
|
615
|
+
let text = sanitizePaneTail(fragment, opts.maxLines ?? 120)
|
|
616
|
+
.join("\n")
|
|
617
|
+
.replace(/\n{3,}/g, "\n\n")
|
|
618
|
+
.trim();
|
|
619
|
+
if (text.replace(/[^\p{L}\p{N}]/gu, "").length < 2)
|
|
620
|
+
return { signature, text: null };
|
|
621
|
+
const maxChars = opts.maxChars ?? 12_000;
|
|
622
|
+
if (text.length > maxChars)
|
|
623
|
+
text = text.slice(0, maxChars);
|
|
624
|
+
return { signature, text };
|
|
625
|
+
}
|
|
532
626
|
const INTERACTIVE_PROMPT_PATTERNS = [
|
|
533
627
|
{ kind: "sudo_password", pattern: /^\s*\[sudo\]\s+password\s+for\s+[^:\n]+:\s*$/im },
|
|
534
628
|
// macOS sudo and several package installers use this exact no-echo prompt.
|
|
@@ -591,6 +685,69 @@ export class InteractivePromptDetector {
|
|
|
591
685
|
this.notifiedSignature = null;
|
|
592
686
|
}
|
|
593
687
|
}
|
|
688
|
+
const BLOCKING_PROCESS_PATTERNS = [
|
|
689
|
+
// kubectl port-forward (including a backgrounded child whose inherited
|
|
690
|
+
// stdout keeps the parent shell tool open).
|
|
691
|
+
/^\s*(?:Forwarding from\s+(?:127\.0\.0\.1|localhost|\[::1\]):\d+\s+->\s+\d+|Handling connection for\s+\d+)\s*$/im,
|
|
692
|
+
// Common development servers. Keep these line-shaped and require an address
|
|
693
|
+
// or port so prose such as "the server is running" does not arm the detector.
|
|
694
|
+
/^\s*(?:(?:INFO:\s*)?Uvicorn running on|Serving HTTP on|Listening on|Server (?:is )?(?:listening|running) (?:at|on)|Application startup complete[^\n]*(?:port|https?:\/\/))[^\n]*(?:https?:\/\/|\b(?:localhost|127\.0\.0\.1|0\.0\.0\.0|\[::\]|port)\b)[^\n]*$/im,
|
|
695
|
+
];
|
|
696
|
+
/**
|
|
697
|
+
* Detect a foreground shell tool which has become a long-lived server.
|
|
698
|
+
*
|
|
699
|
+
* Unlike the generic pane-stuck clock, this clock deliberately survives new
|
|
700
|
+
* output: access logs and `Handling connection` lines are exactly why a
|
|
701
|
+
* foreground server can remain blocked forever without ever looking silent.
|
|
702
|
+
* Once a server marker is seen, the still-running tool identity is the stable
|
|
703
|
+
* signal; the marker may scroll out of the 32-row pane.
|
|
704
|
+
*/
|
|
705
|
+
export class BlockingProcessDetector {
|
|
706
|
+
graceMs;
|
|
707
|
+
activity = null;
|
|
708
|
+
evidence = null;
|
|
709
|
+
detectedAt = 0;
|
|
710
|
+
notified = false;
|
|
711
|
+
constructor(graceMs = DEFAULT_BLOCKING_PROCESS_GRACE_MS) {
|
|
712
|
+
this.graceMs = graceMs;
|
|
713
|
+
}
|
|
714
|
+
observe(pane, activity, now = Date.now()) {
|
|
715
|
+
const shellActivity = activity && /^(?:shell|bash|exec_command|terminal|command)(?::|$)/i.test(activity);
|
|
716
|
+
if (!shellActivity) {
|
|
717
|
+
this.reset();
|
|
718
|
+
return null;
|
|
719
|
+
}
|
|
720
|
+
if (activity !== this.activity) {
|
|
721
|
+
this.reset();
|
|
722
|
+
this.activity = activity;
|
|
723
|
+
}
|
|
724
|
+
if (!this.evidence) {
|
|
725
|
+
const tail = sanitizePaneTail(pane, 40).join("\n");
|
|
726
|
+
for (const pattern of BLOCKING_PROCESS_PATTERNS) {
|
|
727
|
+
const match = tail.match(pattern);
|
|
728
|
+
if (!match)
|
|
729
|
+
continue;
|
|
730
|
+
this.evidence = match[0].trim().slice(0, 200);
|
|
731
|
+
this.detectedAt = now;
|
|
732
|
+
break;
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
if (!this.evidence || this.notified || now - this.detectedAt < this.graceMs)
|
|
736
|
+
return null;
|
|
737
|
+
this.notified = true;
|
|
738
|
+
return {
|
|
739
|
+
activity,
|
|
740
|
+
evidence: this.evidence,
|
|
741
|
+
blockedForMs: now - this.detectedAt,
|
|
742
|
+
};
|
|
743
|
+
}
|
|
744
|
+
reset() {
|
|
745
|
+
this.activity = null;
|
|
746
|
+
this.evidence = null;
|
|
747
|
+
this.detectedAt = 0;
|
|
748
|
+
this.notified = false;
|
|
749
|
+
}
|
|
750
|
+
}
|
|
594
751
|
export class Daemon extends EventEmitter {
|
|
595
752
|
name;
|
|
596
753
|
config;
|
|
@@ -599,6 +756,9 @@ export class Daemon extends EventEmitter {
|
|
|
599
756
|
backend;
|
|
600
757
|
controlClient;
|
|
601
758
|
runtimeIdentity;
|
|
759
|
+
spawnGate;
|
|
760
|
+
stormWindow;
|
|
761
|
+
backendOutage;
|
|
602
762
|
logger;
|
|
603
763
|
tmuxSessionName;
|
|
604
764
|
tmux = null;
|
|
@@ -608,6 +768,13 @@ export class Daemon extends EventEmitter {
|
|
|
608
768
|
guardian = null;
|
|
609
769
|
adapter = null;
|
|
610
770
|
pendingIpcRequests = new Map();
|
|
771
|
+
/**
|
|
772
|
+
* Daemon-global request sequence. MCP request ids are only process-local:
|
|
773
|
+
* Claude can have more than one MCP child connected to the same daemon and
|
|
774
|
+
* each starts at 1, so using requestId alone lets one caller steal another's
|
|
775
|
+
* response.
|
|
776
|
+
*/
|
|
777
|
+
fleetRequestSeq = 0;
|
|
611
778
|
// Track chatId/threadId from inbound messages for automatic outbound routing
|
|
612
779
|
lastChatId;
|
|
613
780
|
lastThreadId;
|
|
@@ -618,10 +785,12 @@ export class Daemon extends EventEmitter {
|
|
|
618
785
|
currentActivity = null;
|
|
619
786
|
// Session identity: map IPC socket → sessionName (from mcp_ready)
|
|
620
787
|
socketSessionNames = new Map();
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
788
|
+
/** pid each connected MCP server announced in mcp_ready (or pong); keyed like socketSessionNames. */
|
|
789
|
+
socketPids = new Map();
|
|
790
|
+
mcpPingSeq = 0;
|
|
791
|
+
mcpPingWaiters = new Map();
|
|
792
|
+
// Crash recovery is fleet-owned; all daemon objects share the coordinators
|
|
793
|
+
// injected by FleetManager below.
|
|
625
794
|
healthCheckTimer = null;
|
|
626
795
|
crashCount = 0;
|
|
627
796
|
lastCrashAt = 0;
|
|
@@ -642,6 +811,12 @@ export class Daemon extends EventEmitter {
|
|
|
642
811
|
resolveSpawnSettled = null;
|
|
643
812
|
spawnDepth = 0;
|
|
644
813
|
skipResume = false;
|
|
814
|
+
startupAborted = false;
|
|
815
|
+
/** First time the current on-screen blocking dialog was seen (0 = none); drives the parked report. */
|
|
816
|
+
dialogParkedSince = 0;
|
|
817
|
+
/** Identity of that dialog: its pattern, not its description (two tables may describe one screen differently). */
|
|
818
|
+
dialogParkedKey = null;
|
|
819
|
+
dialogParkedReported = false;
|
|
645
820
|
backgroundSessionRecoveryAttempted = false;
|
|
646
821
|
/** Whether the last spawn started a fresh session (not resumed). */
|
|
647
822
|
isNewSession = false;
|
|
@@ -655,6 +830,7 @@ export class Daemon extends EventEmitter {
|
|
|
655
830
|
/** Fallback safety sweep used only when no shared tmux control client exists. */
|
|
656
831
|
instanceStateMonitorTimer = null;
|
|
657
832
|
instanceStateIdleTimer = null;
|
|
833
|
+
instanceStateOutputProbeTimer = null;
|
|
658
834
|
instanceStateStuckTimer = null;
|
|
659
835
|
instanceStateOutputListener = null;
|
|
660
836
|
instanceStateOutputEventName = null;
|
|
@@ -673,8 +849,12 @@ export class Daemon extends EventEmitter {
|
|
|
673
849
|
// between, the agent's answer exists only on screen — the daemon relays it.
|
|
674
850
|
turnHadInbound = false;
|
|
675
851
|
turnOutboundDelivered = false;
|
|
852
|
+
/** Successful `reply` specifically; cross-instance tools do not satisfy #648. */
|
|
853
|
+
turnReplyDelivered = false;
|
|
676
854
|
turnCorrelationId;
|
|
677
855
|
turnInboundMarker;
|
|
856
|
+
/** Prevent a visible stale XML fragment from being recovered on later turns. */
|
|
857
|
+
lastMalformedToolCallSignature;
|
|
678
858
|
proxyReplySeq = 0;
|
|
679
859
|
autoPauseController;
|
|
680
860
|
pauseRequested = false;
|
|
@@ -685,6 +865,35 @@ export class Daemon extends EventEmitter {
|
|
|
685
865
|
* would silently no-op exactly when we most need to stop feeding the CLI.
|
|
686
866
|
*/
|
|
687
867
|
pausePending = false;
|
|
868
|
+
/**
|
|
869
|
+
* An auth failure was detected and nothing has resolved it yet.
|
|
870
|
+
*
|
|
871
|
+
* Separate from `pausePending` because it must outlive it: the deferred pause
|
|
872
|
+
* clears `pausePending` as soon as it fires, but if that pause is refused the
|
|
873
|
+
* instance is still auth-broken and must not resume raising alarms.
|
|
874
|
+
*/
|
|
875
|
+
authFailureUnresolved = false;
|
|
876
|
+
/** One login-screen auth report per spawn — the screen persists across polls. */
|
|
877
|
+
loginScreenReported = false;
|
|
878
|
+
/** A fatal startup screen (see StartupDialog.fatal) was already reported this run. */
|
|
879
|
+
fatalStartupReported = false;
|
|
880
|
+
/**
|
|
881
|
+
* The CLI is parked on a fatal startup screen and every pane write must be
|
|
882
|
+
* refused. This is deliberately separate from the pause that the reported
|
|
883
|
+
* pty_error triggers: the pause is applied by an async handler, and a queued
|
|
884
|
+
* delivery would race it into the modal (where paste+Enter can confirm
|
|
885
|
+
* "Exit and fix manually"). Set synchronously at dialog-match time; cleared
|
|
886
|
+
* when a startup scan reaches a real ready prompt, or on wake().
|
|
887
|
+
*/
|
|
888
|
+
fatalStartupBlocked = false;
|
|
889
|
+
/** model_error seen mid-turn: notify only if it survives to the idle screen. */
|
|
890
|
+
pendingModelErrorKey = null;
|
|
891
|
+
/**
|
|
892
|
+
* Let the next pause() proceed from a stuck pane. Set only for the auth-deferred
|
|
893
|
+
* pause — see the pausePending consumption site for why waiting for idle there
|
|
894
|
+
* is waiting forever.
|
|
895
|
+
*/
|
|
896
|
+
pauseAllowStuck = false;
|
|
688
897
|
pauseWakeState = "active";
|
|
689
898
|
pauseWakeTransition = null;
|
|
690
899
|
// Model failover: override model on next spawn when rate-limited
|
|
@@ -698,6 +907,12 @@ export class Daemon extends EventEmitter {
|
|
|
698
907
|
* and ⏳/👀/✅ reactions live here). It does NOT cover the other writers that
|
|
699
908
|
* reach the pane — {@link paneWriteLock} does. */
|
|
700
909
|
pasteLock = Promise.resolve();
|
|
910
|
+
/**
|
|
911
|
+
* Fleet cancellation epoch. Queue entries retain the epoch they arrived with;
|
|
912
|
+
* advancing it invalidates only work received before the cancel, never a new
|
|
913
|
+
* user message which arrives immediately afterwards.
|
|
914
|
+
*/
|
|
915
|
+
deliveryEpoch = 0;
|
|
701
916
|
/** Mutual exclusion for *every* write into the pane, whichever subsystem it
|
|
702
917
|
* comes from. See PaneWriteLock for why interleaving is destructive. */
|
|
703
918
|
paneWriteLock = new PaneWriteLock();
|
|
@@ -711,10 +926,23 @@ export class Daemon extends EventEmitter {
|
|
|
711
926
|
lastBuiltInstructions = "";
|
|
712
927
|
pasteQueueDepth = 0;
|
|
713
928
|
firstDeliveryDelay = new FirstDeliveryDelay();
|
|
929
|
+
/** Orders /steer pastes against each other. Deliberately NOT pasteLock:
|
|
930
|
+
* a steer must not queue behind the normal deliveries it exists to
|
|
931
|
+
* overtake. Pane-level exclusion still comes from paneWriteLock inside
|
|
932
|
+
* deliverMessage, so a steer and a normal delivery can never interleave
|
|
933
|
+
* their PTY writes — the steer just doesn't wait for idle. */
|
|
934
|
+
steerLock = Promise.resolve();
|
|
935
|
+
/** Tool lines shown in the channel processing bubble for the current turn. */
|
|
936
|
+
turnProgress = new ProgressAccumulator();
|
|
937
|
+
lastProgressBroadcast = "";
|
|
938
|
+
progressBroadcastTimer = null;
|
|
939
|
+
lastProgressBroadcastAt = 0;
|
|
714
940
|
// PTY error pattern monitoring
|
|
715
941
|
errorMonitorTimer = null;
|
|
716
942
|
interactivePromptDetector = new InteractivePromptDetector();
|
|
943
|
+
blockingProcessDetector = new BlockingProcessDetector();
|
|
717
944
|
/** Same 5-min gate the error monitor uses, so a dead MCP server alerts once. */
|
|
945
|
+
static MCP_PING_TIMEOUT_MS = 2_000;
|
|
718
946
|
static MCP_DEATH_COOLDOWN_MS = 5 * 60_000;
|
|
719
947
|
lastMcpDeathNotifiedAt = 0;
|
|
720
948
|
mcpDeathNotifiedForPid = null;
|
|
@@ -725,6 +953,16 @@ export class Daemon extends EventEmitter {
|
|
|
725
953
|
* past this point interrupting the turn costs less than staying mute.
|
|
726
954
|
*/
|
|
727
955
|
static MCP_RESTART_MAX_WAIT_MS = 30 * 60_000;
|
|
956
|
+
/**
|
|
957
|
+
* How long the revival restart waits, at the point of firing, for a
|
|
958
|
+
* CLI-respawned MCP server to announce itself.
|
|
959
|
+
*
|
|
960
|
+
* Codex <0.146 tears down and recreates ALL MCP connections on an auth/config
|
|
961
|
+
* change — healthy ones included — so the daemon sees the old pid die while
|
|
962
|
+
* the CLI is fine and a replacement is seconds away. Module load measured
|
|
963
|
+
* ~0.36s, so 2s is roughly 5x headroom without making a genuine loss wait.
|
|
964
|
+
*/
|
|
965
|
+
static MCP_REPLACEMENT_GRACE_MS = 2_000;
|
|
728
966
|
/**
|
|
729
967
|
* Sticky "restart to revive the MCP server as soon as the pane idles" — same
|
|
730
968
|
* shape as pausePending, because the death is usually detected mid-turn when
|
|
@@ -733,6 +971,25 @@ export class Daemon extends EventEmitter {
|
|
|
733
971
|
*/
|
|
734
972
|
mcpRestartPending = false;
|
|
735
973
|
mcpRestartStaleTimer = null;
|
|
974
|
+
/** In-flight replacement grace; also the guard against three triggers stacking. */
|
|
975
|
+
mcpRestartGraceTimer = null;
|
|
976
|
+
/**
|
|
977
|
+
* Which trigger the in-flight grace will commit under.
|
|
978
|
+
*
|
|
979
|
+
* The single-timer guard would otherwise SWALLOW a stale_timeout that lands
|
|
980
|
+
* during an idle-gated grace — and because the stale timer is one-shot and
|
|
981
|
+
* already cleared itself, the 30-minute backstop would be gone for good on an
|
|
982
|
+
* instance that never idles, which is the exact case it exists for.
|
|
983
|
+
*/
|
|
984
|
+
mcpRestartGraceTrigger = null;
|
|
985
|
+
/**
|
|
986
|
+
* A dead-MCP sighting held back once, pending a possible replacement.
|
|
987
|
+
*
|
|
988
|
+
* Deliberately NOT the same field as `mcpDeathNotifiedForPid`: deferring must
|
|
989
|
+
* not consume the once-per-pid budget, or a suspicion that turns out to be a
|
|
990
|
+
* real loss could never be reported at all.
|
|
991
|
+
*/
|
|
992
|
+
mcpDeathDeferredForPid = null;
|
|
736
993
|
/** Prevent in-flight monitor callbacks from re-arming after a pause. */
|
|
737
994
|
runtimeMonitorsFrozen = false;
|
|
738
995
|
errorWaitingForRecovery = false; // true = error detected, waiting for ready pattern
|
|
@@ -741,7 +998,7 @@ export class Daemon extends EventEmitter {
|
|
|
741
998
|
activeErrorPatternKey = null;
|
|
742
999
|
/** Whether this instance is in an abnormal error state (auto-pause is normal). */
|
|
743
1000
|
get isErrorState() {
|
|
744
|
-
return this.errorWaitingForRecovery || (this.healthCheckPaused && !this.isPaused) ||
|
|
1001
|
+
return this.errorWaitingForRecovery || (this.healthCheckPaused && !this.isPaused) || this.stormWindow?.isActive() === true;
|
|
745
1002
|
}
|
|
746
1003
|
get isPaused() { return this.pauseWakeState !== "active"; }
|
|
747
1004
|
/** Current CLI process state for status surfaces and recovery commands. */
|
|
@@ -781,7 +1038,9 @@ export class Daemon extends EventEmitter {
|
|
|
781
1038
|
this.errorRecoveryDeadlineAt = 0;
|
|
782
1039
|
this.activeErrorPatternKey = null;
|
|
783
1040
|
}
|
|
784
|
-
constructor(name, config, instanceDir, topicMode = false, backend, controlClient, rootLogger, runtimeIdentity
|
|
1041
|
+
constructor(name, config, instanceDir, topicMode = false, backend, controlClient, rootLogger, runtimeIdentity, spawnGate, stormWindow,
|
|
1042
|
+
/** Fleet-level "is this CLI's backend down?" memory — see backend-outage.ts. */
|
|
1043
|
+
backendOutage) {
|
|
785
1044
|
super();
|
|
786
1045
|
this.name = name;
|
|
787
1046
|
this.config = config;
|
|
@@ -790,6 +1049,9 @@ export class Daemon extends EventEmitter {
|
|
|
790
1049
|
this.backend = backend;
|
|
791
1050
|
this.controlClient = controlClient;
|
|
792
1051
|
this.runtimeIdentity = runtimeIdentity;
|
|
1052
|
+
this.spawnGate = spawnGate;
|
|
1053
|
+
this.stormWindow = stormWindow;
|
|
1054
|
+
this.backendOutage = backendOutage;
|
|
793
1055
|
if (!rootLogger)
|
|
794
1056
|
throw new Error("Daemon requires a shared root logger");
|
|
795
1057
|
this.runtimeIdentity ??= {
|
|
@@ -878,20 +1140,10 @@ export class Daemon extends EventEmitter {
|
|
|
878
1140
|
this.handleToolCall(msg, socket);
|
|
879
1141
|
}
|
|
880
1142
|
else if (msg.type === "mcp_ready") {
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
this.socketSessionNames.delete(socket);
|
|
886
|
-
// Notify fleet manager so it can clean up sessionRegistry
|
|
887
|
-
if (sessionName !== this.name) {
|
|
888
|
-
this.ipcServer?.broadcast({ type: "session_disconnected", sessionName });
|
|
889
|
-
}
|
|
890
|
-
});
|
|
891
|
-
}
|
|
892
|
-
this.logger.debug({ sessionName }, "MCP channel server connected and ready");
|
|
893
|
-
// Notify FleetManager's IPC client that MCP is ready
|
|
894
|
-
this.ipcServer?.broadcast({ type: "mcp_ready", sessionName });
|
|
1143
|
+
this.handleMcpReady(msg, socket);
|
|
1144
|
+
}
|
|
1145
|
+
else if (msg.type === "pong") {
|
|
1146
|
+
this.handleMcpPong(msg, socket);
|
|
895
1147
|
}
|
|
896
1148
|
else if (msg.type === "query_sessions") {
|
|
897
1149
|
// Fleet manager asks for all registered session names (catches sessions
|
|
@@ -911,43 +1163,38 @@ export class Daemon extends EventEmitter {
|
|
|
911
1163
|
// Fleet manager routed a message to us (topic mode)
|
|
912
1164
|
const meta = msg.meta;
|
|
913
1165
|
const targetSession = msg.targetSession;
|
|
1166
|
+
const deliveryEpoch = this.captureDeliveryEpoch(msg.delivery_epoch);
|
|
914
1167
|
void this.wake().then(() => {
|
|
915
|
-
this.pushChannelMessage(msg.content, meta, targetSession);
|
|
1168
|
+
this.pushChannelMessage(msg.content, meta, targetSession, deliveryEpoch);
|
|
916
1169
|
}).catch(err => {
|
|
917
1170
|
this.logger.error({ err: err.message }, "Wake failed for inbound delivery");
|
|
918
1171
|
});
|
|
919
1172
|
}
|
|
920
1173
|
else if (msg.type === "raw_paste") {
|
|
921
1174
|
// Paste raw text directly to CLI without [user:] wrapping.
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
1175
|
+
this.queueRawPaste(msg.content, this.captureDeliveryEpoch(msg.delivery_epoch), msg.confirm_clear === true);
|
|
1176
|
+
}
|
|
1177
|
+
else if (msg.type === "config_update") {
|
|
1178
|
+
this.applyConfigUpdate(msg.config);
|
|
1179
|
+
}
|
|
1180
|
+
else if (msg.type === "steer") {
|
|
1181
|
+
const meta = (msg.meta ?? {});
|
|
1182
|
+
this.steerMessage(msg.content, meta, this.captureDeliveryEpoch(msg.delivery_epoch));
|
|
1183
|
+
}
|
|
1184
|
+
else if (msg.type === "btw") {
|
|
1185
|
+
const meta = (msg.meta ?? {});
|
|
1186
|
+
this.btwMessage(msg.content, meta, this.captureDeliveryEpoch(msg.delivery_epoch));
|
|
932
1187
|
}
|
|
933
1188
|
else if (msg.type === "fleet_schedule_trigger") {
|
|
934
1189
|
const payload = msg.payload;
|
|
935
1190
|
const meta = msg.meta;
|
|
936
|
-
|
|
1191
|
+
const deliveryEpoch = this.captureDeliveryEpoch(msg.delivery_epoch);
|
|
1192
|
+
void this.wake().then(() => this.pushChannelMessage(payload.message, meta, undefined, deliveryEpoch)).catch(err => {
|
|
937
1193
|
this.logger.error({ err: err.message }, "Wake failed for scheduled delivery");
|
|
938
1194
|
});
|
|
939
1195
|
}
|
|
940
1196
|
else if (msg.type === "query_instance_state") {
|
|
941
|
-
|
|
942
|
-
this.ipcServer?.send(socket, {
|
|
943
|
-
type: "instance_state_response",
|
|
944
|
-
requestId: msg.requestId,
|
|
945
|
-
instanceName: this.name,
|
|
946
|
-
...snapshot,
|
|
947
|
-
state: this.isPaused ? "paused" : snapshot.state,
|
|
948
|
-
processStatus: this.processStatus,
|
|
949
|
-
pausedAt: this.lastPausedAt,
|
|
950
|
-
});
|
|
1197
|
+
void this.respondToInstanceStateQuery(msg, socket);
|
|
951
1198
|
}
|
|
952
1199
|
});
|
|
953
1200
|
// 2. Tmux — ensure session, create window if not alive
|
|
@@ -1033,8 +1280,11 @@ export class Daemon extends EventEmitter {
|
|
|
1033
1280
|
const outputLog = join(this.instanceDir, "output.log");
|
|
1034
1281
|
rotateLogIfNeeded(outputLog);
|
|
1035
1282
|
await this.tmux.pipeOutput(outputLog).catch(() => { });
|
|
1036
|
-
// 4. Transcript monitor
|
|
1037
|
-
|
|
1283
|
+
// 4. Transcript monitor. claude-code is handled inside the monitor
|
|
1284
|
+
// (statusline transcript); codex/kiro/opencode read their CLI's own
|
|
1285
|
+
// conversation store via a pluggable source. Backends with no known
|
|
1286
|
+
// source stay inert exactly as before.
|
|
1287
|
+
this.transcriptMonitor = new TranscriptMonitor(this.instanceDir, this.logger, createTranscriptSource(this.config.backend ?? "claude-code", this.config.working_directory));
|
|
1038
1288
|
// 5. Wire transcript events
|
|
1039
1289
|
const ackIfPending = () => {
|
|
1040
1290
|
if (!this.pendingAckMessage || !this.adapter)
|
|
@@ -1050,6 +1300,7 @@ export class Daemon extends EventEmitter {
|
|
|
1050
1300
|
this.recordRecentEvent({ type: "tool_use", name, preview: this.summarizeTool(name, input) });
|
|
1051
1301
|
this.recordRecentToolActivity(this.summarizeTool(name, input));
|
|
1052
1302
|
this.publishActivity(this.summarizeTool(name, input));
|
|
1303
|
+
this.recordToolProgress(name, input);
|
|
1053
1304
|
});
|
|
1054
1305
|
this.transcriptMonitor.on("tool_result", (name, _output) => {
|
|
1055
1306
|
this.recordRecentEvent({ type: "tool_result", name });
|
|
@@ -1112,30 +1363,119 @@ export class Daemon extends EventEmitter {
|
|
|
1112
1363
|
* process would have no client reading it. Only the CLI can restore its own
|
|
1113
1364
|
* tools — hence notify, and let the operator decide about restarting.
|
|
1114
1365
|
*/
|
|
1366
|
+
/**
|
|
1367
|
+
* An MCP server has connected and announced itself.
|
|
1368
|
+
*
|
|
1369
|
+
* Beyond the existing session bookkeeping this is the earliest evidence that
|
|
1370
|
+
* tools are back — roughly 30s ahead of the next health tick, which is
|
|
1371
|
+
* exactly the window #663's false restarts fired in.
|
|
1372
|
+
*/
|
|
1373
|
+
handleMcpReady(msg, socket) {
|
|
1374
|
+
const sessionName = msg.sessionName;
|
|
1375
|
+
if (typeof msg.pid === "number")
|
|
1376
|
+
this.socketPids.set(socket, msg.pid);
|
|
1377
|
+
if (sessionName) {
|
|
1378
|
+
this.socketSessionNames.set(socket, sessionName);
|
|
1379
|
+
socket.on("close", () => {
|
|
1380
|
+
this.socketSessionNames.delete(socket);
|
|
1381
|
+
this.socketPids.delete(socket);
|
|
1382
|
+
// Notify fleet manager so it can clean up sessionRegistry
|
|
1383
|
+
if (sessionName !== this.name) {
|
|
1384
|
+
this.ipcServer?.broadcast({ type: "session_disconnected", sessionName });
|
|
1385
|
+
}
|
|
1386
|
+
});
|
|
1387
|
+
}
|
|
1388
|
+
this.logger.debug({ sessionName, pid: msg.pid }, "MCP channel server connected and ready");
|
|
1389
|
+
// Gated on ownership: this ipcServer also accepts EXTERNAL sessions (a
|
|
1390
|
+
// `claude` run in another directory registers as `external-<cwd>-<pid>`).
|
|
1391
|
+
// Without the gate an unrelated session could stand down a restart armed
|
|
1392
|
+
// for THIS instance's genuinely dead server. The liveness re-read below is
|
|
1393
|
+
// a second guard, but ownership is the one that is semantically right.
|
|
1394
|
+
if (sessionName === this.name) {
|
|
1395
|
+
// A server for THIS instance just connected: that is proof of life at
|
|
1396
|
+
// the IPC layer, whatever channel.mcp.pid says (a replacement or a
|
|
1397
|
+
// sub-agent's server may have left a dead pid in the slot).
|
|
1398
|
+
this.noteMcpProofOfLife("mcp_ready", typeof msg.pid === "number" ? msg.pid : undefined);
|
|
1399
|
+
}
|
|
1400
|
+
this.ipcServer?.broadcast({ type: "mcp_ready", sessionName });
|
|
1401
|
+
}
|
|
1115
1402
|
checkMcpServerAlive() {
|
|
1116
1403
|
if (this.isPaused)
|
|
1117
1404
|
return;
|
|
1118
1405
|
const status = mcpServerState(this.instanceDir);
|
|
1119
|
-
if (status.state === "unknown")
|
|
1120
|
-
return;
|
|
1121
1406
|
if (status.state === "alive") {
|
|
1407
|
+
if (this.mcpDeathNotifiedForPid != null)
|
|
1408
|
+
this.emitMcpRecovered("pid", status.pid);
|
|
1122
1409
|
this.mcpDeathNotifiedForPid = null; // the CLI respawned it — re-arm
|
|
1410
|
+
this.mcpDeathDeferredForPid = null;
|
|
1123
1411
|
this.clearMcpRestartRequest(); // tools are back — stand down a pending auto-restart
|
|
1124
1412
|
return;
|
|
1125
1413
|
}
|
|
1414
|
+
// The pid slot does not name a live process — dead, or missing. It is a
|
|
1415
|
+
// single last-writer slot shared by every mcp-server process of this
|
|
1416
|
+
// instance (a CLI-driven replacement, a sub-agent's server), so "that pid
|
|
1417
|
+
// is gone" is not "no server is serving", and "the last writer exited
|
|
1418
|
+
// cleanly and unlinked its slot" is not "nothing ever started". Ask the
|
|
1419
|
+
// IPC layer — the one thing every serving mcp-server must hold — before
|
|
1420
|
+
// deciding. Reproduced deterministically: two servers, SIGKILL the last
|
|
1421
|
+
// writer, the other keeps serving while the slot reads DEAD forever; or
|
|
1422
|
+
// the last writer exits cleanly and the slot stays MISSING, so the
|
|
1423
|
+
// survivor's later real death would never be seen.
|
|
1424
|
+
const sockets = this.liveMcpSockets();
|
|
1425
|
+
if (sockets.length > 0) {
|
|
1426
|
+
const livePid = this.socketPids.get(sockets[sockets.length - 1]);
|
|
1427
|
+
const stalePid = status.state === "dead" ? status.pid : undefined;
|
|
1428
|
+
this.logger.info({ slot: status.state, stalePid, livePid, connections: sockets.length }, "channel.mcp.pid does not name a live process, but a live MCP connection still serves this instance — not an incident");
|
|
1429
|
+
this.repairMcpPidSlot(livePid, status.state === "dead" ? "stale pid slot" : "missing pid slot");
|
|
1430
|
+
this.pingMcpSockets(sockets);
|
|
1431
|
+
this.noteMcpProofOfLife("connection", livePid);
|
|
1432
|
+
return;
|
|
1433
|
+
}
|
|
1434
|
+
if (status.state === "unknown")
|
|
1435
|
+
return; // never started, or nothing left to watch — as before
|
|
1126
1436
|
// Dead: report once per pid, and at most once per cooldown window.
|
|
1127
1437
|
if (this.mcpDeathNotifiedForPid === status.pid)
|
|
1128
1438
|
return;
|
|
1439
|
+
// An auth-broken CLI cannot keep an MCP server alive, and restarting it
|
|
1440
|
+
// only loops back to the sign-in screen — hold the auto-restart until a
|
|
1441
|
+
// re-login (which restarts the instance and replaces this daemon anyway).
|
|
1442
|
+
const authSuspected = this.authFailureUnresolved;
|
|
1443
|
+
const autoRestart = this.config.mcp_auto_restart !== false && !authSuspected;
|
|
1444
|
+
// Codex <0.146 kills and recreates its MCP connections wholesale, so the
|
|
1445
|
+
// FIRST sighting of a dead pid under a live Codex is more likely a swap
|
|
1446
|
+
// than a loss. Hold the alarm for one tick — only the alarm: the restart is
|
|
1447
|
+
// still armed, and the chokepoint's re-read cancels it if the replacement
|
|
1448
|
+
// arrives. Not generalised to other backends, where a dead MCP under a live
|
|
1449
|
+
// CLI is a genuine incident that silence would mask.
|
|
1450
|
+
//
|
|
1451
|
+
// Crucially this does NOT touch mcpDeathNotifiedForPid or the cooldown
|
|
1452
|
+
// clock: if the suspicion is wrong the incident must still be reportable.
|
|
1453
|
+
if (this.suspectsTransientMcpReplacement() && this.mcpDeathDeferredForPid !== status.pid) {
|
|
1454
|
+
this.mcpDeathDeferredForPid = status.pid;
|
|
1455
|
+
this.logger.warn({ pid: status.pid }, "MCP server pid is gone under a live Codex — holding the alarm one tick for a possible replacement");
|
|
1456
|
+
if (autoRestart)
|
|
1457
|
+
this.armMcpRestartWhenIdle();
|
|
1458
|
+
return;
|
|
1459
|
+
}
|
|
1129
1460
|
if (Date.now() - this.lastMcpDeathNotifiedAt < Daemon.MCP_DEATH_COOLDOWN_MS)
|
|
1130
1461
|
return;
|
|
1131
1462
|
this.mcpDeathNotifiedForPid = status.pid;
|
|
1463
|
+
this.mcpDeathDeferredForPid = null;
|
|
1132
1464
|
this.lastMcpDeathNotifiedAt = Date.now();
|
|
1133
|
-
|
|
1134
|
-
this.
|
|
1135
|
-
this.emit("mcp_died", { name: this.name, pid: status.pid, autoRestart });
|
|
1465
|
+
this.logger.error({ pid: status.pid, autoRestart, authSuspected }, "MCP server process is gone — instance has no agend tools");
|
|
1466
|
+
this.emit("mcp_died", { name: this.name, pid: status.pid, autoRestart, authSuspected });
|
|
1136
1467
|
if (autoRestart)
|
|
1137
1468
|
this.armMcpRestartWhenIdle();
|
|
1138
1469
|
}
|
|
1470
|
+
/**
|
|
1471
|
+
* Whether a dead MCP pid on this backend is more likely a CLI-driven
|
|
1472
|
+
* replacement than a real loss. Codex only, and deliberately narrow: it is
|
|
1473
|
+
* the backend whose released fix (0.146) confirms it tore down healthy MCP
|
|
1474
|
+
* connections alongside the one it meant to restart.
|
|
1475
|
+
*/
|
|
1476
|
+
suspectsTransientMcpReplacement() {
|
|
1477
|
+
return (this.config.backend ?? this.backend?.binaryName) === "codex";
|
|
1478
|
+
}
|
|
1139
1479
|
/**
|
|
1140
1480
|
* Arm an automatic instance restart to revive a dead MCP server. Fires on the
|
|
1141
1481
|
* next busy→idle edge — or immediately when the pane is already idle, which is
|
|
@@ -1158,21 +1498,210 @@ export class Daemon extends EventEmitter {
|
|
|
1158
1498
|
this.fireMcpRestartRequest("already_idle");
|
|
1159
1499
|
}
|
|
1160
1500
|
}
|
|
1501
|
+
/**
|
|
1502
|
+
* Single chokepoint for all three revival triggers (already_idle, idle_edge,
|
|
1503
|
+
* stale_timeout) — and therefore the one place worth re-checking reality.
|
|
1504
|
+
*
|
|
1505
|
+
* The pending flag is armed by a health tick that saw the OLD pid die, and
|
|
1506
|
+
* was previously fired without ever looking again. When a CLI legitimately
|
|
1507
|
+
* respawns its own MCP child, the replacement is already serving by the time
|
|
1508
|
+
* the turn ends — but the idle edge fires inside the ~30s before the next
|
|
1509
|
+
* health tick can stand the request down, restarting a healthy instance. One
|
|
1510
|
+
* external user was restarted 18 times in two days this way (#663).
|
|
1511
|
+
*
|
|
1512
|
+
* Re-reading here is safe for every backend: worst case it clears a pending
|
|
1513
|
+
* restart that should not have existed.
|
|
1514
|
+
*/
|
|
1161
1515
|
fireMcpRestartRequest(trigger) {
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
//
|
|
1165
|
-
//
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1516
|
+
if (this.mcpRestartCancelledByRecovery(trigger))
|
|
1517
|
+
return;
|
|
1518
|
+
// Authoritative re-read — the CLI may have replaced the server since the
|
|
1519
|
+
// health tick that armed this.
|
|
1520
|
+
if (this.mcpServerAlive().alive) {
|
|
1521
|
+
this.logger.info({ trigger }, "MCP revival restart stood down — a live MCP server is already serving");
|
|
1522
|
+
this.clearMcpRestartRequest();
|
|
1523
|
+
return;
|
|
1524
|
+
}
|
|
1525
|
+
// Not back yet: give a CLI-driven replacement a bounded moment rather than
|
|
1526
|
+
// restarting the instance out from under it. One timer covers all three
|
|
1527
|
+
// triggers; a second trigger during the wait is a no-op.
|
|
1528
|
+
if (this.mcpRestartGraceTimer) {
|
|
1529
|
+
// stale_timeout outranks the idle-gated triggers: it is the backstop for
|
|
1530
|
+
// a turn that never ends, so it must not be lost to an in-flight grace.
|
|
1531
|
+
if (trigger === "stale_timeout")
|
|
1532
|
+
this.mcpRestartGraceTrigger = "stale_timeout";
|
|
1533
|
+
return;
|
|
1534
|
+
}
|
|
1535
|
+
this.mcpRestartGraceTrigger = trigger;
|
|
1536
|
+
this.mcpRestartGraceTimer = setTimeout(() => {
|
|
1537
|
+
this.mcpRestartGraceTimer = null;
|
|
1538
|
+
const committed = this.mcpRestartGraceTrigger ?? trigger;
|
|
1539
|
+
this.mcpRestartGraceTrigger = null;
|
|
1540
|
+
if (!this.mcpRestartPending || this.runtimeMonitorsFrozen)
|
|
1541
|
+
return;
|
|
1542
|
+
if (this.mcpRestartCancelledByRecovery(committed))
|
|
1543
|
+
return;
|
|
1544
|
+
if (this.mcpServerAlive().alive) {
|
|
1545
|
+
this.logger.info({ trigger: committed }, "MCP revival restart stood down — replacement came up during the grace window");
|
|
1546
|
+
this.clearMcpRestartRequest();
|
|
1547
|
+
return;
|
|
1548
|
+
}
|
|
1549
|
+
// Re-check the idle gate. The grace turned what used to be a synchronous
|
|
1550
|
+
// decision into a 2s window, and "idle edge, then a queued message lands"
|
|
1551
|
+
// is the NORMAL ordering for queued delivery — so without this the
|
|
1552
|
+
// restart tears down a turn that started during the wait. Worse than the
|
|
1553
|
+
// race it replaced: the #485 proxy reply only runs at turn end, so an
|
|
1554
|
+
// in-flight answer would be swallowed with no fallback. Only the
|
|
1555
|
+
// 30-minute stale timer may interrupt a live turn, by design.
|
|
1556
|
+
//
|
|
1557
|
+
// Deliberately does NOT clear pending: the MCP really is gone, so the
|
|
1558
|
+
// request must survive to the next idle edge (which re-enters here), and
|
|
1559
|
+
// the stale timer keeps running underneath as the backstop.
|
|
1560
|
+
if (committed !== "stale_timeout" && !(this.instanceState === "idle" && this.pasteQueueDepth === 0)) {
|
|
1561
|
+
this.logger.info({ trigger: committed, instanceState: this.instanceState, pasteQueueDepth: this.pasteQueueDepth }, "MCP revival restart deferred — the pane went busy during the grace window");
|
|
1562
|
+
return;
|
|
1563
|
+
}
|
|
1564
|
+
// The suspicion was wrong: a real loss. Report it BEFORE asking for the
|
|
1565
|
+
// restart — instance-lifecycle's suppression path documents that it
|
|
1566
|
+
// assumes the mcp_died notification has already gone out, and this daemon
|
|
1567
|
+
// does not survive the restart to send it afterwards.
|
|
1568
|
+
this.reportDeferredMcpDeath();
|
|
1170
1569
|
this.clearMcpRestartRequest();
|
|
1570
|
+
this.logger.warn({ trigger: committed }, "Requesting instance restart to revive its MCP server");
|
|
1571
|
+
this.emit("mcp_restart_requested", { name: this.name, trigger: committed });
|
|
1572
|
+
}, Daemon.MCP_REPLACEMENT_GRACE_MS);
|
|
1573
|
+
this.mcpRestartGraceTimer.unref?.();
|
|
1574
|
+
}
|
|
1575
|
+
/**
|
|
1576
|
+
* A recovery is already reshaping the CLI: crash-loop handling
|
|
1577
|
+
* (healthCheckPaused), a respawn in flight (spawning), or a pause
|
|
1578
|
+
* (isPaused / frozen monitors). Each ends in a fresh CLI — and with it a
|
|
1579
|
+
* fresh MCP server — or in supervision ending, where a restart on top would
|
|
1580
|
+
* fight the recovery. Either way the revival restart is moot.
|
|
1581
|
+
*/
|
|
1582
|
+
mcpRestartCancelledByRecovery(trigger) {
|
|
1583
|
+
if (!(this.healthCheckPaused || this.spawning || this.isPaused || this.runtimeMonitorsFrozen || this.stormWindow?.isActive()))
|
|
1584
|
+
return false;
|
|
1585
|
+
this.logger.warn({ trigger }, "MCP revival restart cancelled — instance is already pausing/spawning/recovering");
|
|
1586
|
+
this.clearMcpRestartRequest();
|
|
1587
|
+
return true;
|
|
1588
|
+
}
|
|
1589
|
+
/**
|
|
1590
|
+
* Emit a dead-MCP report that was held back as a suspected replacement, once
|
|
1591
|
+
* it is clear no replacement is coming. Consumes the once-per-pid budget
|
|
1592
|
+
* here — deferring was to avoid spending it on a false alarm, not to drop
|
|
1593
|
+
* the incident.
|
|
1594
|
+
*/
|
|
1595
|
+
reportDeferredMcpDeath() {
|
|
1596
|
+
const pid = this.mcpDeathDeferredForPid;
|
|
1597
|
+
if (pid == null || this.mcpDeathNotifiedForPid === pid)
|
|
1598
|
+
return;
|
|
1599
|
+
if (this.mcpServerAlive().alive) {
|
|
1600
|
+
this.logger.info({ pid }, "Deferred MCP death not reported — a live MCP server is serving");
|
|
1601
|
+
this.mcpDeathDeferredForPid = null;
|
|
1171
1602
|
return;
|
|
1172
1603
|
}
|
|
1604
|
+
const authSuspected = this.authFailureUnresolved;
|
|
1605
|
+
const autoRestart = this.config.mcp_auto_restart !== false && !authSuspected;
|
|
1606
|
+
this.mcpDeathNotifiedForPid = pid;
|
|
1607
|
+
this.lastMcpDeathNotifiedAt = Date.now();
|
|
1608
|
+
this.logger.error({ pid, autoRestart, authSuspected }, "MCP server process is gone — no replacement arrived, instance has no agend tools");
|
|
1609
|
+
this.emit("mcp_died", { name: this.name, pid, autoRestart, authSuspected });
|
|
1610
|
+
}
|
|
1611
|
+
/** Socket-layer answer to pingMcpSockets(); records the pid the server reports. */
|
|
1612
|
+
handleMcpPong(msg, socket) {
|
|
1613
|
+
const waiter = this.mcpPingWaiters.get(msg.requestId);
|
|
1614
|
+
if (waiter) {
|
|
1615
|
+
this.mcpPingWaiters.delete(msg.requestId);
|
|
1616
|
+
waiter(true);
|
|
1617
|
+
}
|
|
1618
|
+
if (typeof msg.pid === "number")
|
|
1619
|
+
this.socketPids.set(socket, msg.pid);
|
|
1620
|
+
}
|
|
1621
|
+
/** Live IPC sockets of mcp-server processes that announced THIS instance's session. */
|
|
1622
|
+
liveMcpSockets() {
|
|
1623
|
+
const out = [];
|
|
1624
|
+
for (const [socket, sessionName] of this.socketSessionNames) {
|
|
1625
|
+
if (sessionName === this.name && !socket.destroyed)
|
|
1626
|
+
out.push(socket);
|
|
1627
|
+
}
|
|
1628
|
+
return out;
|
|
1629
|
+
}
|
|
1630
|
+
/**
|
|
1631
|
+
* MCP liveness with the pid slot as the first opinion and the IPC layer as
|
|
1632
|
+
* the authority: a live connection from a server announcing our session is
|
|
1633
|
+
* proof that tools are being served, whatever channel.mcp.pid says.
|
|
1634
|
+
*/
|
|
1635
|
+
mcpServerAlive() {
|
|
1636
|
+
const status = mcpServerState(this.instanceDir);
|
|
1637
|
+
if (status.state === "alive")
|
|
1638
|
+
return { alive: true, source: "pid", pid: status.pid };
|
|
1639
|
+
const sockets = this.liveMcpSockets();
|
|
1640
|
+
if (sockets.length > 0)
|
|
1641
|
+
return { alive: true, source: "connection", pid: this.socketPids.get(sockets[sockets.length - 1]) };
|
|
1642
|
+
return { alive: false, source: "none", pid: status.state === "dead" ? status.pid : undefined };
|
|
1643
|
+
}
|
|
1644
|
+
/** Point the pid slot at the server that is actually serving, so pid-based readers agree with the IPC layer. */
|
|
1645
|
+
repairMcpPidSlot(pid, reason) {
|
|
1646
|
+
if (!pid)
|
|
1647
|
+
return;
|
|
1648
|
+
try {
|
|
1649
|
+
writeFileSync(join(this.instanceDir, "channel.mcp.pid"), String(pid));
|
|
1650
|
+
this.logger.info({ pid, reason }, "channel.mcp.pid repaired to the live MCP server");
|
|
1651
|
+
}
|
|
1652
|
+
catch (err) {
|
|
1653
|
+
this.logger.debug({ err }, "could not repair channel.mcp.pid");
|
|
1654
|
+
}
|
|
1655
|
+
}
|
|
1656
|
+
/**
|
|
1657
|
+
* Active, LLM-free confirmation: ask each live socket to answer a ping at
|
|
1658
|
+
* the socket layer. Only logged — the open connection is already the
|
|
1659
|
+
* decision; a missing pong means an older mcp-server build or a wedged
|
|
1660
|
+
* process, and a truly dead socket closes on its own and drops out of
|
|
1661
|
+
* liveMcpSockets() by the next tick.
|
|
1662
|
+
*/
|
|
1663
|
+
pingMcpSockets(sockets) {
|
|
1664
|
+
for (const socket of sockets) {
|
|
1665
|
+
const requestId = ++this.mcpPingSeq;
|
|
1666
|
+
const timer = setTimeout(() => {
|
|
1667
|
+
if (!this.mcpPingWaiters.delete(requestId))
|
|
1668
|
+
return;
|
|
1669
|
+
this.logger.warn({ pid: this.socketPids.get(socket) }, "connected MCP server did not answer ping within 2s (older mcp-server build, or wedged)");
|
|
1670
|
+
}, Daemon.MCP_PING_TIMEOUT_MS);
|
|
1671
|
+
timer.unref?.();
|
|
1672
|
+
this.mcpPingWaiters.set(requestId, () => {
|
|
1673
|
+
clearTimeout(timer);
|
|
1674
|
+
this.logger.debug({ pid: this.socketPids.get(socket) }, "MCP server answered ping");
|
|
1675
|
+
});
|
|
1676
|
+
if (!this.ipcServer?.send(socket, { type: "ping", requestId })) {
|
|
1677
|
+
this.mcpPingWaiters.delete(requestId);
|
|
1678
|
+
clearTimeout(timer);
|
|
1679
|
+
}
|
|
1680
|
+
}
|
|
1681
|
+
}
|
|
1682
|
+
/**
|
|
1683
|
+
* Evidence that a server for this instance is serving (mcp_ready, a tool
|
|
1684
|
+
* call, a live connection). Stands down any pending revival restart and,
|
|
1685
|
+
* if an alarm had already gone out, retracts it.
|
|
1686
|
+
*/
|
|
1687
|
+
noteMcpProofOfLife(source, pid) {
|
|
1688
|
+
const hadAlarm = this.mcpDeathNotifiedForPid != null;
|
|
1689
|
+
const hadPending = this.mcpRestartPending || this.mcpDeathDeferredForPid != null;
|
|
1690
|
+
if (!hadAlarm && !hadPending)
|
|
1691
|
+
return;
|
|
1692
|
+
if (hadPending)
|
|
1693
|
+
this.logger.info({ source, pid }, "MCP server is serving again — standing down the pending revival restart");
|
|
1694
|
+
if (hadAlarm)
|
|
1695
|
+
this.emitMcpRecovered(source, pid);
|
|
1696
|
+
this.mcpDeathNotifiedForPid = null;
|
|
1697
|
+
this.mcpDeathDeferredForPid = null;
|
|
1173
1698
|
this.clearMcpRestartRequest();
|
|
1174
|
-
this.
|
|
1175
|
-
|
|
1699
|
+
if (pid && mcpServerState(this.instanceDir).state !== "alive")
|
|
1700
|
+
this.repairMcpPidSlot(pid, source);
|
|
1701
|
+
}
|
|
1702
|
+
emitMcpRecovered(source, pid) {
|
|
1703
|
+
this.logger.info({ source, pid }, "MCP server recovered — retracting the earlier death report");
|
|
1704
|
+
this.emit("mcp_recovered", { name: this.name, source, pid });
|
|
1176
1705
|
}
|
|
1177
1706
|
clearMcpRestartRequest() {
|
|
1178
1707
|
this.mcpRestartPending = false;
|
|
@@ -1180,6 +1709,11 @@ export class Daemon extends EventEmitter {
|
|
|
1180
1709
|
clearTimeout(this.mcpRestartStaleTimer);
|
|
1181
1710
|
this.mcpRestartStaleTimer = null;
|
|
1182
1711
|
}
|
|
1712
|
+
if (this.mcpRestartGraceTimer) {
|
|
1713
|
+
clearTimeout(this.mcpRestartGraceTimer);
|
|
1714
|
+
this.mcpRestartGraceTimer = null;
|
|
1715
|
+
}
|
|
1716
|
+
this.mcpRestartGraceTrigger = null;
|
|
1183
1717
|
}
|
|
1184
1718
|
startHealthCheck() {
|
|
1185
1719
|
if (this.runtimeMonitorsFrozen || this.healthCheckTimer)
|
|
@@ -1213,7 +1747,7 @@ export class Daemon extends EventEmitter {
|
|
|
1213
1747
|
this.emitSupervisionEnded("instance directory was removed from disk", "Recreate the instance, or delete it from fleet.yaml.");
|
|
1214
1748
|
return;
|
|
1215
1749
|
}
|
|
1216
|
-
if (!this.tmux || this.spawning || this.healthCheckPaused
|
|
1750
|
+
if (!this.tmux || this.spawning || this.healthCheckPaused) {
|
|
1217
1751
|
scheduleNext();
|
|
1218
1752
|
return;
|
|
1219
1753
|
}
|
|
@@ -1271,28 +1805,29 @@ export class Daemon extends EventEmitter {
|
|
|
1271
1805
|
let nullReason;
|
|
1272
1806
|
if (!paneStatus) {
|
|
1273
1807
|
const serverAlive = await TmuxManager.sessionExists(this.tmuxSessionName);
|
|
1274
|
-
|
|
1808
|
+
// A server may have already restarted before this daemon's health
|
|
1809
|
+
// tick. The new PID is the durable generation boundary: treat it as
|
|
1810
|
+
// a fleet storm even though `has-session` is true again.
|
|
1811
|
+
const generationChanged = serverAlive
|
|
1812
|
+
? this.stormWindow?.observeServerAlive(await TmuxManager.getServerPid(this.tmuxSessionName)) === true
|
|
1813
|
+
: false;
|
|
1814
|
+
if (generationChanged)
|
|
1815
|
+
this.emit("tmux_server_crash", this.name);
|
|
1816
|
+
if (!serverAlive || generationChanged || this.stormWindow?.needsRecovery(this.name)) {
|
|
1275
1817
|
crashType = "server";
|
|
1276
|
-
nullReason = "server_gone";
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
this.
|
|
1285
|
-
this.emit("tmux_server_crash", this.name);
|
|
1286
|
-
if (!Daemon.tmuxServerRecoveryTimer) {
|
|
1287
|
-
Daemon.tmuxServerRecoveryTimer = setTimeout(() => {
|
|
1288
|
-
Daemon.tmuxServerRecoveryTimer = null;
|
|
1289
|
-
Daemon.tmuxServerPaused = false;
|
|
1290
|
-
}, 30_000);
|
|
1291
|
-
}
|
|
1292
|
-
scheduleNext();
|
|
1293
|
-
return;
|
|
1818
|
+
nullReason = serverAlive ? "server_storm_window_loss" : "server_gone";
|
|
1819
|
+
if (!serverAlive) {
|
|
1820
|
+
this.logger.error(`tmux server died — all ${cliLabel} windows lost`);
|
|
1821
|
+
const distinct = this.stormWindow?.recordServerDead(this.name, [this.name]) ?? true;
|
|
1822
|
+
if (distinct)
|
|
1823
|
+
this.emit("tmux_server_crash", this.name);
|
|
1824
|
+
}
|
|
1825
|
+
else {
|
|
1826
|
+
this.stormWindow?.addAffected(this.name);
|
|
1294
1827
|
}
|
|
1295
|
-
|
|
1828
|
+
// The fleet breaker owns the delay and extends it on every new
|
|
1829
|
+
// server generation. Do not schedule a competing fixed timer.
|
|
1830
|
+
await this.stormWindow?.waitForSpawnAllowed();
|
|
1296
1831
|
}
|
|
1297
1832
|
else {
|
|
1298
1833
|
// null but server alive: window-level disappearance. Probe whether
|
|
@@ -1388,6 +1923,14 @@ export class Daemon extends EventEmitter {
|
|
|
1388
1923
|
}
|
|
1389
1924
|
// Append to crash history
|
|
1390
1925
|
this.appendCrashHistory({ exitCode, lastOutput, crashType, reason: nullReason });
|
|
1926
|
+
// A fleet-wide server loss is one infrastructure incident, not N
|
|
1927
|
+
// independent CLI crash loops. StormWindow owns its escalation; do
|
|
1928
|
+
// not let the same three server generations permanently trip every
|
|
1929
|
+
// daemon's per-instance 3-in-5m breaker.
|
|
1930
|
+
if (crashType === "server") {
|
|
1931
|
+
this.crashTimestamps = [];
|
|
1932
|
+
this.crashCount = 0;
|
|
1933
|
+
}
|
|
1391
1934
|
if (max_retries <= 0) {
|
|
1392
1935
|
this.healthCheckPaused = true;
|
|
1393
1936
|
this.logger.warn(`${cliLabel} window died — automatic restart is disabled`);
|
|
@@ -1474,9 +2017,15 @@ export class Daemon extends EventEmitter {
|
|
|
1474
2017
|
this.setProcessStatus("running");
|
|
1475
2018
|
this.logger.info({ resumed }, `Respawned ${cliLabel} window after crash`);
|
|
1476
2019
|
this.emit("crash_respawn", this.name);
|
|
2020
|
+
// Emit the per-instance audit event while the storm is still active
|
|
2021
|
+
// so its chat notification is suppressed into the fleet summary.
|
|
2022
|
+
if (crashType === "server")
|
|
2023
|
+
this.stormWindow?.markRecovered(this.name);
|
|
1477
2024
|
}
|
|
1478
2025
|
catch (err) {
|
|
1479
|
-
this.
|
|
2026
|
+
if (!this.handOffBackendUnreachableRespawn(err)) {
|
|
2027
|
+
this.logger.error({ err }, `Failed to respawn ${cliLabel} window`);
|
|
2028
|
+
}
|
|
1480
2029
|
}
|
|
1481
2030
|
}
|
|
1482
2031
|
catch (err) {
|
|
@@ -1585,10 +2134,37 @@ export class Daemon extends EventEmitter {
|
|
|
1585
2134
|
// A prompt is not an error and must not enter the PTY recovery gate.
|
|
1586
2135
|
// Continue scanning real errors in this same snapshot.
|
|
1587
2136
|
}
|
|
1588
|
-
//
|
|
2137
|
+
// If a backend can derive activity from this exact pane, trust its
|
|
2138
|
+
// explicit null (tool completed) instead of falling back to a possibly
|
|
2139
|
+
// stale transcript activity from the previous scan.
|
|
2140
|
+
const paneActivity = this.backend?.getPaneActivity
|
|
2141
|
+
? this.backend.getPaneActivity(pane)
|
|
2142
|
+
: this.currentActivity;
|
|
2143
|
+
const hasPendingWork = this.pendingWork.hasPendingWork();
|
|
2144
|
+
if (!hasPendingWork)
|
|
2145
|
+
this.blockingProcessDetector.reset();
|
|
2146
|
+
const blockingProcess = hasPendingWork
|
|
2147
|
+
? this.blockingProcessDetector.observe(pane, paneActivity, Date.now())
|
|
2148
|
+
: null;
|
|
2149
|
+
if (blockingProcess) {
|
|
2150
|
+
// Reuse the hang-notification bridge: it offers explicit Restart/Wait
|
|
2151
|
+
// choices and, unlike pty_error, does not retire a still-useful Cancel
|
|
2152
|
+
// button merely because a foreground process owns stdin.
|
|
2153
|
+
this.logger.warn(blockingProcess, "Foreground process is blocking the agent input loop");
|
|
2154
|
+
this.hangDetector?.emit("hang", { unchangedForMs: blockingProcess.blockedForMs });
|
|
2155
|
+
}
|
|
2156
|
+
// Auto-dismiss runtime dialogs (e.g. Codex rate limit model switch, or
|
|
2157
|
+
// Claude's session-resume prompt painted after the startup scan moved
|
|
2158
|
+
// on). Runtime table only: startup patterns are loose on purpose and
|
|
2159
|
+
// must never send keys into an ordinary transcript.
|
|
2160
|
+
let blockingSeen = null;
|
|
1589
2161
|
for (const dialog of dialogs) {
|
|
1590
|
-
if (!
|
|
2162
|
+
if (!Daemon.dialogMatches(dialog, pane))
|
|
1591
2163
|
continue;
|
|
2164
|
+
if (dialog.blocksDelivery || dialog.holdOnly)
|
|
2165
|
+
blockingSeen = dialog;
|
|
2166
|
+
if (dialog.holdOnly)
|
|
2167
|
+
break; // recognised, deliberately not answered; trackDialogParked reports it
|
|
1592
2168
|
// These keys go straight into the pane. Sent while a message delivery is
|
|
1593
2169
|
// mid-transaction, an `Escape` wipes the pasted text and an `Enter`
|
|
1594
2170
|
// submits it half-composed — the user sees a message that vanished. Skip
|
|
@@ -1608,10 +2184,14 @@ export class Daemon extends EventEmitter {
|
|
|
1608
2184
|
}
|
|
1609
2185
|
});
|
|
1610
2186
|
if (!dismissed) {
|
|
1611
|
-
this.logger.
|
|
2187
|
+
this.logger.info({ dialog: dialog.description }, "Dialog dismissal deferred — pane write in flight");
|
|
1612
2188
|
}
|
|
2189
|
+
this.trackDialogParked(blockingSeen);
|
|
1613
2190
|
return; // Dialog handled (or deliberately deferred): skip error checks this cycle
|
|
1614
2191
|
}
|
|
2192
|
+
this.trackDialogParked(blockingSeen);
|
|
2193
|
+
if (blockingSeen)
|
|
2194
|
+
return; // held dialog: skip error checks this cycle
|
|
1615
2195
|
this.evaluateErrorPatterns(pane, patterns, readyPattern, Date.now(), busyPattern);
|
|
1616
2196
|
}
|
|
1617
2197
|
catch {
|
|
@@ -1657,6 +2237,7 @@ export class Daemon extends EventEmitter {
|
|
|
1657
2237
|
if (seen > 0)
|
|
1658
2238
|
this.lastErrorCount.set(Daemon.errorPatternKey(ep), seen);
|
|
1659
2239
|
}
|
|
2240
|
+
this.authFailureUnresolved = false;
|
|
1660
2241
|
this.clearErrorRecoveryGate();
|
|
1661
2242
|
this.logger.info({ downtime_s: downtime }, "PTY error recovered — agent is ready again");
|
|
1662
2243
|
this.emit("pty_recovered", { name: this.name, downtime_s: downtime });
|
|
@@ -1677,6 +2258,19 @@ export class Daemon extends EventEmitter {
|
|
|
1677
2258
|
this.logger.warn({ errorType: this.lastDetectedErrorType }, "PTY error recovery deadline expired — resuming error scan");
|
|
1678
2259
|
this.clearErrorRecoveryGate();
|
|
1679
2260
|
}
|
|
2261
|
+
// A model_error deferred mid-turn: the turn is over (or the pane left
|
|
2262
|
+
// "working"), so decide now — still on screen means the CLI gave up and
|
|
2263
|
+
// the user must act; gone means a retry got past it and nobody needs to know.
|
|
2264
|
+
if (this.pendingModelErrorKey && this.instanceState !== "working") {
|
|
2265
|
+
const key = this.pendingModelErrorKey;
|
|
2266
|
+
this.pendingModelErrorKey = null;
|
|
2267
|
+
const ep = patterns.find(candidate => Daemon.errorPatternKey(candidate) === key);
|
|
2268
|
+
if (ep && ep.pattern.test(pane)) {
|
|
2269
|
+
this.emitErrorPattern(ep, key, pane, now);
|
|
2270
|
+
return;
|
|
2271
|
+
}
|
|
2272
|
+
this.logger.debug({ errorKey: key }, "Deferred model_error cleared before idle — dropped");
|
|
2273
|
+
}
|
|
1680
2274
|
// State: monitoring — count-based new-error detection over the full pane.
|
|
1681
2275
|
for (const ep of patterns) {
|
|
1682
2276
|
const key = Daemon.errorPatternKey(ep);
|
|
@@ -1704,25 +2298,49 @@ export class Daemon extends EventEmitter {
|
|
|
1704
2298
|
continue;
|
|
1705
2299
|
}
|
|
1706
2300
|
this.lastErrorCount.set(key, count);
|
|
1707
|
-
//
|
|
1708
|
-
//
|
|
1709
|
-
//
|
|
1710
|
-
if (
|
|
1711
|
-
this.
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
this.
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
this.
|
|
2301
|
+
// Kiro's `auto` model picks a concrete model per turn and reroutes around
|
|
2302
|
+
// a temporarily unavailable one on its own — telling the user to run
|
|
2303
|
+
// /model would ask them to fix something the CLI is already handling.
|
|
2304
|
+
if (ep.type === "model_error" && ep.action === "notify" && this.config.model === "auto") {
|
|
2305
|
+
this.logger.debug({ errorType: ep.type }, "model_error suppressed — model 'auto' reroutes on its own");
|
|
2306
|
+
continue;
|
|
2307
|
+
}
|
|
2308
|
+
// A mid-turn model error is often a transient the CLI retries past. Hold
|
|
2309
|
+
// the notification until the turn ends and notify only if the error is
|
|
2310
|
+
// still on the idle screen (see the deferred check above).
|
|
2311
|
+
if (ep.type === "model_error" && ep.action === "notify" && this.instanceState === "working") {
|
|
2312
|
+
this.pendingModelErrorKey = key;
|
|
2313
|
+
this.logger.debug({ errorType: ep.type }, "model_error deferred — instance is mid-turn, re-checking at idle");
|
|
2314
|
+
continue;
|
|
2315
|
+
}
|
|
2316
|
+
this.emitErrorPattern(ep, key, pane, now);
|
|
1723
2317
|
break; // Only handle first unsuppressed new error per scan
|
|
1724
2318
|
}
|
|
1725
2319
|
}
|
|
2320
|
+
/** Arm the recovery gates and emit one detected error pattern. */
|
|
2321
|
+
emitErrorPattern(ep, key, pane, now) {
|
|
2322
|
+
// skipRecoveryWait: this error self-recovers (e.g. a timeout — Kiro is
|
|
2323
|
+
// back at its prompt immediately). Keep monitoring so the next occurrence
|
|
2324
|
+
// can fire without waiting for a possibly startup-only ready pattern.
|
|
2325
|
+
if (!ep.skipRecoveryWait) {
|
|
2326
|
+
this.errorWaitingForRecovery = true;
|
|
2327
|
+
this.errorDetectedAt = now;
|
|
2328
|
+
this.errorRecoveryDeadlineAt = now + Daemon.ERROR_RECOVERY_TIMEOUT_MS;
|
|
2329
|
+
this.activeErrorPatternKey = key;
|
|
2330
|
+
this.lastDetectedErrorType = ep.type;
|
|
2331
|
+
}
|
|
2332
|
+
this.lastErrorNotifiedAt.set(key, now);
|
|
2333
|
+
if (ep.action === "failover")
|
|
2334
|
+
this.lastFailoverAt = now;
|
|
2335
|
+
const message = this.resolveErrorMessage(pane, ep);
|
|
2336
|
+
// Remember an auth failure for as long as it is unresolved. An auth-broken
|
|
2337
|
+
// CLI cannot finish the turn it is holding, so every other detector in this
|
|
2338
|
+
// scan would otherwise keep reporting the same dead instance forever.
|
|
2339
|
+
if (ep.type === "auth_error")
|
|
2340
|
+
this.authFailureUnresolved = true;
|
|
2341
|
+
this.logger.warn({ errorType: ep.type, action: ep.action }, `PTY error detected: ${message}`);
|
|
2342
|
+
this.emit("pty_error", { name: this.name, ...ep, message });
|
|
2343
|
+
}
|
|
1726
2344
|
/**
|
|
1727
2345
|
* Notification text for a detected pattern. `formatMessage` patterns build it
|
|
1728
2346
|
* from the match so the user gets the specifics (e.g. which period and what
|
|
@@ -1761,6 +2379,97 @@ export class Daemon extends EventEmitter {
|
|
|
1761
2379
|
const cancelKey = this.backend?.getCancelKey() ?? "Escape";
|
|
1762
2380
|
await this.tmux?.sendSpecialKey(cancelKey);
|
|
1763
2381
|
}
|
|
2382
|
+
/**
|
|
2383
|
+
* Drop every not-yet-started pane delivery which predates a user cancel.
|
|
2384
|
+
* Promise chains cannot be removed, so entries become generation-checked
|
|
2385
|
+
* no-ops. An entry already past its check is allowed to finish its current PTY
|
|
2386
|
+
* transaction; the interrupt key still cancels the CLI generation.
|
|
2387
|
+
*/
|
|
2388
|
+
clearPendingDeliveries(fleetEpoch) {
|
|
2389
|
+
this.deliveryEpoch = fleetEpoch === undefined
|
|
2390
|
+
? this.deliveryEpoch + 1
|
|
2391
|
+
: Math.max(this.deliveryEpoch, fleetEpoch);
|
|
2392
|
+
this.logger.info({ deliveryEpoch: this.deliveryEpoch }, "Pending delivery queue cleared by user cancel");
|
|
2393
|
+
}
|
|
2394
|
+
captureDeliveryEpoch(value) {
|
|
2395
|
+
const epoch = typeof value === "number" && Number.isSafeInteger(value) && value >= 0
|
|
2396
|
+
? value
|
|
2397
|
+
: this.deliveryEpoch;
|
|
2398
|
+
if (epoch > this.deliveryEpoch)
|
|
2399
|
+
this.deliveryEpoch = epoch;
|
|
2400
|
+
return epoch;
|
|
2401
|
+
}
|
|
2402
|
+
isDeliveryEpochCurrent(epoch) {
|
|
2403
|
+
return epoch === this.deliveryEpoch;
|
|
2404
|
+
}
|
|
2405
|
+
queueRawPaste(rawText, deliveryEpoch = this.deliveryEpoch, confirmClear = false) {
|
|
2406
|
+
if (!this.tmux || !this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
2407
|
+
return;
|
|
2408
|
+
this.pasteLock = this.pasteLock.then(async () => {
|
|
2409
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch)) {
|
|
2410
|
+
this.logger.info("Pending raw delivery dropped by user cancel");
|
|
2411
|
+
return;
|
|
2412
|
+
}
|
|
2413
|
+
await this.wake();
|
|
2414
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
2415
|
+
return;
|
|
2416
|
+
await this.deliverMessage(rawText, undefined, { deliveryEpoch });
|
|
2417
|
+
if (confirmClear)
|
|
2418
|
+
await this.confirmBackendClearDialog();
|
|
2419
|
+
this.logger.debug({ text: rawText.slice(0, 100) }, "Raw paste delivered");
|
|
2420
|
+
}).catch(err => {
|
|
2421
|
+
this.logger.warn({ err: err.message }, "raw_paste delivery error");
|
|
2422
|
+
});
|
|
2423
|
+
}
|
|
2424
|
+
/**
|
|
2425
|
+
* Confirm a backend's terminal-level /clear prompt after the channel-level
|
|
2426
|
+
* admin confirmation has already succeeded. Poll instead of pasting `/clear\ny`
|
|
2427
|
+
* as one buffer: Kiro versions that clear immediately interpret that queued
|
|
2428
|
+
* `y` as a brand-new user message.
|
|
2429
|
+
*/
|
|
2430
|
+
async confirmBackendClearDialog() {
|
|
2431
|
+
const dialog = this.backend?.getClearConfirmationDialog?.();
|
|
2432
|
+
if (!dialog || !this.tmux)
|
|
2433
|
+
return;
|
|
2434
|
+
const matches = (pane) => {
|
|
2435
|
+
dialog.pattern.lastIndex = 0;
|
|
2436
|
+
return dialog.pattern.test(pane);
|
|
2437
|
+
};
|
|
2438
|
+
const deadline = Date.now() + 3_000;
|
|
2439
|
+
while (Date.now() <= deadline) {
|
|
2440
|
+
const pane = await this.tmux.capturePane().catch(() => "");
|
|
2441
|
+
if (matches(pane)) {
|
|
2442
|
+
const sent = await this.paneWriteLock.run(async () => {
|
|
2443
|
+
// Re-check under the write lock so a prompt that disappeared between
|
|
2444
|
+
// capture and lock acquisition never receives a stray literal `y`.
|
|
2445
|
+
const currentPane = await this.tmux.capturePane().catch(() => "");
|
|
2446
|
+
if (!matches(currentPane))
|
|
2447
|
+
return false;
|
|
2448
|
+
const specialKeys = new Set(["Up", "Down", "Enter", "Escape", "Right", "Left"]);
|
|
2449
|
+
for (const key of dialog.keys) {
|
|
2450
|
+
const ok = specialKeys.has(key)
|
|
2451
|
+
? await this.tmux.sendSpecialKey(key)
|
|
2452
|
+
: await this.tmux.sendKeys(key);
|
|
2453
|
+
if (!ok)
|
|
2454
|
+
return false;
|
|
2455
|
+
await new Promise(resolve => setTimeout(resolve, 100));
|
|
2456
|
+
}
|
|
2457
|
+
return true;
|
|
2458
|
+
});
|
|
2459
|
+
if (sent) {
|
|
2460
|
+
this.logger.info({ dialog: dialog.description }, "Auto-confirmed backend clear dialog");
|
|
2461
|
+
}
|
|
2462
|
+
else {
|
|
2463
|
+
this.logger.warn({ dialog: dialog.description }, "Backend clear dialog disappeared or confirmation keys failed");
|
|
2464
|
+
}
|
|
2465
|
+
return;
|
|
2466
|
+
}
|
|
2467
|
+
await new Promise(resolve => setTimeout(resolve, 100));
|
|
2468
|
+
}
|
|
2469
|
+
// Recent Kiro releases may clear immediately without a second prompt. That
|
|
2470
|
+
// is a successful no-op here, not an error.
|
|
2471
|
+
this.logger.debug({ dialog: dialog.description }, "Backend clear dialog did not appear");
|
|
2472
|
+
}
|
|
1764
2473
|
/** Send the backend-specific graceful quit command/key sequence. */
|
|
1765
2474
|
async sendQuitSequence() {
|
|
1766
2475
|
if (!this.tmux || !this.backend)
|
|
@@ -1768,18 +2477,20 @@ export class Daemon extends EventEmitter {
|
|
|
1768
2477
|
const quitCmd = this.backend.getQuitCommand();
|
|
1769
2478
|
const quitKey = this.backend.getQuitKey?.();
|
|
1770
2479
|
if (quitCmd) {
|
|
1771
|
-
await this.tmux.sendKeys(quitCmd)
|
|
2480
|
+
if (!await this.tmux.sendKeys(quitCmd))
|
|
2481
|
+
return false;
|
|
1772
2482
|
// Delay before Enter to prevent tmux server races when instances stop in
|
|
1773
2483
|
// parallel (same pattern as pasteText).
|
|
1774
2484
|
await new Promise(r => setTimeout(r, 150));
|
|
1775
|
-
|
|
1776
|
-
return true;
|
|
2485
|
+
return this.tmux.sendSpecialKey("Enter");
|
|
1777
2486
|
}
|
|
1778
2487
|
if (!quitKey)
|
|
1779
2488
|
return false;
|
|
1780
2489
|
const presses = Math.max(1, Math.floor(this.backend.getQuitKeyPresses?.() ?? 1));
|
|
1781
2490
|
for (let i = 0; i < presses; i++) {
|
|
1782
|
-
await this.tmux.sendSpecialKey(quitKey);
|
|
2491
|
+
const sent = await this.tmux.sendSpecialKey(quitKey);
|
|
2492
|
+
if (!sent)
|
|
2493
|
+
return false;
|
|
1783
2494
|
if (i + 1 < presses)
|
|
1784
2495
|
await new Promise(r => setTimeout(r, 250));
|
|
1785
2496
|
}
|
|
@@ -1812,8 +2523,23 @@ export class Daemon extends EventEmitter {
|
|
|
1812
2523
|
}
|
|
1813
2524
|
}
|
|
1814
2525
|
}
|
|
1815
|
-
if (!killed)
|
|
1816
|
-
this.logger.warn("CLI did not exit gracefully within 3s
|
|
2526
|
+
if (!killed) {
|
|
2527
|
+
this.logger.warn({ quitSent }, "CLI did not exit gracefully within 3s — falling back to SIGTERM");
|
|
2528
|
+
await this.killProcessTree("SIGTERM");
|
|
2529
|
+
for (let i = 0; i < 5; i++) {
|
|
2530
|
+
await new Promise(r => setTimeout(r, 200));
|
|
2531
|
+
const status = await this.tmux.getPaneStatus();
|
|
2532
|
+
if (!status || !status.alive) {
|
|
2533
|
+
killed = true;
|
|
2534
|
+
break;
|
|
2535
|
+
}
|
|
2536
|
+
}
|
|
2537
|
+
}
|
|
2538
|
+
if (!killed) {
|
|
2539
|
+
this.logger.warn("CLI process tree survived SIGTERM — falling back to SIGKILL");
|
|
2540
|
+
await this.killProcessTree("SIGKILL");
|
|
2541
|
+
await new Promise(r => setTimeout(r, 200));
|
|
2542
|
+
}
|
|
1817
2543
|
// Always kill window — remain-on-exit keeps dead panes around after CLI exits
|
|
1818
2544
|
const stoppedWindowId = this.tmux.getWindowId();
|
|
1819
2545
|
await this.tmux.killWindow();
|
|
@@ -1862,6 +2588,33 @@ export class Daemon extends EventEmitter {
|
|
|
1862
2588
|
stateChangedAt: Date.now(),
|
|
1863
2589
|
};
|
|
1864
2590
|
}
|
|
2591
|
+
/**
|
|
2592
|
+
* Answer a FleetManager state query. Most callers only need the cached state,
|
|
2593
|
+
* but lifecycle decisions such as post-reply Cancel retirement need an
|
|
2594
|
+
* authoritative pane observation: after startup the last transition can stay
|
|
2595
|
+
* "working" even though the CLI has since settled without emitting another
|
|
2596
|
+
* control-mode output record.
|
|
2597
|
+
*/
|
|
2598
|
+
async respondToInstanceStateQuery(msg, socket) {
|
|
2599
|
+
try {
|
|
2600
|
+
if (msg.refresh === true) {
|
|
2601
|
+
await this.captureAndEvaluateInstanceState("state_query");
|
|
2602
|
+
}
|
|
2603
|
+
const snapshot = this.getInstanceStateSnapshot();
|
|
2604
|
+
this.ipcServer?.send(socket, {
|
|
2605
|
+
type: "instance_state_response",
|
|
2606
|
+
requestId: msg.requestId,
|
|
2607
|
+
instanceName: this.name,
|
|
2608
|
+
...snapshot,
|
|
2609
|
+
state: this.isPaused ? "paused" : snapshot.state,
|
|
2610
|
+
processStatus: this.processStatus,
|
|
2611
|
+
pausedAt: this.lastPausedAt,
|
|
2612
|
+
});
|
|
2613
|
+
}
|
|
2614
|
+
catch (err) {
|
|
2615
|
+
this.logger.debug({ err: err.message }, "Instance state query failed");
|
|
2616
|
+
}
|
|
2617
|
+
}
|
|
1865
2618
|
/** Gracefully stop the CLI while keeping its remain-on-exit tmux window. */
|
|
1866
2619
|
/**
|
|
1867
2620
|
* Mark the instance to pause as soon as its pane is idle. Used for auth
|
|
@@ -1881,7 +2634,14 @@ export class Daemon extends EventEmitter {
|
|
|
1881
2634
|
if (this.getPauseWakeState() !== "active")
|
|
1882
2635
|
return;
|
|
1883
2636
|
}
|
|
1884
|
-
|
|
2637
|
+
// `pauseAllowStuck` is consumed here, not merely read: it authorises exactly
|
|
2638
|
+
// one pause from a stuck pane (the auth-deferred one) and must never leak
|
|
2639
|
+
// into an ordinary idle-timeout pause of a busy instance.
|
|
2640
|
+
const allowStuck = this.pauseAllowStuck;
|
|
2641
|
+
this.pauseAllowStuck = false;
|
|
2642
|
+
const pausableState = this.instanceState === "idle"
|
|
2643
|
+
|| (allowStuck && this.instanceState === "stuck");
|
|
2644
|
+
if (!pausableState || this.pasteQueueDepth > 0) {
|
|
1885
2645
|
this.pauseRequested = false;
|
|
1886
2646
|
return;
|
|
1887
2647
|
}
|
|
@@ -1945,6 +2705,10 @@ export class Daemon extends EventEmitter {
|
|
|
1945
2705
|
// An explicit wake (e.g. after the user re-logs in) cancels a deferred
|
|
1946
2706
|
// auth pause — otherwise the instance would pause again the moment it idles.
|
|
1947
2707
|
this.pausePending = false;
|
|
2708
|
+
this.authFailureUnresolved = false;
|
|
2709
|
+
this.loginScreenReported = false;
|
|
2710
|
+
this.fatalStartupReported = false;
|
|
2711
|
+
this.fatalStartupBlocked = false;
|
|
1948
2712
|
if (this.pauseWakeState === "active")
|
|
1949
2713
|
return;
|
|
1950
2714
|
if (this.pauseWakeState === "pausing")
|
|
@@ -1957,19 +2721,30 @@ export class Daemon extends EventEmitter {
|
|
|
1957
2721
|
}
|
|
1958
2722
|
this.pauseWakeState = "waking";
|
|
1959
2723
|
this.beginSpawn();
|
|
2724
|
+
// A wake is a real resume launch: it gets at least the backend's resume
|
|
2725
|
+
// budget (kiro: 60s — the conversation must come back from the backend
|
|
2726
|
+
// before anything paints), never less than the caller's or the configured
|
|
2727
|
+
// startup_timeout_ms.
|
|
2728
|
+
const budgetMs = this.wakeBudgetMs(timeoutMs);
|
|
2729
|
+
// No outer race. trySpawn cannot be cancelled: racing it against a timer
|
|
2730
|
+
// and giving up left the daemon "paused" (monitors frozen) while the spawn
|
|
2731
|
+
// went on to revive the window behind its back. Every phase inside is
|
|
2732
|
+
// bounded on its own terms — the SpawnGate wait is deliberate storm /
|
|
2733
|
+
// concurrency coordination, first output + idle is `budgetMs`, the dialog
|
|
2734
|
+
// scan is STARTUP_DIALOG_BUDGET_MS — so the outcome is always trySpawn's
|
|
2735
|
+
// own verdict. A soft timer only makes a slow wake visible in the log.
|
|
1960
2736
|
const transition = this.autoPauseController.wakeOnDeliver(async () => {
|
|
1961
|
-
|
|
2737
|
+
const slow = setTimeout(() => {
|
|
2738
|
+
this.logger.warn({ budgetMs }, "Wake is exceeding its budget — still waiting for the spawn (it cannot be cancelled)");
|
|
2739
|
+
}, budgetMs + STARTUP_DIALOG_BUDGET_MS);
|
|
2740
|
+
slow.unref?.();
|
|
1962
2741
|
try {
|
|
1963
|
-
const ready = await
|
|
1964
|
-
this.trySpawn(true, timeoutMs),
|
|
1965
|
-
new Promise(resolve => { timeout = setTimeout(() => resolve(false), timeoutMs); }),
|
|
1966
|
-
]);
|
|
2742
|
+
const ready = await this.trySpawn(true, budgetMs);
|
|
1967
2743
|
if (!ready)
|
|
1968
|
-
throw new Error(`Wake
|
|
2744
|
+
throw new Error(`Wake failed: the CLI did not become ready (budget ${budgetMs}ms)`);
|
|
1969
2745
|
}
|
|
1970
2746
|
finally {
|
|
1971
|
-
|
|
1972
|
-
clearTimeout(timeout);
|
|
2747
|
+
clearTimeout(slow);
|
|
1973
2748
|
}
|
|
1974
2749
|
});
|
|
1975
2750
|
this.pauseWakeTransition = transition;
|
|
@@ -2021,6 +2796,7 @@ export class Daemon extends EventEmitter {
|
|
|
2021
2796
|
// tool_result (interrupted, crashed, cancelled), which would otherwise leave
|
|
2022
2797
|
// the last tool pinned to the progress line for the rest of the session.
|
|
2023
2798
|
this.publishActivity(null);
|
|
2799
|
+
this.resetToolProgress();
|
|
2024
2800
|
// Must run before the mcpRestartPending branch below: the pane text is the
|
|
2025
2801
|
// only copy of the answer, and the revival restart is about to clear it.
|
|
2026
2802
|
this.maybeProxyReplyOnTurnEnd(pane);
|
|
@@ -2040,8 +2816,13 @@ export class Daemon extends EventEmitter {
|
|
|
2040
2816
|
if (snapshot.state !== "idle")
|
|
2041
2817
|
this.pauseRequested = false;
|
|
2042
2818
|
// A pause deferred by an auth failure: retry the moment the pane settles.
|
|
2043
|
-
|
|
2819
|
+
// "stuck" counts as settled here, and that is the whole point — an expired
|
|
2820
|
+
// session cannot complete the turn it is holding, so the pane never returns
|
|
2821
|
+
// to idle and waiting for idle meant waiting forever. The instance stayed
|
|
2822
|
+
// live, kept its warm slot, and re-raised a hang alert on every scan.
|
|
2823
|
+
if (this.pausePending && (snapshot.state === "idle" || snapshot.state === "stuck") && this.pasteQueueDepth === 0) {
|
|
2044
2824
|
this.pausePending = false;
|
|
2825
|
+
this.pauseAllowStuck = snapshot.state === "stuck";
|
|
2045
2826
|
this.emit("auto_pause_requested", { name: this.name, idleSince: snapshot.stateChangedAt });
|
|
2046
2827
|
return;
|
|
2047
2828
|
}
|
|
@@ -2071,6 +2852,7 @@ export class Daemon extends EventEmitter {
|
|
|
2071
2852
|
return;
|
|
2072
2853
|
this.turnHadInbound = true;
|
|
2073
2854
|
this.turnOutboundDelivered = false;
|
|
2855
|
+
this.turnReplyDelivered = false;
|
|
2074
2856
|
this.turnCorrelationId = meta.correlation_id || undefined;
|
|
2075
2857
|
// The last non-empty line of what we pasted: everything on screen after it
|
|
2076
2858
|
// is the agent's own output.
|
|
@@ -2086,13 +2868,38 @@ export class Daemon extends EventEmitter {
|
|
|
2086
2868
|
maybeProxyReplyOnTurnEnd(pane) {
|
|
2087
2869
|
const hadInbound = this.turnHadInbound;
|
|
2088
2870
|
const delivered = this.turnOutboundDelivered;
|
|
2871
|
+
const replyDelivered = this.turnReplyDelivered;
|
|
2089
2872
|
const correlationId = this.turnCorrelationId;
|
|
2090
2873
|
const inboundMarker = this.turnInboundMarker;
|
|
2091
2874
|
this.turnHadInbound = false;
|
|
2092
2875
|
this.turnOutboundDelivered = false;
|
|
2876
|
+
this.turnReplyDelivered = false;
|
|
2093
2877
|
this.turnCorrelationId = undefined;
|
|
2094
2878
|
this.turnInboundMarker = undefined;
|
|
2095
|
-
if (!hadInbound ||
|
|
2879
|
+
if (!hadInbound || this.isPaused)
|
|
2880
|
+
return;
|
|
2881
|
+
// Claude sometimes prints a broken XML tool call as plain text and returns
|
|
2882
|
+
// idle without ever invoking `reply`. This is independent of MCP liveness:
|
|
2883
|
+
// the model malformed the call before the server could receive it.
|
|
2884
|
+
if (!replyDelivered && this.isClaudeCodeBackend() && pane) {
|
|
2885
|
+
const malformed = detectMalformedClaudeToolCall(pane, { inboundMarker });
|
|
2886
|
+
if (malformed) {
|
|
2887
|
+
const stale = malformed.signature === this.lastMalformedToolCallSignature;
|
|
2888
|
+
this.lastMalformedToolCallSignature = malformed.signature;
|
|
2889
|
+
if (!stale) {
|
|
2890
|
+
const recovered = malformed.text != null;
|
|
2891
|
+
this.logger.warn({ correlationId, recovered }, recovered
|
|
2892
|
+
? "Malformed Claude tool call detected — attempting to recover reply text"
|
|
2893
|
+
: "Malformed Claude tool call detected — reply text could not be extracted");
|
|
2894
|
+
this.emit("malformed_tool_call", { name: this.name, correlationId, recovered });
|
|
2895
|
+
if (malformed.text)
|
|
2896
|
+
this.sendRecoveredMalformedReply(malformed.text, correlationId);
|
|
2897
|
+
}
|
|
2898
|
+
// Never let the broader dead-MCP fallback relay the XML/chrome too.
|
|
2899
|
+
return;
|
|
2900
|
+
}
|
|
2901
|
+
}
|
|
2902
|
+
if (delivered)
|
|
2096
2903
|
return;
|
|
2097
2904
|
// Opt-in: raw pane text can carry secrets past the regex redaction.
|
|
2098
2905
|
if (this.config.mcp_proxy_reply !== true)
|
|
@@ -2102,6 +2909,11 @@ export class Daemon extends EventEmitter {
|
|
|
2102
2909
|
return;
|
|
2103
2910
|
void this.sendProxyReply(pane, inboundMarker, correlationId);
|
|
2104
2911
|
}
|
|
2912
|
+
isClaudeCodeBackend() {
|
|
2913
|
+
return this.runtimeIdentity?.backend === "claude-code"
|
|
2914
|
+
|| this.config.backend === "claude-code"
|
|
2915
|
+
|| this.backend?.binaryName === "claude";
|
|
2916
|
+
}
|
|
2105
2917
|
/** Relay the pane's final text to the channel, marked as a daemon proxy reply. */
|
|
2106
2918
|
async sendProxyReply(pane, inboundMarker, correlationId) {
|
|
2107
2919
|
try {
|
|
@@ -2119,55 +2931,92 @@ export class Daemon extends EventEmitter {
|
|
|
2119
2931
|
let body = `⚠️ [MCP unavailable — proxy reply]\n\n${text}`;
|
|
2120
2932
|
if (correlationId)
|
|
2121
2933
|
body += `\n\n(correlation_id: ${correlationId})`;
|
|
2122
|
-
const args = { text: body };
|
|
2123
|
-
// Same context-bound routing as the reply tool in handleToolCall.
|
|
2124
|
-
if (this.lastChatId) {
|
|
2125
|
-
args.chat_id = this.lastChatId;
|
|
2126
|
-
if (this.lastThreadId)
|
|
2127
|
-
args.thread_id = this.lastThreadId;
|
|
2128
|
-
}
|
|
2129
2934
|
this.logger.warn({ correlationId }, "MCP server dead and the turn sent no reply — relaying the pane text as a proxy reply");
|
|
2130
2935
|
this.emit("mcp_proxy_reply", { name: this.name, correlationId });
|
|
2131
|
-
|
|
2132
|
-
if (adapters.length > 0) {
|
|
2133
|
-
routeToolCall(adapters[0], "reply", args, this.lastThreadId, (_result, error) => {
|
|
2134
|
-
if (error)
|
|
2135
|
-
this.logger.error({ error }, "Dead-MCP proxy reply failed");
|
|
2136
|
-
});
|
|
2137
|
-
return;
|
|
2138
|
-
}
|
|
2139
|
-
if (!this.ipcServer)
|
|
2140
|
-
return;
|
|
2141
|
-
const fleetReqId = `proxyreply_${++this.proxyReplySeq}`;
|
|
2142
|
-
this.ipcServer.broadcast({
|
|
2143
|
-
type: "fleet_outbound",
|
|
2144
|
-
tool: "reply",
|
|
2145
|
-
args,
|
|
2146
|
-
fleetRequestId: fleetReqId,
|
|
2147
|
-
adapterId: this.lastAdapterId,
|
|
2148
|
-
});
|
|
2149
|
-
const timeout = setTimeout(() => {
|
|
2150
|
-
this.pendingIpcRequests.delete(fleetReqId);
|
|
2151
|
-
this.logger.error("Dead-MCP proxy reply timed out waiting for the fleet manager");
|
|
2152
|
-
}, daemonBudgetMs("reply"));
|
|
2153
|
-
timeout.unref?.();
|
|
2154
|
-
this.pendingIpcRequests.set(fleetReqId, (respMsg) => {
|
|
2155
|
-
clearTimeout(timeout);
|
|
2156
|
-
if (respMsg.error)
|
|
2157
|
-
this.logger.error({ error: respMsg.error }, "Dead-MCP proxy reply failed");
|
|
2158
|
-
else
|
|
2159
|
-
this.logger.info("Dead-MCP proxy reply delivered");
|
|
2160
|
-
});
|
|
2936
|
+
this.deliverDaemonReply(body, "proxyreply", "Dead-MCP proxy reply");
|
|
2161
2937
|
}
|
|
2162
2938
|
catch (err) {
|
|
2163
2939
|
this.logger.error({ err: err.message }, "Dead-MCP proxy reply attempt failed");
|
|
2164
2940
|
}
|
|
2165
2941
|
}
|
|
2942
|
+
/** Send a narrowly extracted #648 reply; operator notification is a separate event. */
|
|
2943
|
+
sendRecoveredMalformedReply(text, correlationId) {
|
|
2944
|
+
try {
|
|
2945
|
+
this.deliverDaemonReply(text, "malformedreply", "Malformed tool-call recovery");
|
|
2946
|
+
}
|
|
2947
|
+
catch (err) {
|
|
2948
|
+
this.logger.error({ err: err.message, correlationId }, "Malformed tool-call recovery attempt failed");
|
|
2949
|
+
}
|
|
2950
|
+
}
|
|
2951
|
+
/** Context-bound reply path shared by dead-MCP and malformed-call recovery. */
|
|
2952
|
+
deliverDaemonReply(body, requestPrefix, logLabel) {
|
|
2953
|
+
const args = { text: body };
|
|
2954
|
+
if (this.lastChatId) {
|
|
2955
|
+
args.chat_id = this.lastChatId;
|
|
2956
|
+
if (this.lastThreadId)
|
|
2957
|
+
args.thread_id = this.lastThreadId;
|
|
2958
|
+
}
|
|
2959
|
+
const adapters = this.messageBus.getAllAdapters();
|
|
2960
|
+
if (adapters.length > 0) {
|
|
2961
|
+
routeToolCall(adapters[0], "reply", args, this.lastThreadId, (_result, error) => {
|
|
2962
|
+
if (error)
|
|
2963
|
+
this.logger.error({ error }, `${logLabel} failed`);
|
|
2964
|
+
else
|
|
2965
|
+
this.logger.info(`${logLabel} delivered`);
|
|
2966
|
+
});
|
|
2967
|
+
return;
|
|
2968
|
+
}
|
|
2969
|
+
if (!this.ipcServer) {
|
|
2970
|
+
this.logger.error(`${logLabel} failed — no adapter or fleet IPC route`);
|
|
2971
|
+
return;
|
|
2972
|
+
}
|
|
2973
|
+
const fleetReqId = `${requestPrefix}_${++this.proxyReplySeq}`;
|
|
2974
|
+
const timeout = setTimeout(() => {
|
|
2975
|
+
this.pendingIpcRequests.delete(fleetReqId);
|
|
2976
|
+
this.logger.error(`${logLabel} timed out waiting for the fleet manager`);
|
|
2977
|
+
}, daemonBudgetMs("reply"));
|
|
2978
|
+
timeout.unref?.();
|
|
2979
|
+
// Register before broadcast: an in-process/fast adapter can answer in the
|
|
2980
|
+
// same tick, and a waiter installed afterwards would miss that response.
|
|
2981
|
+
this.pendingIpcRequests.set(fleetReqId, (respMsg) => {
|
|
2982
|
+
clearTimeout(timeout);
|
|
2983
|
+
if (respMsg.error)
|
|
2984
|
+
this.logger.error({ error: respMsg.error }, `${logLabel} failed`);
|
|
2985
|
+
else
|
|
2986
|
+
this.logger.info(`${logLabel} delivered`);
|
|
2987
|
+
});
|
|
2988
|
+
this.ipcServer.broadcast({
|
|
2989
|
+
type: "fleet_outbound",
|
|
2990
|
+
tool: "reply",
|
|
2991
|
+
args,
|
|
2992
|
+
fleetRequestId: fleetReqId,
|
|
2993
|
+
adapterId: this.lastAdapterId,
|
|
2994
|
+
});
|
|
2995
|
+
}
|
|
2166
2996
|
clearInstanceStateIdleTimer() {
|
|
2167
2997
|
if (this.instanceStateIdleTimer)
|
|
2168
2998
|
clearTimeout(this.instanceStateIdleTimer);
|
|
2169
2999
|
this.instanceStateIdleTimer = null;
|
|
2170
3000
|
}
|
|
3001
|
+
clearInstanceStateOutputProbeTimer() {
|
|
3002
|
+
if (this.instanceStateOutputProbeTimer)
|
|
3003
|
+
clearTimeout(this.instanceStateOutputProbeTimer);
|
|
3004
|
+
this.instanceStateOutputProbeTimer = null;
|
|
3005
|
+
}
|
|
3006
|
+
/**
|
|
3007
|
+
* Periodic status-line hooks can generate `%output` without changing a single
|
|
3008
|
+
* visible cell. Probe the pane once per burst: a changed capture is real work;
|
|
3009
|
+
* an identical ready capture remains idle and emits no state edge.
|
|
3010
|
+
*/
|
|
3011
|
+
scheduleInstanceStateOutputProbe() {
|
|
3012
|
+
if (this.instanceStateOutputProbeTimer)
|
|
3013
|
+
return;
|
|
3014
|
+
this.instanceStateOutputProbeTimer = setTimeout(() => {
|
|
3015
|
+
this.instanceStateOutputProbeTimer = null;
|
|
3016
|
+
void this.captureAndEvaluateInstanceState("output_probe", this.instanceStateLastOutputAt);
|
|
3017
|
+
}, PERIODIC_REDRAW_PROBE_MS);
|
|
3018
|
+
this.instanceStateOutputProbeTimer.unref?.();
|
|
3019
|
+
}
|
|
2171
3020
|
clearInstanceStateStuckTimer() {
|
|
2172
3021
|
if (this.instanceStateStuckTimer)
|
|
2173
3022
|
clearTimeout(this.instanceStateStuckTimer);
|
|
@@ -2212,6 +3061,8 @@ export class Daemon extends EventEmitter {
|
|
|
2212
3061
|
if (this.statePollInFlight) {
|
|
2213
3062
|
if (reason === "idle_debounce")
|
|
2214
3063
|
this.scheduleInstanceStateIdleCapture();
|
|
3064
|
+
if (reason === "output_probe")
|
|
3065
|
+
this.scheduleInstanceStateOutputProbe();
|
|
2215
3066
|
return;
|
|
2216
3067
|
}
|
|
2217
3068
|
this.statePollInFlight = true;
|
|
@@ -2220,9 +3071,19 @@ export class Daemon extends EventEmitter {
|
|
|
2220
3071
|
const pane = await this.tmux.capturePane();
|
|
2221
3072
|
// Output received while capture-pane was in flight makes this snapshot
|
|
2222
3073
|
// stale. Its output handler has already armed a new debounce.
|
|
2223
|
-
|
|
2224
|
-
|| (this.instanceStateLastOutputAt > 0 && this.instanceStateLastOutputAt >= captureStartedAt)
|
|
3074
|
+
const outputMovedDuringCapture = (expectedOutputAt > 0 && this.instanceStateLastOutputAt > expectedOutputAt)
|
|
3075
|
+
|| (this.instanceStateLastOutputAt > 0 && this.instanceStateLastOutputAt >= captureStartedAt);
|
|
3076
|
+
if (outputMovedDuringCapture) {
|
|
3077
|
+
if (reason === "output_probe") {
|
|
3078
|
+
// A second burst overtook the probe. Do not bless its stale capture as
|
|
3079
|
+
// idle: continuous output is the safer working signal, while the next
|
|
3080
|
+
// quiet probe/debounce can still prove that it was only repainting.
|
|
3081
|
+
this.applyInstanceStateSnapshot(this.instanceStateMachine.recordOutput(this.instanceStateLastOutputAt));
|
|
3082
|
+
this.scheduleInstanceStateIdleCapture();
|
|
3083
|
+
this.scheduleInstanceStateStuckDeadline(this.instanceStateLastOutputAt);
|
|
3084
|
+
}
|
|
2225
3085
|
return;
|
|
3086
|
+
}
|
|
2226
3087
|
const observedChangeAt = expectedOutputAt || captureStartedAt;
|
|
2227
3088
|
// One observation per capture. This used to call observe() twice with the
|
|
2228
3089
|
// same pane: the second call always saw an unchanged pane, so the
|
|
@@ -2231,6 +3092,7 @@ export class Daemon extends EventEmitter {
|
|
|
2231
3092
|
// mismatched spinner frame was enough to report idle mid-turn.
|
|
2232
3093
|
const settled = this.instanceStateLastOutputAt === 0
|
|
2233
3094
|
|| captureStartedAt - this.instanceStateLastOutputAt >= this.instanceStateIdleDebounceMs;
|
|
3095
|
+
const paneActivity = this.backend?.getPaneActivity?.(pane) ?? null;
|
|
2234
3096
|
// The two times are genuinely different and both matter: the content
|
|
2235
3097
|
// changed when tmux reported output (observedChangeAt), but idle/stuck are
|
|
2236
3098
|
// decisions about *now*. The old double-observe expressed that by calling
|
|
@@ -2238,6 +3100,7 @@ export class Daemon extends EventEmitter {
|
|
|
2238
3100
|
const snapshot = this.instanceStateMachine.observe(pane, Date.now(), {
|
|
2239
3101
|
settled,
|
|
2240
3102
|
changeAt: observedChangeAt,
|
|
3103
|
+
forceBusy: paneActivity !== null,
|
|
2241
3104
|
});
|
|
2242
3105
|
this.applyInstanceStateSnapshot(snapshot, pane);
|
|
2243
3106
|
// Backends without a transcript feed can still say what they are doing, if
|
|
@@ -2245,7 +3108,7 @@ export class Daemon extends EventEmitter {
|
|
|
2245
3108
|
// extra tmux call. Cadence is the capture cadence (idle debounce + the 60s
|
|
2246
3109
|
// safety sweep), which matches the progress ticker's own 60s interval.
|
|
2247
3110
|
if (this.backend?.getPaneActivity) {
|
|
2248
|
-
this.publishActivity(snapshot.state === "idle" ? null :
|
|
3111
|
+
this.publishActivity(snapshot.state === "idle" ? null : paneActivity);
|
|
2249
3112
|
}
|
|
2250
3113
|
if (snapshot.state === "idle") {
|
|
2251
3114
|
this.clearInstanceStateStuckTimer();
|
|
@@ -2272,6 +3135,14 @@ export class Daemon extends EventEmitter {
|
|
|
2272
3135
|
if (!windowId || event.windowId !== windowId || !this.instanceStateMachine)
|
|
2273
3136
|
return;
|
|
2274
3137
|
this.instanceStateLastOutputAt = event.at;
|
|
3138
|
+
if (this.backend?.hasPeriodicPaneRedraw?.() === true && this.instanceState === "idle") {
|
|
3139
|
+
// Do not emit idle→working solely because agy repainted an identical
|
|
3140
|
+
// footer. The short capture probe below still observes genuine pane
|
|
3141
|
+
// changes promptly and moves the state to working.
|
|
3142
|
+
this.scheduleInstanceStateOutputProbe();
|
|
3143
|
+
this.scheduleInstanceStateIdleCapture();
|
|
3144
|
+
return;
|
|
3145
|
+
}
|
|
2275
3146
|
const snapshot = this.instanceStateMachine.recordOutput(event.at);
|
|
2276
3147
|
this.applyInstanceStateSnapshot(snapshot);
|
|
2277
3148
|
this.scheduleInstanceStateIdleCapture();
|
|
@@ -2322,6 +3193,7 @@ export class Daemon extends EventEmitter {
|
|
|
2322
3193
|
stopInstanceStateMonitor() {
|
|
2323
3194
|
this.instanceStateMonitorActive = false;
|
|
2324
3195
|
this.clearInstanceStateIdleTimer();
|
|
3196
|
+
this.clearInstanceStateOutputProbeTimer();
|
|
2325
3197
|
this.clearInstanceStateStuckTimer();
|
|
2326
3198
|
if (this.instanceStateMonitorTimer)
|
|
2327
3199
|
clearInterval(this.instanceStateMonitorTimer);
|
|
@@ -2356,6 +3228,15 @@ export class Daemon extends EventEmitter {
|
|
|
2356
3228
|
this.logger.debug(diagnostic, "Suppressing stuck notification without pending work");
|
|
2357
3229
|
return;
|
|
2358
3230
|
}
|
|
3231
|
+
// An auth-broken instance IS stuck, and stays stuck: it cannot finish the
|
|
3232
|
+
// turn it is holding, so pendingWork never clears and this fired on every
|
|
3233
|
+
// scan forever. The notification is also useless — it offers Restart/Wait
|
|
3234
|
+
// and asks General for help, but only the user re-logging in fixes this, and
|
|
3235
|
+
// they have already been told exactly that by the auth alert.
|
|
3236
|
+
if (this.authFailureUnresolved) {
|
|
3237
|
+
this.logger.debug(diagnostic, "Suppressing stuck notification — unresolved auth failure (re-login required)");
|
|
3238
|
+
return;
|
|
3239
|
+
}
|
|
2359
3240
|
this.logger.warn(diagnostic, "Instance pane stuck with pending work");
|
|
2360
3241
|
this.hangDetector?.emit("hang", { unchangedForMs: snapshot.unchangedForMs });
|
|
2361
3242
|
}
|
|
@@ -2375,6 +3256,7 @@ export class Daemon extends EventEmitter {
|
|
|
2375
3256
|
// fire into a stopped daemon).
|
|
2376
3257
|
this.clearMcpRestartRequest();
|
|
2377
3258
|
this.interactivePromptDetector.reset();
|
|
3259
|
+
this.blockingProcessDetector.reset();
|
|
2378
3260
|
this.stopInstanceStateMonitor();
|
|
2379
3261
|
this.transcriptMonitor?.stop();
|
|
2380
3262
|
this.guardian?.stop();
|
|
@@ -2400,9 +3282,9 @@ export class Daemon extends EventEmitter {
|
|
|
2400
3282
|
* Tell the fleet manager what this instance is doing right now, for the live
|
|
2401
3283
|
* progress line on the cancel button.
|
|
2402
3284
|
*
|
|
2403
|
-
*
|
|
2404
|
-
*
|
|
2405
|
-
*
|
|
3285
|
+
* Primarily cosmetic. The foreground-process detector also uses a current
|
|
3286
|
+
* shell activity as one of several positive signals, but never makes a state
|
|
3287
|
+
* decision from an arbitrary activity label alone.
|
|
2406
3288
|
*
|
|
2407
3289
|
* Repeats are dropped: the ticker only edits the channel message when the text
|
|
2408
3290
|
* changes, and a stream of identical broadcasts would defeat that.
|
|
@@ -2418,6 +3300,141 @@ export class Daemon extends EventEmitter {
|
|
|
2418
3300
|
activity: next,
|
|
2419
3301
|
});
|
|
2420
3302
|
}
|
|
3303
|
+
/** Effective tool_progress level, hardened against junk config values.
|
|
3304
|
+
* Off unless explicitly enabled: an update must never make bubbles start
|
|
3305
|
+
* listing tool activity the operator did not ask to broadcast. */
|
|
3306
|
+
toolProgressLevel() {
|
|
3307
|
+
const raw = this.config.tool_progress;
|
|
3308
|
+
return raw === "standard" || raw === "verbose" ? raw : "off";
|
|
3309
|
+
}
|
|
3310
|
+
/**
|
|
3311
|
+
* Apply a tool-progress setting without restarting the CLI session.
|
|
3312
|
+
*
|
|
3313
|
+
* Drop the current accumulator at the boundary: verbose entries may contain
|
|
3314
|
+
* command previews which must not survive a downgrade to standard/off, and a
|
|
3315
|
+
* newly enabled level should start with a clean, internally consistent list.
|
|
3316
|
+
*/
|
|
3317
|
+
updateToolProgress(level) {
|
|
3318
|
+
const previous = this.toolProgressLevel();
|
|
3319
|
+
this.config.tool_progress = level;
|
|
3320
|
+
if (this.toolProgressLevel() !== previous)
|
|
3321
|
+
this.resetToolProgress();
|
|
3322
|
+
}
|
|
3323
|
+
/** Snapshot the configuration actually owned by this live daemon. */
|
|
3324
|
+
getConfigSnapshot() {
|
|
3325
|
+
return structuredClone(this.config);
|
|
3326
|
+
}
|
|
3327
|
+
/**
|
|
3328
|
+
* Apply the whitelisted hot configuration carried by FleetManager IPC.
|
|
3329
|
+
* null removes an optional value; invalid/unlisted fields are ignored rather
|
|
3330
|
+
* than allowing another client on the per-instance socket to mutate cold
|
|
3331
|
+
* process/session settings.
|
|
3332
|
+
*/
|
|
3333
|
+
applyConfigUpdate(value) {
|
|
3334
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
3335
|
+
return;
|
|
3336
|
+
const update = value;
|
|
3337
|
+
if (update.tool_progress === null || ["off", "standard", "verbose"].includes(String(update.tool_progress))) {
|
|
3338
|
+
this.updateToolProgress(update.tool_progress === null ? undefined : update.tool_progress);
|
|
3339
|
+
}
|
|
3340
|
+
if (typeof update.mcp_proxy_reply === "boolean")
|
|
3341
|
+
this.config.mcp_proxy_reply = update.mcp_proxy_reply;
|
|
3342
|
+
else if (update.mcp_proxy_reply === null)
|
|
3343
|
+
delete this.config.mcp_proxy_reply;
|
|
3344
|
+
if (typeof update.warm_cap === "number" && Number.isInteger(update.warm_cap) && update.warm_cap >= 0)
|
|
3345
|
+
this.config.warm_cap = update.warm_cap;
|
|
3346
|
+
else if (update.warm_cap === null)
|
|
3347
|
+
delete this.config.warm_cap;
|
|
3348
|
+
const nextAutoPause = typeof update.auto_pause_after === "number"
|
|
3349
|
+
&& Number.isFinite(update.auto_pause_after)
|
|
3350
|
+
&& update.auto_pause_after >= 0
|
|
3351
|
+
? update.auto_pause_after
|
|
3352
|
+
: update.auto_pause_after === null ? 0 : undefined;
|
|
3353
|
+
if (nextAutoPause !== undefined) {
|
|
3354
|
+
const previousAutoPause = typeof this.config.auto_pause_after === "number" ? this.config.auto_pause_after : 0;
|
|
3355
|
+
if (update.auto_pause_after === null)
|
|
3356
|
+
delete this.config.auto_pause_after;
|
|
3357
|
+
else
|
|
3358
|
+
this.config.auto_pause_after = nextAutoPause;
|
|
3359
|
+
if (nextAutoPause !== previousAutoPause) {
|
|
3360
|
+
const isGeneral = this.config.general_topic === true || this.name === "general";
|
|
3361
|
+
this.autoPauseController.reconfigure((isGeneral ? 0 : nextAutoPause) * 60_000);
|
|
3362
|
+
}
|
|
3363
|
+
}
|
|
3364
|
+
for (const key of ["display_name", "description"]) {
|
|
3365
|
+
if (typeof update[key] === "string")
|
|
3366
|
+
this.config[key] = update[key];
|
|
3367
|
+
else if (update[key] === null)
|
|
3368
|
+
delete this.config[key];
|
|
3369
|
+
}
|
|
3370
|
+
if (Array.isArray(update.tags) && update.tags.every(tag => typeof tag === "string"))
|
|
3371
|
+
this.config.tags = [...update.tags];
|
|
3372
|
+
else if (update.tags === null)
|
|
3373
|
+
delete this.config.tags;
|
|
3374
|
+
if (["trace", "debug", "info", "warn", "error"].includes(String(update.log_level))) {
|
|
3375
|
+
const level = update.log_level;
|
|
3376
|
+
if (this.config.log_level !== level) {
|
|
3377
|
+
this.config.log_level = level;
|
|
3378
|
+
this.logger.level = level;
|
|
3379
|
+
}
|
|
3380
|
+
}
|
|
3381
|
+
}
|
|
3382
|
+
/**
|
|
3383
|
+
* Accumulate one semantic progress line for the channel bubble and schedule
|
|
3384
|
+
* a coalesced broadcast. Separate from publishActivity on purpose: that one
|
|
3385
|
+
* is the single-line statusline detail (terse, operator-facing), this is the
|
|
3386
|
+
* multi-line channel list (semantic, argument-free at `standard`) — see
|
|
3387
|
+
* tool-progress.ts for why the two labellers must not merge.
|
|
3388
|
+
*/
|
|
3389
|
+
recordToolProgress(name, input) {
|
|
3390
|
+
const level = this.toolProgressLevel();
|
|
3391
|
+
if (level === "off")
|
|
3392
|
+
return;
|
|
3393
|
+
if (this.turnProgress.add(summarizeProgress(name, input, level))) {
|
|
3394
|
+
this.scheduleProgressBroadcast();
|
|
3395
|
+
}
|
|
3396
|
+
}
|
|
3397
|
+
/** Coalesce progress broadcasts: at most one per 3s, trailing edge kept. */
|
|
3398
|
+
scheduleProgressBroadcast() {
|
|
3399
|
+
const MIN_INTERVAL_MS = 3_000;
|
|
3400
|
+
const since = Date.now() - this.lastProgressBroadcastAt;
|
|
3401
|
+
if (since >= MIN_INTERVAL_MS) {
|
|
3402
|
+
this.broadcastToolProgress();
|
|
3403
|
+
return;
|
|
3404
|
+
}
|
|
3405
|
+
if (this.progressBroadcastTimer)
|
|
3406
|
+
return;
|
|
3407
|
+
this.progressBroadcastTimer = setTimeout(() => {
|
|
3408
|
+
this.progressBroadcastTimer = null;
|
|
3409
|
+
this.broadcastToolProgress();
|
|
3410
|
+
}, MIN_INTERVAL_MS - since);
|
|
3411
|
+
this.progressBroadcastTimer.unref?.();
|
|
3412
|
+
}
|
|
3413
|
+
broadcastToolProgress() {
|
|
3414
|
+
const rendered = this.turnProgress.render();
|
|
3415
|
+
if (rendered === this.lastProgressBroadcast)
|
|
3416
|
+
return;
|
|
3417
|
+
this.lastProgressBroadcast = rendered;
|
|
3418
|
+
this.lastProgressBroadcastAt = Date.now();
|
|
3419
|
+
this.ipcServer?.broadcast({
|
|
3420
|
+
type: "instance_progress",
|
|
3421
|
+
instanceName: this.name,
|
|
3422
|
+
progress: rendered,
|
|
3423
|
+
});
|
|
3424
|
+
}
|
|
3425
|
+
/** New turn (or turn over): drop the list and tell the fleet to clear it. */
|
|
3426
|
+
resetToolProgress() {
|
|
3427
|
+
if (this.progressBroadcastTimer) {
|
|
3428
|
+
clearTimeout(this.progressBroadcastTimer);
|
|
3429
|
+
this.progressBroadcastTimer = null;
|
|
3430
|
+
}
|
|
3431
|
+
this.turnProgress.reset();
|
|
3432
|
+
if (this.lastProgressBroadcast !== "") {
|
|
3433
|
+
this.lastProgressBroadcast = "";
|
|
3434
|
+
this.lastProgressBroadcastAt = Date.now();
|
|
3435
|
+
this.ipcServer?.broadcast({ type: "instance_progress", instanceName: this.name, progress: "" });
|
|
3436
|
+
}
|
|
3437
|
+
}
|
|
2421
3438
|
/**
|
|
2422
3439
|
* Options for every system-initiated paste (startup notice, session snapshot,
|
|
2423
3440
|
* runtime-dialog keys).
|
|
@@ -2453,12 +3470,148 @@ export class Daemon extends EventEmitter {
|
|
|
2453
3470
|
return ""; // skip channel tools
|
|
2454
3471
|
return name;
|
|
2455
3472
|
}
|
|
3473
|
+
/**
|
|
3474
|
+
* The one place an inbound message grows its metadata wrapper ([user:]/
|
|
3475
|
+
* [from:] prefix, pending reactions, handoff metadata, reply instructions).
|
|
3476
|
+
* Both the normal queued path (pushChannelMessage) and /steer go through
|
|
3477
|
+
* here — a steered message must read EXACTLY like a queued one to the
|
|
3478
|
+
* agent, or the two drift apart in what the agent is told about replying.
|
|
3479
|
+
*/
|
|
3480
|
+
formatInboundMessage(content, meta) {
|
|
3481
|
+
const user = meta.user || "unknown";
|
|
3482
|
+
const fromInstance = meta.from_instance;
|
|
3483
|
+
let formatted;
|
|
3484
|
+
if (fromInstance) {
|
|
3485
|
+
formatted = formatCrossInstanceInboundMessage(content, meta);
|
|
3486
|
+
}
|
|
3487
|
+
else {
|
|
3488
|
+
const via = meta.source ? ` via ${meta.source}` : "";
|
|
3489
|
+
const idTag = meta.user_id ? `, id:${meta.user_id}` : "";
|
|
3490
|
+
formatted = `[user:${user}${via}${idTag}] ${content}`;
|
|
3491
|
+
// Reactions queued since the last real message (#432). One leading line of
|
|
3492
|
+
// context, present only when something is pending — a reaction no longer
|
|
3493
|
+
// costs a turn of its own.
|
|
3494
|
+
if (meta.pending_reactions) {
|
|
3495
|
+
formatted = `[Recent reactions: ${meta.pending_reactions}]\n${formatted}`;
|
|
3496
|
+
}
|
|
3497
|
+
formatted += renderHandoffMetadata(meta);
|
|
3498
|
+
formatted += "\n(Reply using the reply tool — do NOT respond with direct text)";
|
|
3499
|
+
}
|
|
3500
|
+
if (meta.reply_to_text) {
|
|
3501
|
+
formatted += `\n(reply_to: "${meta.reply_to_text}")`;
|
|
3502
|
+
}
|
|
3503
|
+
return formatted;
|
|
3504
|
+
}
|
|
3505
|
+
/** Tell the fleet when an already-accepted cross-instance pane write failed. */
|
|
3506
|
+
reportCrossInstanceDeliveryFailure(meta, error) {
|
|
3507
|
+
if (!meta.from_instance)
|
|
3508
|
+
return;
|
|
3509
|
+
this.ipcServer?.broadcast({
|
|
3510
|
+
type: "cross_instance_delivery_failed",
|
|
3511
|
+
senderSession: meta.from_instance,
|
|
3512
|
+
targetInstance: this.name,
|
|
3513
|
+
correlationId: meta.correlation_id ?? "unknown",
|
|
3514
|
+
error: error ?? this.tmux?.getLastPasteError?.() ?? "target pane rejected the delivery",
|
|
3515
|
+
});
|
|
3516
|
+
}
|
|
3517
|
+
/**
|
|
3518
|
+
* /steer: interject into the CURRENT turn instead of queueing for idle.
|
|
3519
|
+
*
|
|
3520
|
+
* Differences from pushChannelMessage, and nothing else:
|
|
3521
|
+
* - serialized on steerLock, not pasteLock — it may overtake current work,
|
|
3522
|
+
* but never an ordinary delivery already queued ahead of it
|
|
3523
|
+
* - deliverMessage runs with { steer: true }, which takes the busy branch
|
|
3524
|
+
* that pastes immediately (the codex native-queue transaction) instead of
|
|
3525
|
+
* waiting for idle. Verification and silent-loss fallback are the ones
|
|
3526
|
+
* that path already has: pane-capture visibility check, then one
|
|
3527
|
+
* idle-gated redelivery — so on a TUI that swallows busy input the steer
|
|
3528
|
+
* degrades to "next message after this turn", never silently vanishes.
|
|
3529
|
+
*
|
|
3530
|
+
* The steered text goes through formatInboundMessage so the agent sees a
|
|
3531
|
+
* normal inbound message, with a steering notice prepended for context.
|
|
3532
|
+
*/
|
|
3533
|
+
steerMessage(content, meta, deliveryEpoch = this.deliveryEpoch) {
|
|
3534
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
3535
|
+
return;
|
|
3536
|
+
// The fleet facade prevents a steer from overtaking an item still in its
|
|
3537
|
+
// idle queue. Close the second race window here: an earlier item may already
|
|
3538
|
+
// have reached this daemon's paste queue by the time the steer IPC arrives.
|
|
3539
|
+
if (this.pasteQueueDepth > 0) {
|
|
3540
|
+
this.logger.info("Steer queued behind an earlier pane delivery to preserve message order");
|
|
3541
|
+
this.pushChannelMessage(content, meta, undefined, deliveryEpoch);
|
|
3542
|
+
return;
|
|
3543
|
+
}
|
|
3544
|
+
this.updateLastChat(meta.chat_id, meta.thread_id, meta.adapter_id);
|
|
3545
|
+
this.pendingWork.recordInbound();
|
|
3546
|
+
this.recordRecentUserMessage(content, meta);
|
|
3547
|
+
const formatted = "[STEERING — mid-task course correction. Fold this into the CURRENT work if one is active.]\n"
|
|
3548
|
+
+ this.formatInboundMessage(content, meta);
|
|
3549
|
+
const chatId = meta.chat_id;
|
|
3550
|
+
const messageId = meta.message_id;
|
|
3551
|
+
const status = (chatId && messageId)
|
|
3552
|
+
? { chatId: meta.thread_id || chatId, messageId }
|
|
3553
|
+
: undefined;
|
|
3554
|
+
this.steerLock = this.steerLock.then(async () => {
|
|
3555
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
3556
|
+
return;
|
|
3557
|
+
await this.wake();
|
|
3558
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
3559
|
+
return;
|
|
3560
|
+
if (await this.deliverMessage(formatted, status, { steer: true, deliveryEpoch })) {
|
|
3561
|
+
this.markTurnStarted(meta, formatted);
|
|
3562
|
+
}
|
|
3563
|
+
else if (this.isDeliveryEpochCurrent(deliveryEpoch)) {
|
|
3564
|
+
this.reportCrossInstanceDeliveryFailure(meta);
|
|
3565
|
+
}
|
|
3566
|
+
}).catch(err => {
|
|
3567
|
+
this.logger.warn({ err: err.message }, "steer delivery error");
|
|
3568
|
+
if (this.isDeliveryEpochCurrent(deliveryEpoch)) {
|
|
3569
|
+
this.reportCrossInstanceDeliveryFailure(meta, err.message);
|
|
3570
|
+
}
|
|
3571
|
+
});
|
|
3572
|
+
}
|
|
3573
|
+
/**
|
|
3574
|
+
* /btw: submit a separately-labelled side question immediately while asking
|
|
3575
|
+
* the agent not to change the active task. Claude's native `/btw` opens a
|
|
3576
|
+
* branchAndResume panel whose answer never reaches the channel or transcript,
|
|
3577
|
+
* so channel delivery intentionally uses a normal wrapped inbound instead.
|
|
3578
|
+
*/
|
|
3579
|
+
btwMessage(content, meta, deliveryEpoch = this.deliveryEpoch) {
|
|
3580
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
3581
|
+
return;
|
|
3582
|
+
this.updateLastChat(meta.chat_id, meta.thread_id, meta.adapter_id);
|
|
3583
|
+
this.pendingWork.recordInbound();
|
|
3584
|
+
this.recordRecentUserMessage(content, meta);
|
|
3585
|
+
const formatted = "[BTW — side question from the user. Answer it separately without changing or interrupting the CURRENT work.]\n"
|
|
3586
|
+
+ this.formatInboundMessage(content, meta);
|
|
3587
|
+
const chatId = meta.chat_id;
|
|
3588
|
+
const messageId = meta.message_id;
|
|
3589
|
+
const status = (chatId && messageId)
|
|
3590
|
+
? { chatId: meta.thread_id || chatId, messageId }
|
|
3591
|
+
: undefined;
|
|
3592
|
+
this.steerLock = this.steerLock.then(async () => {
|
|
3593
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
3594
|
+
return;
|
|
3595
|
+
await this.wake();
|
|
3596
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
3597
|
+
return;
|
|
3598
|
+
if (await this.deliverMessage(formatted, status, { steer: true, deliveryEpoch })) {
|
|
3599
|
+
this.markTurnStarted(meta, formatted);
|
|
3600
|
+
}
|
|
3601
|
+
}).catch(err => {
|
|
3602
|
+
this.logger.warn({ err: err.message }, "btw delivery error");
|
|
3603
|
+
});
|
|
3604
|
+
}
|
|
2456
3605
|
/**
|
|
2457
3606
|
* Push an inbound channel message to a specific MCP session.
|
|
2458
3607
|
* If targetSession is provided, only send to the matching socket.
|
|
2459
3608
|
* Otherwise send to the instance's own session (this.name).
|
|
2460
3609
|
*/
|
|
2461
|
-
pushChannelMessage(content, meta, _targetSession) {
|
|
3610
|
+
pushChannelMessage(content, meta, _targetSession, deliveryEpoch = this.deliveryEpoch) {
|
|
3611
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch)) {
|
|
3612
|
+
this.logger.info("Pending channel delivery dropped by user cancel");
|
|
3613
|
+
return;
|
|
3614
|
+
}
|
|
2462
3615
|
if (!this.tmux) {
|
|
2463
3616
|
this.logger.warn("Cannot push channel message: tmux not running");
|
|
2464
3617
|
return;
|
|
@@ -2492,46 +3645,19 @@ export class Daemon extends EventEmitter {
|
|
|
2492
3645
|
const rawText = content.slice(5);
|
|
2493
3646
|
this.logger.info({ user }, "Raw paste from topic mode user");
|
|
2494
3647
|
this.pasteLock = this.pasteLock.then(async () => {
|
|
2495
|
-
if (
|
|
3648
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
3649
|
+
return;
|
|
3650
|
+
if (await this.deliverMessage(rawText, undefined, { deliveryEpoch }))
|
|
2496
3651
|
this.markTurnStarted(meta, rawText);
|
|
2497
3652
|
}).catch(err => {
|
|
2498
3653
|
this.logger.warn({ err: err.message }, "pasteLock raw delivery error");
|
|
2499
3654
|
});
|
|
2500
3655
|
return;
|
|
2501
3656
|
}
|
|
2502
|
-
|
|
2503
|
-
if (fromInstance) {
|
|
2504
|
-
// #77: show the sender's display name for readability, keeping the machine
|
|
2505
|
-
// instance name in parens so the recipient's send_to_instance target is valid.
|
|
2506
|
-
const fromLabel = meta.from_display ? `${meta.from_display} (${fromInstance})` : fromInstance;
|
|
2507
|
-
formatted = `[from:${fromLabel}] ${content}`;
|
|
2508
|
-
formatted += renderHandoffMetadata(meta);
|
|
2509
|
-
// A delegated task that requires a reply must not read like a chatty FYI —
|
|
2510
|
-
// the "you may stay silent" line is for the latter only.
|
|
2511
|
-
formatted += meta.requires_reply === "true"
|
|
2512
|
-
? "\n(A reply IS required: use report_result with the correlation_id above — or send_to_instance. Not direct text.)"
|
|
2513
|
-
: "\n(If you need to reply, use send_to_instance tool, NOT direct text. If there is nothing to add, you may stay silent.)";
|
|
2514
|
-
}
|
|
2515
|
-
else {
|
|
2516
|
-
const via = meta.source ? ` via ${meta.source}` : "";
|
|
2517
|
-
const idTag = meta.user_id ? `, id:${meta.user_id}` : "";
|
|
2518
|
-
formatted = `[user:${user}${via}${idTag}] ${content}`;
|
|
2519
|
-
// Reactions queued since the last real message (#432). One leading line of
|
|
2520
|
-
// context, present only when something is pending — a reaction no longer
|
|
2521
|
-
// costs a turn of its own.
|
|
2522
|
-
if (meta.pending_reactions) {
|
|
2523
|
-
formatted = `[Recent reactions: ${meta.pending_reactions}]\n${formatted}`;
|
|
2524
|
-
}
|
|
2525
|
-
formatted += renderHandoffMetadata(meta);
|
|
2526
|
-
formatted += "\n(Reply using the reply tool — do NOT respond with direct text)";
|
|
2527
|
-
}
|
|
2528
|
-
if (meta.reply_to_text) {
|
|
2529
|
-
formatted += `\n(reply_to: "${meta.reply_to_text}")`;
|
|
2530
|
-
}
|
|
3657
|
+
const formatted = this.formatInboundMessage(content, meta);
|
|
2531
3658
|
// Serialize deliveries: each message waits for the previous to complete,
|
|
2532
3659
|
// and each waits for the CLI to be idle before pasting. Messages are never
|
|
2533
|
-
// dropped for age —
|
|
2534
|
-
// (the user can press Cancel to interrupt and let the queue drain sooner).
|
|
3660
|
+
// dropped for age — only an explicit user Cancel invalidates queued work.
|
|
2535
3661
|
const chatId = meta.chat_id;
|
|
2536
3662
|
const messageId = meta.message_id;
|
|
2537
3663
|
const wasQueued = this.pasteQueueDepth > 0;
|
|
@@ -2544,24 +3670,42 @@ export class Daemon extends EventEmitter {
|
|
|
2544
3670
|
}
|
|
2545
3671
|
this.pasteLock = this.pasteLock.then(async () => {
|
|
2546
3672
|
try {
|
|
3673
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch)) {
|
|
3674
|
+
this.logger.info("Pending channel delivery dropped by user cancel");
|
|
3675
|
+
return;
|
|
3676
|
+
}
|
|
2547
3677
|
if (this.config.pre_task_command) {
|
|
2548
|
-
await this.deliverMessage(this.config.pre_task_command);
|
|
3678
|
+
await this.deliverMessage(this.config.pre_task_command, undefined, { deliveryEpoch });
|
|
3679
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
3680
|
+
return;
|
|
2549
3681
|
}
|
|
2550
3682
|
if (this.pendingInstructionsNotice) {
|
|
2551
3683
|
this.pendingInstructionsNotice = false;
|
|
2552
|
-
await this.deliverMessage(buildInstructionReloadNotice(this.backend?.binaryName ?? "unknown", this.name, this.instanceDir));
|
|
3684
|
+
await this.deliverMessage(buildInstructionReloadNotice(this.backend?.binaryName ?? "unknown", this.name, this.instanceDir), undefined, { deliveryEpoch });
|
|
3685
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
3686
|
+
return;
|
|
2553
3687
|
}
|
|
2554
3688
|
const status = (chatId && messageId)
|
|
2555
3689
|
? { chatId: meta.thread_id || chatId, messageId }
|
|
2556
3690
|
: undefined;
|
|
2557
|
-
|
|
3691
|
+
// A fresh delivery begins a fresh turn — its bubble must not inherit
|
|
3692
|
+
// the previous turn's tool list.
|
|
3693
|
+
this.resetToolProgress();
|
|
3694
|
+
if (await this.deliverMessage(formatted, status, { deliveryEpoch })) {
|
|
2558
3695
|
this.markTurnStarted(meta, formatted);
|
|
3696
|
+
}
|
|
3697
|
+
else if (meta.from_instance && this.isDeliveryEpochCurrent(deliveryEpoch)) {
|
|
3698
|
+
this.reportCrossInstanceDeliveryFailure(meta);
|
|
3699
|
+
}
|
|
2559
3700
|
}
|
|
2560
3701
|
finally {
|
|
2561
3702
|
this.pasteQueueDepth--;
|
|
2562
3703
|
}
|
|
2563
3704
|
}).catch(err => {
|
|
2564
3705
|
this.logger.warn({ err: err.message }, "pasteLock delivery error — chain continues");
|
|
3706
|
+
if (this.isDeliveryEpochCurrent(deliveryEpoch)) {
|
|
3707
|
+
this.reportCrossInstanceDeliveryFailure(meta, err.message);
|
|
3708
|
+
}
|
|
2565
3709
|
});
|
|
2566
3710
|
this.logger.debug({ user: meta.user, text: content.slice(0, 100) }, "Queued channel message for delivery");
|
|
2567
3711
|
}
|
|
@@ -2581,31 +3725,67 @@ export class Daemon extends EventEmitter {
|
|
|
2581
3725
|
* immediately and own the application-level queue themselves. The pasteLock remains
|
|
2582
3726
|
* serial in both cases so separate PTY writes can never overlap.
|
|
2583
3727
|
*/
|
|
2584
|
-
async deliverMessage(formatted, status) {
|
|
3728
|
+
async deliverMessage(formatted, status, opts) {
|
|
3729
|
+
const cancelled = () => opts?.deliveryEpoch !== undefined
|
|
3730
|
+
&& !this.isDeliveryEpochCurrent(opts.deliveryEpoch);
|
|
3731
|
+
if (cancelled() || this.stormWindow?.isStopped())
|
|
3732
|
+
return false;
|
|
2585
3733
|
// Sanitize unclosed code fences — they cause CLI to wait for closure on Enter
|
|
2586
3734
|
const fenceCount = (formatted.match(/```/g) || []).length;
|
|
2587
3735
|
if (fenceCount % 2 !== 0) {
|
|
2588
3736
|
// Odd number of fences = unclosed. Remove all code fences from the message.
|
|
2589
3737
|
formatted = formatted.replace(/```/g, "");
|
|
2590
3738
|
}
|
|
3739
|
+
// Direct paths such as /steer, /btw, /raw and schedules do not pass the
|
|
3740
|
+
// FleetManager idle gate. Hold them at the final common pane-write path so
|
|
3741
|
+
// a tmux storm cannot force any text into a half-booted CLI.
|
|
3742
|
+
if (this.stormWindow?.isDeliveryHeld(this.name)) {
|
|
3743
|
+
this.logger.warn("Pane delivery held during tmux server recovery");
|
|
3744
|
+
await this.stormWindow.waitForDeliveryAllowed(this.name);
|
|
3745
|
+
if (cancelled() || this.stormWindow.isStopped())
|
|
3746
|
+
return false;
|
|
3747
|
+
}
|
|
2591
3748
|
// Before anything reads the window id: a spawn in progress is about to change it.
|
|
2592
3749
|
await this.waitForSpawnToSettle();
|
|
3750
|
+
if (cancelled())
|
|
3751
|
+
return false;
|
|
3752
|
+
if (this.refuseFatalStartupDelivery(status))
|
|
3753
|
+
return false;
|
|
2593
3754
|
let windowId = this.getWindowId();
|
|
2594
3755
|
const supportsQueuedInput = this.backend?.supportsQueuedInput?.() === true;
|
|
2595
3756
|
let handingOffToNativeQueue = false;
|
|
2596
3757
|
// If the CLI is busy, either hand the complete submission to its native input
|
|
2597
3758
|
// queue or wait for idle. Native queue support is an explicit backend capability:
|
|
2598
3759
|
// normal Enter input has steering/interrupt semantics in several other CLIs.
|
|
2599
|
-
|
|
3760
|
+
let readiness = "ready";
|
|
3761
|
+
const gateWindowId = windowId;
|
|
3762
|
+
if (gateWindowId && this.controlClient)
|
|
3763
|
+
readiness = await this.paneReadinessForDelivery(gateWindowId);
|
|
3764
|
+
if (readiness !== "ready") {
|
|
2600
3765
|
if (status)
|
|
2601
3766
|
this.emit("message_queued", status);
|
|
2602
|
-
|
|
3767
|
+
// Only a pane that is busy AND verifiably free of a blocking dialog may
|
|
3768
|
+
// be handed to the native input queue or steered — those paths paste and
|
|
3769
|
+
// press Enter at once. A dialog painted a moment ago is still "busy" to
|
|
3770
|
+
// the silence gate, so the probe is mandatory here; "dialog"/"unknown"
|
|
3771
|
+
// fall through to the wait.
|
|
3772
|
+
const canHandOff = (supportsQueuedInput || opts?.steer)
|
|
3773
|
+
&& readiness === "busy"
|
|
3774
|
+
&& (await this.probeBlockingDialog()).state === "clear";
|
|
3775
|
+
if (canHandOff) {
|
|
3776
|
+
// Native queue (codex), or an explicit /steer: hand the complete
|
|
3777
|
+
// paste+Enter transaction to the busy CLI now. For steer this is the
|
|
3778
|
+
// point — the user asked to interject, and the transaction's
|
|
3779
|
+
// visibility check + idle-gated fallback catch TUIs that swallow
|
|
3780
|
+
// busy input (see steerMessage).
|
|
2603
3781
|
handingOffToNativeQueue = true;
|
|
2604
|
-
this.logger.debug("CLI busy — handing message to backend-native input queue");
|
|
3782
|
+
this.logger.debug(opts?.steer ? "CLI busy — steering into the running turn" : "CLI busy — handing message to backend-native input queue");
|
|
2605
3783
|
}
|
|
2606
3784
|
else {
|
|
2607
3785
|
this.logger.debug("CLI busy — queuing message until idle");
|
|
2608
|
-
const becameIdle = await this.
|
|
3786
|
+
const becameIdle = await this.waitForPaneReadyForDelivery(gateWindowId);
|
|
3787
|
+
if (cancelled())
|
|
3788
|
+
return false;
|
|
2609
3789
|
if (!becameIdle) {
|
|
2610
3790
|
// The pane never freed up. Report the failure instead of pasting into a
|
|
2611
3791
|
// wedged CLI (where the text would sit unsubmitted and the next message
|
|
@@ -2617,11 +3797,459 @@ export class Daemon extends EventEmitter {
|
|
|
2617
3797
|
}
|
|
2618
3798
|
}
|
|
2619
3799
|
}
|
|
3800
|
+
// F3 (#kiro-enter-drop): an earlier delivery's Enter may have been dropped,
|
|
3801
|
+
// leaving that message in the input row. Pasting on top would submit both as
|
|
3802
|
+
// one. Submit the stranded text first (a bare Enter — a no-op on an empty
|
|
3803
|
+
// prompt, verified live), then wait for the prompt to come back before our
|
|
3804
|
+
// own paste. The wait happens OUTSIDE the pane lock like every other wait.
|
|
3805
|
+
if (windowId && this.controlClient && !handingOffToNativeQueue) {
|
|
3806
|
+
// Only a positively verified clear input row ("idle") lets the paste
|
|
3807
|
+
// proceed. After "submitted" (the stranded text's turn is running) or
|
|
3808
|
+
// "busy" (the pane changed under us, or could not be read) the prompt
|
|
3809
|
+
// must come back AND the check runs again — the stranded text may still
|
|
3810
|
+
// be there. "failed" (the Enter could not be sent) fails the delivery:
|
|
3811
|
+
// pasting now would submit both messages as one.
|
|
3812
|
+
for (let round = 0;; round++) {
|
|
3813
|
+
// Budget check BEFORE the next mutating call: each round may send one
|
|
3814
|
+
// recovery Enter, so exactly STRANDED_INPUT_MAX_ROUNDS of them go out.
|
|
3815
|
+
if (round >= STRANDED_INPUT_MAX_ROUNDS) {
|
|
3816
|
+
this.logger.error({ round }, "Input row still not clear after the stranded-text recovery budget — reporting delivery failure");
|
|
3817
|
+
if (status)
|
|
3818
|
+
this.emit("message_failed", status); // ❌
|
|
3819
|
+
return false;
|
|
3820
|
+
}
|
|
3821
|
+
const stranded = await this.submitStrandedInputIfAny(windowId);
|
|
3822
|
+
if (cancelled())
|
|
3823
|
+
return false;
|
|
3824
|
+
if (stranded === "idle")
|
|
3825
|
+
break;
|
|
3826
|
+
if (stranded === "failed") {
|
|
3827
|
+
this.logger.error({ round }, "Could not submit the stranded text — reporting delivery failure");
|
|
3828
|
+
if (status)
|
|
3829
|
+
this.emit("message_failed", status); // ❌
|
|
3830
|
+
return false;
|
|
3831
|
+
}
|
|
3832
|
+
const readyAgain = await this.waitForPaneReadyForDelivery(windowId);
|
|
3833
|
+
if (cancelled())
|
|
3834
|
+
return false;
|
|
3835
|
+
if (!readyAgain) {
|
|
3836
|
+
this.logger.error("Pane never returned to its prompt after submitting stranded input — reporting delivery failure");
|
|
3837
|
+
if (status)
|
|
3838
|
+
this.emit("message_failed", status); // ❌
|
|
3839
|
+
return false;
|
|
3840
|
+
}
|
|
3841
|
+
}
|
|
3842
|
+
}
|
|
2620
3843
|
// Everything above is *waiting*; everything below *writes*. Only the write is
|
|
2621
3844
|
// held under the pane lock — holding it across the idle wait (up to 30 min)
|
|
2622
3845
|
// would starve the runtime-dialog dismisser, which is often the very thing
|
|
2623
3846
|
// that would let the pane go idle again.
|
|
2624
|
-
|
|
3847
|
+
for (let round = 0;; round++) {
|
|
3848
|
+
const outcome = await this.paneWriteLock.run(async () => {
|
|
3849
|
+
if (cancelled())
|
|
3850
|
+
return false;
|
|
3851
|
+
if (this.refuseFatalStartupDelivery(status))
|
|
3852
|
+
return false;
|
|
3853
|
+
// TOCTOU: the probes above ran outside this lock, and the CLI repaints
|
|
3854
|
+
// whenever it likes — a resume prompt can be painted between "clear" and
|
|
3855
|
+
// this critical section. Re-probe here, right before the write; a
|
|
3856
|
+
// dialog or an unreadable pane means "not now", never "paste anyway".
|
|
3857
|
+
if (this.deliveryBlockingDialogs().length > 0) {
|
|
3858
|
+
const probe = await this.probeBlockingDialog();
|
|
3859
|
+
if (probe.state !== "clear")
|
|
3860
|
+
return "dialog";
|
|
3861
|
+
}
|
|
3862
|
+
return this.writeMessageToPane(formatted, windowId, handingOffToNativeQueue, status);
|
|
3863
|
+
});
|
|
3864
|
+
if (outcome !== "dialog")
|
|
3865
|
+
return outcome;
|
|
3866
|
+
// Wait OUTSIDE the lock (holding it would starve the runtime dismisser),
|
|
3867
|
+
// then try the write again — bounded, so a dialog nobody answers ends in
|
|
3868
|
+
// an honest failure rather than a message that never lands.
|
|
3869
|
+
if (round + 1 >= LATE_DIALOG_WRITE_ROUNDS) {
|
|
3870
|
+
this.logger.error({ rounds: round + 1 }, "A dialog kept appearing before the pane write — reporting delivery failure");
|
|
3871
|
+
if (status)
|
|
3872
|
+
this.emit("message_failed", status); // ❌
|
|
3873
|
+
return false;
|
|
3874
|
+
}
|
|
3875
|
+
this.logger.info("Dialog appeared before the pane write — waiting for it to clear");
|
|
3876
|
+
const clear = gateWindowId ? await this.waitForPaneReadyForDelivery(gateWindowId) : false;
|
|
3877
|
+
if (cancelled())
|
|
3878
|
+
return false;
|
|
3879
|
+
if (!clear) {
|
|
3880
|
+
if (status)
|
|
3881
|
+
this.emit("message_failed", status); // ❌
|
|
3882
|
+
return false;
|
|
3883
|
+
}
|
|
3884
|
+
}
|
|
3885
|
+
}
|
|
3886
|
+
/**
|
|
3887
|
+
* The runtime dialogs that hold delivery while on screen (`blocksDelivery`
|
|
3888
|
+
* or `holdOnly`). One canonical list and order, shared by the delivery gate
|
|
3889
|
+
* and the runtime scanner, so both observers agree on which dialog they see.
|
|
3890
|
+
* Startup-only patterns are deliberately excluded: several of them are loose
|
|
3891
|
+
* on purpose ("I trust", "Yes, continue") and would match ordinary prose.
|
|
3892
|
+
*/
|
|
3893
|
+
deliveryBlockingDialogs() {
|
|
3894
|
+
return (this.backend?.getRuntimeDialogs?.() ?? []).filter(d => d.blocksDelivery || d.holdOnly);
|
|
3895
|
+
}
|
|
3896
|
+
static dialogKey(dialog) {
|
|
3897
|
+
return `${dialog.pattern.source}/${dialog.pattern.flags}`;
|
|
3898
|
+
}
|
|
3899
|
+
/** A dialog counts only when it is the CURRENT interactive region, if the backend can tell; else by pattern. */
|
|
3900
|
+
static dialogMatches(dialog, pane) {
|
|
3901
|
+
return dialog.isActive ? dialog.isActive(pane) : dialog.pattern.test(pane);
|
|
3902
|
+
}
|
|
3903
|
+
/**
|
|
3904
|
+
* Tri-state liveness. tmux's own isWindowAlive folds every query failure
|
|
3905
|
+
* into `false`, so a transient tmux hiccup would read as "the CLI died" —
|
|
3906
|
+
* and callers that clear the session on death must not act on that.
|
|
3907
|
+
* getPaneStatus returns null on a failed/ambiguous query and a definite
|
|
3908
|
+
* `{alive:false}` for a dead pane; only the latter is evidence.
|
|
3909
|
+
*/
|
|
3910
|
+
async paneLiveness() {
|
|
3911
|
+
if (!this.tmux)
|
|
3912
|
+
return "unknown";
|
|
3913
|
+
const tmux = this.tmux;
|
|
3914
|
+
if (typeof tmux.getPaneStatus === "function") {
|
|
3915
|
+
try {
|
|
3916
|
+
const status = await tmux.getPaneStatus();
|
|
3917
|
+
if (status === null)
|
|
3918
|
+
return "unknown";
|
|
3919
|
+
return status.alive ? "alive" : "dead";
|
|
3920
|
+
}
|
|
3921
|
+
catch {
|
|
3922
|
+
return "unknown";
|
|
3923
|
+
}
|
|
3924
|
+
}
|
|
3925
|
+
try {
|
|
3926
|
+
return (await tmux.isWindowAlive()) ? "alive" : "dead";
|
|
3927
|
+
}
|
|
3928
|
+
catch {
|
|
3929
|
+
return "unknown";
|
|
3930
|
+
}
|
|
3931
|
+
}
|
|
3932
|
+
/**
|
|
3933
|
+
* Capture the pane and classify it. "unknown" (capture failed) is a distinct
|
|
3934
|
+
* outcome on purpose: the resume dialog is silent, so silence cannot vouch
|
|
3935
|
+
* for the pane, and treating an unreadable pane as clear is exactly the
|
|
3936
|
+
* fail-open that answers a dialog with its default. Callers retry unknown
|
|
3937
|
+
* within a bound and then fail loudly.
|
|
3938
|
+
*/
|
|
3939
|
+
async probeBlockingDialog() {
|
|
3940
|
+
if (!this.tmux || !this.backend)
|
|
3941
|
+
return { state: "clear" };
|
|
3942
|
+
const dialogs = this.deliveryBlockingDialogs();
|
|
3943
|
+
if (dialogs.length === 0)
|
|
3944
|
+
return { state: "clear" };
|
|
3945
|
+
let pane;
|
|
3946
|
+
try {
|
|
3947
|
+
pane = await this.tmux.capturePane();
|
|
3948
|
+
}
|
|
3949
|
+
catch (err) {
|
|
3950
|
+
this.logger.debug({ err }, "capture-pane failed during the dialog probe — pane state unknown");
|
|
3951
|
+
return { state: "unknown" };
|
|
3952
|
+
}
|
|
3953
|
+
for (const dialog of dialogs) {
|
|
3954
|
+
if (Daemon.dialogMatches(dialog, pane)) {
|
|
3955
|
+
this.trackDialogParked(dialog);
|
|
3956
|
+
return { state: "dialog", dialog };
|
|
3957
|
+
}
|
|
3958
|
+
}
|
|
3959
|
+
this.trackDialogParked(null);
|
|
3960
|
+
return { state: "clear" };
|
|
3961
|
+
}
|
|
3962
|
+
/**
|
|
3963
|
+
* Remember how long a blocking dialog has been on screen. Once it has
|
|
3964
|
+
* outlived DIALOG_PARKED_NOTIFY_MS — the auto-dismiss did not take, or the
|
|
3965
|
+
* dialog is a hold-only variant — report it ONCE so a human answers it.
|
|
3966
|
+
* Never presses a key: the whole point is that we do not know which option
|
|
3967
|
+
* is safe. Identity is the pattern, so the delivery gate and the runtime
|
|
3968
|
+
* scanner (same canonical list) never reset each other's clock.
|
|
3969
|
+
*/
|
|
3970
|
+
trackDialogParked(dialog) {
|
|
3971
|
+
if (!dialog) {
|
|
3972
|
+
if (this.dialogParkedSince !== 0)
|
|
3973
|
+
this.logger.info({ dialog: this.dialogParkedKey }, "Blocking dialog is gone from the pane");
|
|
3974
|
+
this.dialogParkedSince = 0;
|
|
3975
|
+
this.dialogParkedKey = null;
|
|
3976
|
+
this.dialogParkedReported = false;
|
|
3977
|
+
return;
|
|
3978
|
+
}
|
|
3979
|
+
const key = Daemon.dialogKey(dialog);
|
|
3980
|
+
if (this.dialogParkedSince === 0 || this.dialogParkedKey !== key) {
|
|
3981
|
+
this.dialogParkedSince = Date.now();
|
|
3982
|
+
this.dialogParkedKey = key;
|
|
3983
|
+
this.dialogParkedReported = false;
|
|
3984
|
+
return;
|
|
3985
|
+
}
|
|
3986
|
+
if (!this.dialogParkedReported && Date.now() - this.dialogParkedSince >= DIALOG_PARKED_NOTIFY_MS) {
|
|
3987
|
+
this.dialogParkedReported = true;
|
|
3988
|
+
this.logger.warn({ dialog: dialog.description, parkedForMs: Date.now() - this.dialogParkedSince }, "CLI dialog is still on screen — not auto-answering, reporting for a human");
|
|
3989
|
+
this.emit("dialog_parked", { name: this.name, description: dialog.description, holdOnly: dialog.holdOnly === true });
|
|
3990
|
+
}
|
|
3991
|
+
}
|
|
3992
|
+
/**
|
|
3993
|
+
* Bottom-anchored readiness for TUIs that drop Enter while busy (see
|
|
3994
|
+
* CliBackend.dropsEnterWhileBusy). Output silence is necessary but not
|
|
3995
|
+
* sufficient for them: Kiro paints nothing for the whole duration of a shell
|
|
3996
|
+
* or MCP tool call (and for each 10s backend retry), yet an Enter arriving
|
|
3997
|
+
* then is discarded. Reproduced deterministically: `sleep 9` tool, paste at
|
|
3998
|
+
* t+6s → text stranded in the prompt row, next message submitted both.
|
|
3999
|
+
* Other backends keep the silence-based gate byte-for-byte.
|
|
4000
|
+
*/
|
|
4001
|
+
async isPaneReadyForDelivery(windowId) {
|
|
4002
|
+
return (await this.paneReadinessForDelivery(windowId)) === "ready";
|
|
4003
|
+
}
|
|
4004
|
+
/**
|
|
4005
|
+
* Why a pane is not deliverable matters. "busy" may be handed to a native
|
|
4006
|
+
* input queue or steered (after its own dialog probe); "dialog" and "unknown"
|
|
4007
|
+
* must never be — a paste + Enter into a dialog answers it with its default,
|
|
4008
|
+
* and an unreadable pane proves nothing. The silence gate comes first and
|
|
4009
|
+
* costs no capture, so a CLI that is simply working never pays for the probe.
|
|
4010
|
+
*/
|
|
4011
|
+
async paneReadinessForDelivery(windowId) {
|
|
4012
|
+
if (!this.isPaneIdleForDelivery(windowId))
|
|
4013
|
+
return "busy";
|
|
4014
|
+
const probe = await this.probeBlockingDialog();
|
|
4015
|
+
if (probe.state !== "clear")
|
|
4016
|
+
return probe.state;
|
|
4017
|
+
if (this.backend?.dropsEnterWhileBusy?.() !== true)
|
|
4018
|
+
return "ready";
|
|
4019
|
+
const prompt = this.backend.getBottomReadyPattern?.();
|
|
4020
|
+
if (!prompt || !this.tmux)
|
|
4021
|
+
return "ready";
|
|
4022
|
+
try {
|
|
4023
|
+
const pane = await this.tmux.capturePane();
|
|
4024
|
+
if (this.backend.getBusyPattern?.()?.test(pane))
|
|
4025
|
+
return "busy";
|
|
4026
|
+
if (bottomRowIsReady(pane, prompt))
|
|
4027
|
+
return "ready";
|
|
4028
|
+
// FAIL CLOSED on any non-blank screen without the prompt at the bottom.
|
|
4029
|
+
// The obvious "unknown layout → fall back to silence" shortcut re-opened
|
|
4030
|
+
// the bug: a long tool output scrolls the turn's own `N% !>` row out of
|
|
4031
|
+
// the visible pane (capture-pane -p is the viewport only), the bottom row
|
|
4032
|
+
// is tool output, the pane is quiet — and a paste there strands the text
|
|
4033
|
+
// exactly as before. The prompt row is the only evidence Kiro accepts an
|
|
4034
|
+
// Enter; a modal or panel is handled by the dialog dismisser, and the
|
|
4035
|
+
// bounded wait reports a failure rather than pasting into a screen we do
|
|
4036
|
+
// not understand. That includes a completely BLANK capture: a clear /
|
|
4037
|
+
// redraw / alternate-screen transition paints nothing for a frame, tmux
|
|
4038
|
+
// still accepts a paste, and "nothing visible" is not evidence the TUI is
|
|
4039
|
+
// idle. The next 250ms poll sees the prompt if it is coming.
|
|
4040
|
+
return "busy";
|
|
4041
|
+
}
|
|
4042
|
+
catch (err) {
|
|
4043
|
+
// Same rule for a capture that fails: no positive proof, no paste. The
|
|
4044
|
+
// poll retries; a persistently unreadable pane ends in the bounded
|
|
4045
|
+
// timeout's ❌ rather than in a silently stranded message.
|
|
4046
|
+
this.logger.debug({ err }, "capture-pane failed during the readiness check — treating the pane as not ready");
|
|
4047
|
+
return "busy";
|
|
4048
|
+
}
|
|
4049
|
+
}
|
|
4050
|
+
/**
|
|
4051
|
+
* Like waitForPaneIdleForDelivery, plus the bottom-anchored prompt check for
|
|
4052
|
+
* Enter-dropping TUIs. Polls at a coarse cadence (each poll is a capture-pane)
|
|
4053
|
+
* only after the pane is already quiet, so the common fast-idle delivery still
|
|
4054
|
+
* costs a single capture.
|
|
4055
|
+
*/
|
|
4056
|
+
async waitForPaneReadyForDelivery(windowId, timeoutMs = 30 * 60_000) {
|
|
4057
|
+
// Every backend polls: readiness now also means "no blocking dialog on
|
|
4058
|
+
// screen", and a dialog disappears without any output edge the silence
|
|
4059
|
+
// gate could see.
|
|
4060
|
+
const deadline = Date.now() + timeoutMs;
|
|
4061
|
+
const bottomGated = this.backend?.dropsEnterWhileBusy?.() === true;
|
|
4062
|
+
let unknownStreak = 0;
|
|
4063
|
+
for (;;) {
|
|
4064
|
+
const remaining = deadline - Date.now();
|
|
4065
|
+
if (remaining <= 0)
|
|
4066
|
+
return false;
|
|
4067
|
+
const idle = await this.waitForPaneIdleForDelivery(windowId, remaining);
|
|
4068
|
+
if (!idle)
|
|
4069
|
+
return false;
|
|
4070
|
+
// The idle wait just resolved, so re-reading the silence gate would only
|
|
4071
|
+
// add a stale-cache race; ask the pane the remaining questions directly.
|
|
4072
|
+
if (bottomGated) {
|
|
4073
|
+
if (await this.isPaneReadyForDelivery(windowId))
|
|
4074
|
+
return true;
|
|
4075
|
+
}
|
|
4076
|
+
else {
|
|
4077
|
+
const probe = await this.probeBlockingDialog();
|
|
4078
|
+
if (probe.state === "clear")
|
|
4079
|
+
return true;
|
|
4080
|
+
if (probe.state === "unknown") {
|
|
4081
|
+
if (++unknownStreak >= DIALOG_PROBE_UNKNOWN_MAX) {
|
|
4082
|
+
this.logger.error({ probes: unknownStreak }, "Pane stayed unreadable during the dialog probe — refusing to deliver blind");
|
|
4083
|
+
return false;
|
|
4084
|
+
}
|
|
4085
|
+
}
|
|
4086
|
+
else {
|
|
4087
|
+
unknownStreak = 0;
|
|
4088
|
+
}
|
|
4089
|
+
}
|
|
4090
|
+
await new Promise(r => setTimeout(r, BOTTOM_READY_POLL_MS));
|
|
4091
|
+
}
|
|
4092
|
+
}
|
|
4093
|
+
/**
|
|
4094
|
+
* F3: if an AgEnD message is sitting unsubmitted in the input row (a previous
|
|
4095
|
+
* delivery's Enter was dropped), submit it with one bare Enter under the pane
|
|
4096
|
+
* lock. Only acts on a positive signature — the `[user:`/`[from:` marker every
|
|
4097
|
+
* pasted message carries — never on Kiro's own placeholder hints.
|
|
4098
|
+
*
|
|
4099
|
+
* Outcomes: "idle" = positively verified nothing stranded (the caller may
|
|
4100
|
+
* paste); "submitted" = an Enter went out for the stranded text; "busy" = the
|
|
4101
|
+
* pane is not at its prompt or could not be read (the caller must wait for
|
|
4102
|
+
* the prompt and run this check AGAIN before pasting — never assume someone
|
|
4103
|
+
* else submitted it); "failed" = the Enter could not be sent (the delivery
|
|
4104
|
+
* must fail rather than paste on top of the stranded text).
|
|
4105
|
+
*/
|
|
4106
|
+
async submitStrandedInputIfAny(windowId) {
|
|
4107
|
+
if (this.backend?.dropsEnterWhileBusy?.() !== true || !this.tmux)
|
|
4108
|
+
return "idle";
|
|
4109
|
+
const prompt = this.backend.getBottomReadyPattern?.();
|
|
4110
|
+
if (!prompt)
|
|
4111
|
+
return "idle";
|
|
4112
|
+
// Pre-check outside the lock so the common case (verified clear) never
|
|
4113
|
+
// contends with the dialog dismisser. Only a POSITIVE "clear" skips the
|
|
4114
|
+
// lock; an unreadable pane is re-examined under it, not waved through.
|
|
4115
|
+
const pre = await this.strandedInputState(prompt);
|
|
4116
|
+
if (pre === "clear")
|
|
4117
|
+
return "idle";
|
|
4118
|
+
if (pre === "busy")
|
|
4119
|
+
return "busy";
|
|
4120
|
+
if (pre === "stranded") {
|
|
4121
|
+
this.logger.warn({ windowId }, "Unsubmitted message found in the input row — submitting it before this delivery");
|
|
4122
|
+
}
|
|
4123
|
+
return this.paneWriteLock.run(async () => {
|
|
4124
|
+
// Re-check UNDER the lock right before the key goes out: between the
|
|
4125
|
+
// pre-check and acquiring the lock the CLI may have started generating
|
|
4126
|
+
// (the stranded text got submitted by someone else, or the dismisser
|
|
4127
|
+
// acted), or a fatal startup screen may have come up. A bare Enter is a
|
|
4128
|
+
// no-op at an empty prompt, but into a modal it is a confirmation.
|
|
4129
|
+
if (this.fatalStartupBlocked)
|
|
4130
|
+
return "busy";
|
|
4131
|
+
const state = await this.strandedInputState(prompt);
|
|
4132
|
+
if (state === "clear")
|
|
4133
|
+
return "idle";
|
|
4134
|
+
if (state !== "stranded")
|
|
4135
|
+
return "busy"; // busy, or unreadable: no evidence the prompt is free
|
|
4136
|
+
const sent = await this.sendDeliveryEnter("stranded-input-submit");
|
|
4137
|
+
return sent ? "submitted" : "failed";
|
|
4138
|
+
});
|
|
4139
|
+
}
|
|
4140
|
+
/**
|
|
4141
|
+
* "stranded": an AgEnD message sits in the input row; "clear": the prompt is
|
|
4142
|
+
* at the bottom with no stranded text; "busy": generating, or the prompt is
|
|
4143
|
+
* not at the bottom (incl. a blank frame); "unknown": the pane could not be
|
|
4144
|
+
* read. Only "clear" is positive evidence.
|
|
4145
|
+
*/
|
|
4146
|
+
async strandedInputState(prompt) {
|
|
4147
|
+
let pane;
|
|
4148
|
+
try {
|
|
4149
|
+
pane = await this.tmux.capturePane();
|
|
4150
|
+
}
|
|
4151
|
+
catch {
|
|
4152
|
+
return "unknown";
|
|
4153
|
+
}
|
|
4154
|
+
if (this.backend?.getBusyPattern?.()?.test(pane))
|
|
4155
|
+
return "busy";
|
|
4156
|
+
if (strandedAgendMessageInInput(pane, prompt))
|
|
4157
|
+
return "stranded";
|
|
4158
|
+
return bottomRowIsReady(pane, prompt) ? "clear" : "busy";
|
|
4159
|
+
}
|
|
4160
|
+
/**
|
|
4161
|
+
* F2: for Enter-dropping TUIs, "some output after Enter" proves nothing — the
|
|
4162
|
+
* turn that made the pane look idle keeps producing output. Confirm the
|
|
4163
|
+
* SUBMISSION instead: the pasted text is gone from the input row ("absent"),
|
|
4164
|
+
* or the TUI's own busy marker is up. "unknown" = the pane could not be read;
|
|
4165
|
+
* it never counts as confirmed.
|
|
4166
|
+
*/
|
|
4167
|
+
async pasteResidue(formatted) {
|
|
4168
|
+
const prompt = this.backend?.getBottomReadyPattern?.();
|
|
4169
|
+
if (!prompt || !this.tmux)
|
|
4170
|
+
return "absent";
|
|
4171
|
+
let pane;
|
|
4172
|
+
try {
|
|
4173
|
+
pane = await this.tmux.capturePane();
|
|
4174
|
+
}
|
|
4175
|
+
catch {
|
|
4176
|
+
return "unknown";
|
|
4177
|
+
}
|
|
4178
|
+
if (this.backend?.getBusyPattern?.()?.test(pane))
|
|
4179
|
+
return "absent"; // generating = submitted
|
|
4180
|
+
return pasteLeftInInput(pane, prompt, formatted) ? "present" : "absent";
|
|
4181
|
+
}
|
|
4182
|
+
/**
|
|
4183
|
+
* Submission = the legacy idle→busy output signal AND the pasted text has
|
|
4184
|
+
* verifiably left the input row. The first alone is what produced false ✅s:
|
|
4185
|
+
* the turn that made the pane look idle keeps emitting output after our
|
|
4186
|
+
* (dropped) Enter — and an unreadable pane is not a verified absence either.
|
|
4187
|
+
*/
|
|
4188
|
+
async confirmSubmittedAfterEnter(windowId, enterAt, formatted) {
|
|
4189
|
+
const sawOutput = await this.confirmBusyAfterEnter(windowId, enterAt);
|
|
4190
|
+
const residue = await this.pasteResidue(formatted);
|
|
4191
|
+
if (residue === "unknown")
|
|
4192
|
+
this.logger.warn("Could not read the pane after Enter — not confirming the submission");
|
|
4193
|
+
return sawOutput && residue === "absent";
|
|
4194
|
+
}
|
|
4195
|
+
/**
|
|
4196
|
+
* While the CLI is parked on a fatal startup screen (see fatalStartupBlocked),
|
|
4197
|
+
* no pane write may happen: paste+Enter into the corrupt-config modal would
|
|
4198
|
+
* confirm "Exit and fix manually" and exit the CLI. The message is failed (❌)
|
|
4199
|
+
* rather than held — the accompanying config_error notification tells the user
|
|
4200
|
+
* what to fix, and holding would only build a queue that floods the pane the
|
|
4201
|
+
* moment the flag clears.
|
|
4202
|
+
*/
|
|
4203
|
+
refuseFatalStartupDelivery(status) {
|
|
4204
|
+
if (!this.fatalStartupBlocked)
|
|
4205
|
+
return false;
|
|
4206
|
+
this.logger.error("Delivery refused — CLI is parked on a fatal startup screen");
|
|
4207
|
+
if (status)
|
|
4208
|
+
this.emit("message_failed", status); // ❌
|
|
4209
|
+
return true;
|
|
4210
|
+
}
|
|
4211
|
+
/**
|
|
4212
|
+
* A raw `%output` silence check is authoritative for ordinary TUIs. It is not
|
|
4213
|
+
* for a backend such as agy whose status-line hook periodically redraws an
|
|
4214
|
+
* identical screen. In that case trust an idle pane-state snapshot only when
|
|
4215
|
+
* it was captured after both the latest output and the latest control-client
|
|
4216
|
+
* observation reset.
|
|
4217
|
+
*/
|
|
4218
|
+
isPaneIdleForDelivery(windowId) {
|
|
4219
|
+
if (!this.controlClient)
|
|
4220
|
+
return true;
|
|
4221
|
+
if (this.controlClient.isIdle(windowId))
|
|
4222
|
+
return true;
|
|
4223
|
+
if (this.backend?.hasPeriodicPaneRedraw?.() !== true || this.instanceState !== "idle")
|
|
4224
|
+
return false;
|
|
4225
|
+
const lastOutputAt = this.controlClient.getLastOutputAt(windowId);
|
|
4226
|
+
if (lastOutputAt === undefined || !this.instanceStateMachine)
|
|
4227
|
+
return false;
|
|
4228
|
+
const observedAt = this.instanceStateMachine.snapshot().observedAt;
|
|
4229
|
+
return observedAt >= lastOutputAt
|
|
4230
|
+
&& observedAt >= this.controlClient.getObservationResetAt();
|
|
4231
|
+
}
|
|
4232
|
+
waitForPaneIdleForDelivery(windowId, timeoutMs = 30 * 60_000) {
|
|
4233
|
+
if (!this.controlClient)
|
|
4234
|
+
return Promise.resolve(true);
|
|
4235
|
+
if (this.backend?.hasPeriodicPaneRedraw?.() !== true) {
|
|
4236
|
+
return this.controlClient.waitUntilIdle(windowId, timeoutMs);
|
|
4237
|
+
}
|
|
4238
|
+
if (this.isPaneIdleForDelivery(windowId))
|
|
4239
|
+
return Promise.resolve(true);
|
|
4240
|
+
return new Promise(resolve => {
|
|
4241
|
+
const check = setInterval(() => {
|
|
4242
|
+
if (!this.isPaneIdleForDelivery(windowId))
|
|
4243
|
+
return;
|
|
4244
|
+
clearInterval(check);
|
|
4245
|
+
clearTimeout(timer);
|
|
4246
|
+
resolve(true);
|
|
4247
|
+
}, 50);
|
|
4248
|
+
const timer = setTimeout(() => {
|
|
4249
|
+
clearInterval(check);
|
|
4250
|
+
resolve(false);
|
|
4251
|
+
}, timeoutMs);
|
|
4252
|
+
});
|
|
2625
4253
|
}
|
|
2626
4254
|
/**
|
|
2627
4255
|
* The write half of a delivery: paste → Enter → confirm, with retries.
|
|
@@ -2649,7 +4277,16 @@ export class Daemon extends EventEmitter {
|
|
|
2649
4277
|
const pasteStartedAt = Date.now();
|
|
2650
4278
|
const pasted = await this.tmux.pasteBuffer(formatted);
|
|
2651
4279
|
if (!pasted) {
|
|
2652
|
-
this.
|
|
4280
|
+
const tmuxError = this.tmux.getLastPasteError?.() ?? "unknown tmux paste failure";
|
|
4281
|
+
const recoverable = this.tmux.isLastPasteFailureRecoverable?.() ?? true;
|
|
4282
|
+
this.logger[recoverable ? "warn" : "error"]({ attempt, tmuxError, recoverable }, recoverable
|
|
4283
|
+
? "pasteBuffer failed — recovering window and backing off"
|
|
4284
|
+
: "pasteBuffer failed — non-retryable tmux error");
|
|
4285
|
+
if (!recoverable) {
|
|
4286
|
+
if (status)
|
|
4287
|
+
this.emit("message_failed", status);
|
|
4288
|
+
return false;
|
|
4289
|
+
}
|
|
2653
4290
|
windowId = (await this.recoverWindow()) ?? windowId;
|
|
2654
4291
|
if (attempt < maxAttempts)
|
|
2655
4292
|
await new Promise(r => setTimeout(r, 2000));
|
|
@@ -2731,7 +4368,9 @@ export class Daemon extends EventEmitter {
|
|
|
2731
4368
|
}
|
|
2732
4369
|
const repasted = await this.tmux.pasteBuffer(formatted);
|
|
2733
4370
|
if (!repasted) {
|
|
2734
|
-
this.logger.error(
|
|
4371
|
+
this.logger.error({
|
|
4372
|
+
tmuxError: this.tmux.getLastPasteError?.() ?? "unknown tmux paste failure",
|
|
4373
|
+
}, "Idle-gated redelivery paste failed after native-queue silent loss");
|
|
2735
4374
|
if (status)
|
|
2736
4375
|
this.emit("message_failed", status); // ❌
|
|
2737
4376
|
return false;
|
|
@@ -2771,7 +4410,36 @@ export class Daemon extends EventEmitter {
|
|
|
2771
4410
|
this.emit("message_failed", status); // ❌
|
|
2772
4411
|
return false;
|
|
2773
4412
|
}
|
|
2774
|
-
if (windowId && this.controlClient) {
|
|
4413
|
+
if (windowId && this.controlClient && this.backend?.dropsEnterWhileBusy?.() === true) {
|
|
4414
|
+
// F2: output after Enter is necessary but not sufficient — the paste
|
|
4415
|
+
// must also have LEFT the input row.
|
|
4416
|
+
let submitted = await this.confirmSubmittedAfterEnter(windowId, enterAt, formatted);
|
|
4417
|
+
if (!submitted) {
|
|
4418
|
+
// The Enter landed while the TUI was still busy (it kept the text but
|
|
4419
|
+
// dropped the key). A retry is only useful once the prompt is back —
|
|
4420
|
+
// bounded, since we hold the pane lock here.
|
|
4421
|
+
this.logger.warn("Message not submitted after Enter — waiting for the prompt, then re-sending Enter once");
|
|
4422
|
+
const promptBack = await this.waitForPaneReadyForDelivery(windowId, STRANDED_RETRY_READY_WAIT_MS);
|
|
4423
|
+
const retryAt = Date.now();
|
|
4424
|
+
if (promptBack && !(await this.sendDeliveryEnter("stranded-text-retry"))) {
|
|
4425
|
+
if (status)
|
|
4426
|
+
this.emit("message_failed", status); // ❌
|
|
4427
|
+
return false;
|
|
4428
|
+
}
|
|
4429
|
+
submitted = promptBack && await this.confirmSubmittedAfterEnter(windowId, retryAt, formatted);
|
|
4430
|
+
}
|
|
4431
|
+
if (submitted) {
|
|
4432
|
+
if (status)
|
|
4433
|
+
this.emit("message_confirmed", status); // ✅
|
|
4434
|
+
}
|
|
4435
|
+
else {
|
|
4436
|
+
this.logger.error("Message pasted but never submitted (text still in the input row after Enter retry)");
|
|
4437
|
+
if (status)
|
|
4438
|
+
this.emit("message_failed", status); // ❌
|
|
4439
|
+
return false;
|
|
4440
|
+
}
|
|
4441
|
+
}
|
|
4442
|
+
else if (windowId && this.controlClient) {
|
|
2775
4443
|
let becameBusy = await this.confirmBusyAfterEnter(windowId, enterAt);
|
|
2776
4444
|
if (!becameBusy) {
|
|
2777
4445
|
this.logger.warn("No idle→busy transition after Enter — re-sending Enter once");
|
|
@@ -2925,14 +4593,24 @@ export class Daemon extends EventEmitter {
|
|
|
2925
4593
|
const args = (msg.args ?? {});
|
|
2926
4594
|
const requestId = msg.requestId;
|
|
2927
4595
|
this.logger.debug({ tool, requestId }, "Tool call from MCP server");
|
|
4596
|
+
// A tool call can only come from a live MCP server of ours — proof of life
|
|
4597
|
+
// that outranks whatever the pid slot says.
|
|
4598
|
+
if (this.socketSessionNames.get(socket) === this.name) {
|
|
4599
|
+
this.noteMcpProofOfLife("tool_call", this.socketPids.get(socket));
|
|
4600
|
+
}
|
|
2928
4601
|
// For now, log and respond. Full adapter routing will be wired in fleet manager.
|
|
2929
4602
|
const respond = (result, error) => {
|
|
2930
4603
|
// A message that verifiably went out stands down the dead-MCP proxy reply
|
|
2931
4604
|
// for this turn: the agent proved it can still speak for itself.
|
|
2932
4605
|
if (!error && result != null && TURN_REPLY_TOOLS.has(tool)) {
|
|
2933
4606
|
this.turnOutboundDelivered = true;
|
|
4607
|
+
if (tool === "reply")
|
|
4608
|
+
this.turnReplyDelivered = true;
|
|
4609
|
+
}
|
|
4610
|
+
const sent = this.ipcServer?.send(socket, { requestId, result, error }) ?? false;
|
|
4611
|
+
if (!sent) {
|
|
4612
|
+
this.logger.warn({ tool, requestId }, "Tool result could not be returned — originating MCP socket is closed");
|
|
2934
4613
|
}
|
|
2935
|
-
this.ipcServer?.send(socket, { requestId, result, error });
|
|
2936
4614
|
};
|
|
2937
4615
|
// Repo checkout — handled locally in daemon (no fleet-manager)
|
|
2938
4616
|
if (tool === "checkout_repo") {
|
|
@@ -2945,13 +4623,7 @@ export class Daemon extends EventEmitter {
|
|
|
2945
4623
|
}
|
|
2946
4624
|
if (tool === "set_display_name" || tool === "set_description") {
|
|
2947
4625
|
const type = tool === "set_display_name" ? "fleet_set_display_name" : "fleet_set_description";
|
|
2948
|
-
const fleetReqId = `${tool === "set_display_name" ? "dn" : "desc"}_${requestId}`;
|
|
2949
|
-
this.ipcServer?.broadcast({
|
|
2950
|
-
type,
|
|
2951
|
-
payload: args,
|
|
2952
|
-
meta: { instance_name: this.name },
|
|
2953
|
-
fleetRequestId: fleetReqId,
|
|
2954
|
-
});
|
|
4626
|
+
const fleetReqId = `${tool === "set_display_name" ? "dn" : "desc"}_${++this.fleetRequestSeq}_${requestId}`;
|
|
2955
4627
|
const timeout = setTimeout(() => {
|
|
2956
4628
|
this.pendingIpcRequests.delete(fleetReqId);
|
|
2957
4629
|
respond(null, `${tool} timed out`);
|
|
@@ -2960,16 +4632,16 @@ export class Daemon extends EventEmitter {
|
|
|
2960
4632
|
clearTimeout(timeout);
|
|
2961
4633
|
respond(respMsg.result, respMsg.error);
|
|
2962
4634
|
});
|
|
2963
|
-
return;
|
|
2964
|
-
}
|
|
2965
|
-
if (tool === TASK_TOOL) {
|
|
2966
|
-
const fleetReqId = `task_${requestId}`;
|
|
2967
4635
|
this.ipcServer?.broadcast({
|
|
2968
|
-
type
|
|
4636
|
+
type,
|
|
2969
4637
|
payload: args,
|
|
2970
4638
|
meta: { instance_name: this.name },
|
|
2971
4639
|
fleetRequestId: fleetReqId,
|
|
2972
4640
|
});
|
|
4641
|
+
return;
|
|
4642
|
+
}
|
|
4643
|
+
if (tool === TASK_TOOL) {
|
|
4644
|
+
const fleetReqId = `task_${++this.fleetRequestSeq}_${requestId}`;
|
|
2973
4645
|
const timeout = setTimeout(() => {
|
|
2974
4646
|
this.pendingIpcRequests.delete(fleetReqId);
|
|
2975
4647
|
respond(null, `Task operation timed out after ${daemonBudgetMs(tool) / 1000}s`);
|
|
@@ -2978,6 +4650,12 @@ export class Daemon extends EventEmitter {
|
|
|
2978
4650
|
clearTimeout(timeout);
|
|
2979
4651
|
respond(respMsg.result, respMsg.error);
|
|
2980
4652
|
});
|
|
4653
|
+
this.ipcServer?.broadcast({
|
|
4654
|
+
type: "fleet_task",
|
|
4655
|
+
payload: args,
|
|
4656
|
+
meta: { instance_name: this.name },
|
|
4657
|
+
fleetRequestId: fleetReqId,
|
|
4658
|
+
});
|
|
2981
4659
|
return;
|
|
2982
4660
|
}
|
|
2983
4661
|
if (DECISION_TOOLS.has(tool)) {
|
|
@@ -2986,13 +4664,7 @@ export class Daemon extends EventEmitter {
|
|
|
2986
4664
|
list_decisions: "fleet_decision_list",
|
|
2987
4665
|
update_decision: "fleet_decision_update",
|
|
2988
4666
|
};
|
|
2989
|
-
const fleetReqId = `dec_${requestId}`;
|
|
2990
|
-
this.ipcServer?.broadcast({
|
|
2991
|
-
type: typeMap[tool],
|
|
2992
|
-
payload: args,
|
|
2993
|
-
meta: { instance_name: this.name, working_directory: this.config.working_directory },
|
|
2994
|
-
fleetRequestId: fleetReqId,
|
|
2995
|
-
});
|
|
4667
|
+
const fleetReqId = `dec_${++this.fleetRequestSeq}_${requestId}`;
|
|
2996
4668
|
const timeout = setTimeout(() => {
|
|
2997
4669
|
this.pendingIpcRequests.delete(fleetReqId);
|
|
2998
4670
|
respond(null, `Decision operation timed out after ${daemonBudgetMs(tool) / 1000}s`);
|
|
@@ -3001,6 +4673,12 @@ export class Daemon extends EventEmitter {
|
|
|
3001
4673
|
clearTimeout(timeout);
|
|
3002
4674
|
respond(respMsg.result, respMsg.error);
|
|
3003
4675
|
});
|
|
4676
|
+
this.ipcServer?.broadcast({
|
|
4677
|
+
type: typeMap[tool],
|
|
4678
|
+
payload: args,
|
|
4679
|
+
meta: { instance_name: this.name, working_directory: this.config.working_directory },
|
|
4680
|
+
fleetRequestId: fleetReqId,
|
|
4681
|
+
});
|
|
3004
4682
|
return;
|
|
3005
4683
|
}
|
|
3006
4684
|
if (SCHEDULE_TOOLS.has(tool)) {
|
|
@@ -3012,13 +4690,7 @@ export class Daemon extends EventEmitter {
|
|
|
3012
4690
|
};
|
|
3013
4691
|
// Use fleetRequestId (not requestId) to avoid MCP server resolving the
|
|
3014
4692
|
// pending tool call prematurely when it receives the broadcast.
|
|
3015
|
-
const fleetReqId = `sched_${requestId}`;
|
|
3016
|
-
this.ipcServer?.broadcast({
|
|
3017
|
-
type: typeMap[tool],
|
|
3018
|
-
payload: args,
|
|
3019
|
-
meta: { chat_id: this.lastChatId, thread_id: this.lastThreadId, instance_name: this.name },
|
|
3020
|
-
fleetRequestId: fleetReqId,
|
|
3021
|
-
});
|
|
4693
|
+
const fleetReqId = `sched_${++this.fleetRequestSeq}_${requestId}`;
|
|
3022
4694
|
// Wait for fleet_schedule_response via pending request map
|
|
3023
4695
|
const timeout = setTimeout(() => {
|
|
3024
4696
|
this.pendingIpcRequests.delete(fleetReqId);
|
|
@@ -3028,6 +4700,12 @@ export class Daemon extends EventEmitter {
|
|
|
3028
4700
|
clearTimeout(timeout);
|
|
3029
4701
|
respond(respMsg.result, respMsg.error);
|
|
3030
4702
|
});
|
|
4703
|
+
this.ipcServer?.broadcast({
|
|
4704
|
+
type: typeMap[tool],
|
|
4705
|
+
payload: args,
|
|
4706
|
+
meta: { chat_id: this.lastChatId, thread_id: this.lastThreadId, instance_name: this.name },
|
|
4707
|
+
fleetRequestId: fleetReqId,
|
|
4708
|
+
});
|
|
3031
4709
|
return;
|
|
3032
4710
|
}
|
|
3033
4711
|
if (CROSS_INSTANCE_TOOLS.has(tool)) {
|
|
@@ -3035,15 +4713,8 @@ export class Daemon extends EventEmitter {
|
|
|
3035
4713
|
if (this.topicMode && this.ipcServer) {
|
|
3036
4714
|
// Use fleetRequestId (not requestId) to avoid MCP server resolving the
|
|
3037
4715
|
// pending tool call prematurely when it receives the broadcast.
|
|
3038
|
-
const fleetReqId = `xmsg_${requestId}`;
|
|
4716
|
+
const fleetReqId = `xmsg_${++this.fleetRequestSeq}_${requestId}`;
|
|
3039
4717
|
const senderSessionName = this.socketSessionNames.get(socket);
|
|
3040
|
-
this.ipcServer.broadcast({
|
|
3041
|
-
type: "fleet_outbound",
|
|
3042
|
-
tool,
|
|
3043
|
-
args,
|
|
3044
|
-
fleetRequestId: fleetReqId,
|
|
3045
|
-
senderSessionName,
|
|
3046
|
-
});
|
|
3047
4718
|
// Shared table, so delete_instance and the deployment tools get the same
|
|
3048
4719
|
// budget here as the MCP client expects (the ad-hoc list omitted them).
|
|
3049
4720
|
const crossTimeoutMs = daemonBudgetMs(tool);
|
|
@@ -3055,6 +4726,13 @@ export class Daemon extends EventEmitter {
|
|
|
3055
4726
|
clearTimeout(timeout);
|
|
3056
4727
|
respond(respMsg.result, respMsg.error);
|
|
3057
4728
|
});
|
|
4729
|
+
this.ipcServer.broadcast({
|
|
4730
|
+
type: "fleet_outbound",
|
|
4731
|
+
tool,
|
|
4732
|
+
args,
|
|
4733
|
+
fleetRequestId: fleetReqId,
|
|
4734
|
+
senderSessionName,
|
|
4735
|
+
});
|
|
3058
4736
|
}
|
|
3059
4737
|
else {
|
|
3060
4738
|
respond(null, "Cross-instance messaging requires topic mode");
|
|
@@ -3088,8 +4766,24 @@ export class Daemon extends EventEmitter {
|
|
|
3088
4766
|
// The fleet manager's IPC client receives this and routes to shared adapter.
|
|
3089
4767
|
// Use fleetRequestId (not requestId) to avoid other MCP sessions on this daemon
|
|
3090
4768
|
// from prematurely resolving their pending requests when they receive the broadcast.
|
|
3091
|
-
|
|
4769
|
+
// requestId is local to one MCP server process. Claude may keep multiple
|
|
4770
|
+
// MCP children attached (main session, resumed session, subagent); those
|
|
4771
|
+
// processes all emit requestId=1,2,... and previously overwrote this
|
|
4772
|
+
// shared pending map. The wrong caller could then receive another POST's
|
|
4773
|
+
// success + messageId while its own reply had never been confirmed.
|
|
4774
|
+
const fleetReqId = `tool_${++this.fleetRequestSeq}_${requestId}`;
|
|
3092
4775
|
const outboundKey = fleetReqId;
|
|
4776
|
+
const timeout = setTimeout(() => {
|
|
4777
|
+
this.pendingIpcRequests.delete(outboundKey);
|
|
4778
|
+
respond(null, `Fleet outbound timed out after ${daemonBudgetMs(tool) / 1000}s`);
|
|
4779
|
+
}, daemonBudgetMs(tool));
|
|
4780
|
+
this.pendingIpcRequests.set(outboundKey, (respMsg) => {
|
|
4781
|
+
clearTimeout(timeout);
|
|
4782
|
+
respond(respMsg.result, respMsg.error);
|
|
4783
|
+
});
|
|
4784
|
+
// Register the waiter before broadcasting. A fast/mock adapter can settle
|
|
4785
|
+
// immediately; broadcasting first left a response-before-registration
|
|
4786
|
+
// window that degraded into a timeout despite a completed POST.
|
|
3093
4787
|
this.ipcServer?.broadcast({
|
|
3094
4788
|
type: "fleet_outbound",
|
|
3095
4789
|
tool,
|
|
@@ -3100,14 +4794,6 @@ export class Daemon extends EventEmitter {
|
|
|
3100
4794
|
// from a secondary world, and message ids are not portable across bots.
|
|
3101
4795
|
adapterId: this.lastAdapterId,
|
|
3102
4796
|
});
|
|
3103
|
-
const timeout = setTimeout(() => {
|
|
3104
|
-
this.pendingIpcRequests.delete(outboundKey);
|
|
3105
|
-
respond(null, `Fleet outbound timed out after ${daemonBudgetMs(tool) / 1000}s`);
|
|
3106
|
-
}, daemonBudgetMs(tool));
|
|
3107
|
-
this.pendingIpcRequests.set(outboundKey, (respMsg) => {
|
|
3108
|
-
clearTimeout(timeout);
|
|
3109
|
-
respond(respMsg.result, respMsg.error);
|
|
3110
|
-
});
|
|
3111
4797
|
return;
|
|
3112
4798
|
}
|
|
3113
4799
|
const adapter = adapters[0];
|
|
@@ -3117,7 +4803,10 @@ export class Daemon extends EventEmitter {
|
|
|
3117
4803
|
}
|
|
3118
4804
|
/** Build config object for the CLI backend */
|
|
3119
4805
|
buildBackendConfig() {
|
|
3120
|
-
|
|
4806
|
+
// Antigravity 1.1.17 has no per-workspace MCP config, but its global MCP
|
|
4807
|
+
// children inherit the CLI process environment. The backend installs one
|
|
4808
|
+
// environment-selecting launcher, so MCP is now the safe default here too.
|
|
4809
|
+
const isCliMode = this.config.agent_mode === "cli";
|
|
3121
4810
|
const sockPath = join(this.instanceDir, "channel.sock");
|
|
3122
4811
|
let serverJs = join(__dirname, "channel", "mcp-server.js");
|
|
3123
4812
|
if (!existsSync(serverJs)) {
|
|
@@ -3376,9 +5065,35 @@ export class Daemon extends EventEmitter {
|
|
|
3376
5065
|
throw new Error("No backend configured — cannot spawn CLI window");
|
|
3377
5066
|
}
|
|
3378
5067
|
const attemptedResume = !this.skipResume;
|
|
3379
|
-
|
|
5068
|
+
// A resume launch may get a longer budget than a fresh one (kiro: the
|
|
5069
|
+
// conversation must come back from the backend before anything paints).
|
|
5070
|
+
const resumeBudget = attemptedResume ? this.startupBudgetFor(true) : undefined;
|
|
5071
|
+
let alive = await this.trySpawn(false, resumeBudget);
|
|
5072
|
+
if (!alive && attemptedResume) {
|
|
5073
|
+
// Resume failed. Before abandoning the session:
|
|
5074
|
+
// 1. If the backend is unreachable (fleet-level memory, or this very pane
|
|
5075
|
+
// shows the outage text), clearing the session would trade a
|
|
5076
|
+
// transient outage for permanent conversation loss — and the fresh
|
|
5077
|
+
// start would hang on the same backend. Fail the startup and keep the
|
|
5078
|
+
// session; the fleet's delayed retry comes back once it is up.
|
|
5079
|
+
// 2. Otherwise, for backends that ask for it, retry resume ONCE — the
|
|
5080
|
+
// first miss is usually slowness, not a broken session.
|
|
5081
|
+
await this.noteStartupPaneForBackendOutage();
|
|
5082
|
+
await this.failStartupIfBackendUnreachable();
|
|
5083
|
+
if (this.backend.retriesResumeOnStartupFailure?.() === true) {
|
|
5084
|
+
this.logger.warn("Resume startup failed — retrying resume once before abandoning the session");
|
|
5085
|
+
await this.killProcessTree();
|
|
5086
|
+
await this.tmux.killWindow();
|
|
5087
|
+
alive = await this.trySpawn(false, resumeBudget);
|
|
5088
|
+
if (!alive) {
|
|
5089
|
+
await this.noteStartupPaneForBackendOutage();
|
|
5090
|
+
await this.failStartupIfBackendUnreachable();
|
|
5091
|
+
}
|
|
5092
|
+
}
|
|
5093
|
+
}
|
|
3380
5094
|
if (!alive) {
|
|
3381
|
-
//
|
|
5095
|
+
// Resume (or a fresh start) failed for a reason we do not recognise as an
|
|
5096
|
+
// outage (stale --resume, crash, rate limit, etc.).
|
|
3382
5097
|
// Clean slate: clear session-id, skip resume, and retry once.
|
|
3383
5098
|
this.logger.warn("CLI startup failed — clearing session-id and retrying without resume");
|
|
3384
5099
|
const sidFile = join(this.instanceDir, "session-id");
|
|
@@ -3389,7 +5104,7 @@ export class Daemon extends EventEmitter {
|
|
|
3389
5104
|
this.skipResume = true;
|
|
3390
5105
|
await this.killProcessTree();
|
|
3391
5106
|
await this.tmux.killWindow();
|
|
3392
|
-
const retryAlive = await this.trySpawn();
|
|
5107
|
+
const retryAlive = await this.trySpawn(false, this.startupBudgetFor(false));
|
|
3393
5108
|
if (!retryAlive) {
|
|
3394
5109
|
await this.killProcessTree();
|
|
3395
5110
|
await this.tmux.killWindow();
|
|
@@ -3398,6 +5113,9 @@ export class Daemon extends EventEmitter {
|
|
|
3398
5113
|
}
|
|
3399
5114
|
else if (attemptedResume) {
|
|
3400
5115
|
resumedSuccessfully = true;
|
|
5116
|
+
// A resume needs the backend: its success is positive proof the backend
|
|
5117
|
+
// is reachable again (a fresh prompt is local and proves nothing).
|
|
5118
|
+
this.backendOutage?.clear(this.backendKey());
|
|
3401
5119
|
}
|
|
3402
5120
|
this.lastSpawnAt = Date.now();
|
|
3403
5121
|
this.skipResume = false; // CLI started successfully — reset for next spawn
|
|
@@ -3408,6 +5126,122 @@ export class Daemon extends EventEmitter {
|
|
|
3408
5126
|
}
|
|
3409
5127
|
return resumedSuccessfully;
|
|
3410
5128
|
}
|
|
5129
|
+
/**
|
|
5130
|
+
* Startup budget for this launch: the backend's override (resume-aware), never
|
|
5131
|
+
* below a user-configured startup_timeout_ms; undefined = trySpawn's default.
|
|
5132
|
+
*/
|
|
5133
|
+
startupBudgetFor(resume) {
|
|
5134
|
+
const override = this.backend?.getStartupBudgetMs?.({ resume });
|
|
5135
|
+
if (override == null)
|
|
5136
|
+
return undefined;
|
|
5137
|
+
const configured = this.config.startup_timeout_ms;
|
|
5138
|
+
return configured != null ? Math.max(configured, override) : override;
|
|
5139
|
+
}
|
|
5140
|
+
/** Effective wake budget = max(caller timeout, backend resume override, configured startup_timeout_ms). */
|
|
5141
|
+
wakeBudgetMs(timeoutMs) {
|
|
5142
|
+
return Math.max(timeoutMs, this.startupBudgetFor(!this.skipResume) ?? 0, this.config.startup_timeout_ms ?? 0);
|
|
5143
|
+
}
|
|
5144
|
+
/** The fleet-level backend key this instance runs on (matches the lifecycle's backendOf). */
|
|
5145
|
+
backendKey() {
|
|
5146
|
+
return this.runtimeIdentity?.backend ?? this.config.backend ?? this.backend?.binaryName ?? "unknown";
|
|
5147
|
+
}
|
|
5148
|
+
/**
|
|
5149
|
+
* After a failed launch, read the pane for the backend's fleet-wide outage
|
|
5150
|
+
* text (e.g. kiro's `dispatch failure (timeout) … kiro.dev`) and record it.
|
|
5151
|
+
* The lifecycle only learns about outages from RUNNING instances; during a
|
|
5152
|
+
* post-update herd nothing is running yet, so the startup path must report
|
|
5153
|
+
* what it sees or the first instance in would clear its session for nothing.
|
|
5154
|
+
*/
|
|
5155
|
+
async noteStartupPaneForBackendOutage() {
|
|
5156
|
+
if (!this.backendOutage || !this.backend || !this.tmux)
|
|
5157
|
+
return;
|
|
5158
|
+
const outagePattern = this.backend.getErrorPatterns?.().find(p => p.fleetWide && p.type === "network");
|
|
5159
|
+
if (!outagePattern)
|
|
5160
|
+
return;
|
|
5161
|
+
try {
|
|
5162
|
+
const pane = await this.tmux.capturePane();
|
|
5163
|
+
if (!outagePattern.pattern.test(pane))
|
|
5164
|
+
return;
|
|
5165
|
+
const detail = this.resolveErrorMessage(pane, outagePattern);
|
|
5166
|
+
this.backendOutage.record(this.backendKey(), this.name, detail);
|
|
5167
|
+
this.logger.warn({ detail }, "Startup pane shows the backend outage text");
|
|
5168
|
+
}
|
|
5169
|
+
catch { /* capture failed — nothing to learn */ }
|
|
5170
|
+
}
|
|
5171
|
+
/**
|
|
5172
|
+
* Fail the startup WITHOUT touching the session while the backend is known to
|
|
5173
|
+
* be down. The failed window is torn down first: the alternative — a live but
|
|
5174
|
+
* never-ready CLI left behind while the daemon reports `crashed` — would keep
|
|
5175
|
+
* the health monitor from ever seeing a dead pane and retrying.
|
|
5176
|
+
*/
|
|
5177
|
+
async failStartupIfBackendUnreachable() {
|
|
5178
|
+
if (!this.backendOutage?.isActive(this.backendKey()))
|
|
5179
|
+
return;
|
|
5180
|
+
this.logger.warn("Backend unreachable — keeping the session and failing startup for a delayed retry");
|
|
5181
|
+
await this.killProcessTree();
|
|
5182
|
+
await this.tmux.killWindow();
|
|
5183
|
+
throw new BackendUnreachableStartupError(this.backendKey());
|
|
5184
|
+
}
|
|
5185
|
+
/**
|
|
5186
|
+
* Crash-respawn ran into the backend outage: the ordinary path would leave
|
|
5187
|
+
* `crashed` + a paused monitor + nothing scheduled (and count it as a crash).
|
|
5188
|
+
* Hand the instance to the fleet instead — the lifecycle stops this daemon
|
|
5189
|
+
* and the fleet's delayed startup retry re-creates it later with the session
|
|
5190
|
+
* intact. Returns true when someone took the hand-off.
|
|
5191
|
+
*/
|
|
5192
|
+
handOffBackendUnreachableRespawn(err) {
|
|
5193
|
+
if (!(err instanceof BackendUnreachableStartupError))
|
|
5194
|
+
return false;
|
|
5195
|
+
if (this.listenerCount("startup_backend_unreachable") === 0)
|
|
5196
|
+
return false;
|
|
5197
|
+
this.logger.warn({ backend: err.backend }, "Respawn blocked by backend outage — handing off to the fleet's delayed startup retry");
|
|
5198
|
+
this.healthCheckPaused = true;
|
|
5199
|
+
this.emit("startup_backend_unreachable", { name: this.name, backend: err.backend });
|
|
5200
|
+
return true;
|
|
5201
|
+
}
|
|
5202
|
+
/**
|
|
5203
|
+
* Dispose a Daemon whose start() rejected. The lifecycle registers a daemon
|
|
5204
|
+
* only after start() resolves, so nothing else owns this object — and by the
|
|
5205
|
+
* time the spawn fails it has already bound its IPC server (channel.sock),
|
|
5206
|
+
* written daemon.pid and possibly created a window. Left alone, each failed
|
|
5207
|
+
* start leaks one live-but-unreachable server handle (the next start unlinks
|
|
5208
|
+
* the socket path and binds a new one), multiplied by the automatic startup
|
|
5209
|
+
* retries. Idempotent, and fast: no graceful CLI quit, the CLI never reached
|
|
5210
|
+
* its prompt.
|
|
5211
|
+
*/
|
|
5212
|
+
async abortStartup() {
|
|
5213
|
+
if (this.startupAborted)
|
|
5214
|
+
return;
|
|
5215
|
+
this.startupAborted = true;
|
|
5216
|
+
this.freezeRuntimeMonitors();
|
|
5217
|
+
this.pendingIpcRequests.clear();
|
|
5218
|
+
try {
|
|
5219
|
+
await this.killProcessTree();
|
|
5220
|
+
}
|
|
5221
|
+
catch { /* nothing running */ }
|
|
5222
|
+
if (this.tmux) {
|
|
5223
|
+
const windowId = this.tmux.getWindowId();
|
|
5224
|
+
try {
|
|
5225
|
+
await this.tmux.killWindow();
|
|
5226
|
+
}
|
|
5227
|
+
catch { /* window may not exist */ }
|
|
5228
|
+
if (windowId)
|
|
5229
|
+
this.controlClient?.unregisterWindow(windowId);
|
|
5230
|
+
}
|
|
5231
|
+
try {
|
|
5232
|
+
await this.ipcServer?.close();
|
|
5233
|
+
}
|
|
5234
|
+
catch (err) {
|
|
5235
|
+
this.logger.debug({ err }, "IPC server close failed during startup abort");
|
|
5236
|
+
}
|
|
5237
|
+
this.ipcServer = null;
|
|
5238
|
+
for (const file of ["daemon.pid", "window-id"]) {
|
|
5239
|
+
try {
|
|
5240
|
+
unlinkSync(join(this.instanceDir, file));
|
|
5241
|
+
}
|
|
5242
|
+
catch { /* absent */ }
|
|
5243
|
+
}
|
|
5244
|
+
}
|
|
3411
5245
|
/** Kill the entire process tree of the current tmux pane (CLI + MCP server). */
|
|
3412
5246
|
async killProcessTree(signal = "SIGTERM") {
|
|
3413
5247
|
if (!this.tmux)
|
|
@@ -3428,6 +5262,15 @@ export class Daemon extends EventEmitter {
|
|
|
3428
5262
|
* Returns true if CLI is ready, false if it failed or got stuck.
|
|
3429
5263
|
*/
|
|
3430
5264
|
async trySpawn(reuseWindow = false, startupTimeoutMs) {
|
|
5265
|
+
if (!this.spawnGate)
|
|
5266
|
+
return this.trySpawnInsideGate(reuseWindow, startupTimeoutMs);
|
|
5267
|
+
return this.spawnGate.run({
|
|
5268
|
+
instanceName: this.name,
|
|
5269
|
+
workingDirectory: this.config.working_directory,
|
|
5270
|
+
reason: reuseWindow ? "wake" : this.lastSpawnAt > 0 ? "recovery" : "startup",
|
|
5271
|
+
}, () => this.trySpawnInsideGate(reuseWindow, startupTimeoutMs));
|
|
5272
|
+
}
|
|
5273
|
+
async trySpawnInsideGate(reuseWindow = false, startupTimeoutMs) {
|
|
3431
5274
|
const backendConfig = this.buildBackendConfig();
|
|
3432
5275
|
// Compare freshly-built instructions against the last value the agent was
|
|
3433
5276
|
// told about. Computed for ALL backends (not gated by
|
|
@@ -3492,6 +5335,9 @@ export class Daemon extends EventEmitter {
|
|
|
3492
5335
|
const cmd = `${envPrefix} ` + this.backend.buildCommand(backendConfig);
|
|
3493
5336
|
// Ensure tmux session exists (may have been destroyed if all windows died)
|
|
3494
5337
|
await TmuxManager.ensureSession(this.tmuxSessionName);
|
|
5338
|
+
if (this.stormWindow?.observeServerAlive(await TmuxManager.getServerPid(this.tmuxSessionName))) {
|
|
5339
|
+
this.emit("tmux_server_crash", this.name);
|
|
5340
|
+
}
|
|
3495
5341
|
let windowId;
|
|
3496
5342
|
if (reuseWindow) {
|
|
3497
5343
|
this.controlClient?.unregisterWindow(this.tmux.getWindowId());
|
|
@@ -3543,18 +5389,47 @@ export class Daemon extends EventEmitter {
|
|
|
3543
5389
|
// Dismiss confirmation dialogs and verify CLI reached prompt.
|
|
3544
5390
|
// With remain-on-exit, isWindowAlive() returns true even for dead panes,
|
|
3545
5391
|
// but a startup crash would already be caught by waitForOutput/waitForIdle above.
|
|
3546
|
-
|
|
5392
|
+
return this.finishStartupScan(STARTUP_DIALOG_BUDGET_MS);
|
|
5393
|
+
}
|
|
5394
|
+
/**
|
|
5395
|
+
* The last leg of a spawn: confirm the pane is not positively dead, then run
|
|
5396
|
+
* the dialog scan. The old pre-check used tmux's isWindowAlive(), which folds
|
|
5397
|
+
* every query failure into `false` — so one transient tmux error right here
|
|
5398
|
+
* read as "the CLI died", the caller abandoned --resume and cleared the
|
|
5399
|
+
* session, before the tri-state scanner ever got a look. Only a POSITIVE
|
|
5400
|
+
* dead pane fails; an unknown liveness is the scanner's problem (bounded,
|
|
5401
|
+
* session-preserving).
|
|
5402
|
+
*/
|
|
5403
|
+
async finishStartupScan(budgetMs) {
|
|
5404
|
+
if ((await this.paneLiveness()) === "dead")
|
|
3547
5405
|
return false;
|
|
3548
|
-
const ready = await this.dismissDialogsUntilReady(
|
|
3549
|
-
|
|
5406
|
+
const ready = await this.dismissDialogsUntilReady(budgetMs);
|
|
5407
|
+
// A fatal startup screen returns true (the spawn is over — respawning
|
|
5408
|
+
// would only re-show the same screen) but is NOT ready-for-delivery:
|
|
5409
|
+
// fatalStartupBlocked gates every pane write in deliverMessage.
|
|
5410
|
+
if (ready && !this.fatalStartupBlocked)
|
|
3550
5411
|
this.firstDeliveryDelay.recordReady();
|
|
3551
5412
|
return ready;
|
|
3552
5413
|
}
|
|
3553
5414
|
/**
|
|
3554
|
-
*
|
|
3555
|
-
* and return true once CLI
|
|
5415
|
+
* Poll the pane within a wall-clock budget, dismiss the confirmation dialogs
|
|
5416
|
+
* the backend knows, and return true once the CLI shows a ready prompt with
|
|
5417
|
+
* no dialog on screen for two consecutive polls.
|
|
5418
|
+
*
|
|
5419
|
+
* Polls are spaced by `pollMs` and every key sequence / render wait counts
|
|
5420
|
+
* against the budget: Claude paints its session-resume prompt only after
|
|
5421
|
+
* loading the session, i.e. AFTER the first quiet moment, and three
|
|
5422
|
+
* back-to-back captures used to miss it — the scan then "assumed ok", the
|
|
5423
|
+
* `❯` inside the dialog satisfied the ready pattern, and the instance sat on
|
|
5424
|
+
* the dialog with deliveries free to press Enter into it.
|
|
5425
|
+
*
|
|
5426
|
+
* Only hard evidence returns false: a dead window or "command not found".
|
|
5427
|
+
* A transient capture failure is logged and retried; exhausting the budget is
|
|
5428
|
+
* logged at WARN and returns true — false makes the caller abandon --resume
|
|
5429
|
+
* and clear the session, which is the loss this code exists to prevent. The
|
|
5430
|
+
* delivery gate keeps refusing pastes while a blocking dialog is visible.
|
|
3556
5431
|
*/
|
|
3557
|
-
async dismissDialogsUntilReady(
|
|
5432
|
+
async dismissDialogsUntilReady(budgetMs, pollMs = STARTUP_DIALOG_POLL_MS) {
|
|
3558
5433
|
// Backend-specific startup dialogs, with hardcoded fallback for backward compat
|
|
3559
5434
|
const startupDialogs = this.backend?.getStartupDialogs?.() ?? [
|
|
3560
5435
|
{ pattern: /[❯›]\s*\d+\.\s*No/m, keys: ["Down", "Enter"], description: "Confirmation dialog — navigate past No" },
|
|
@@ -3562,14 +5437,76 @@ export class Daemon extends EventEmitter {
|
|
|
3562
5437
|
{ pattern: /No, exit|No, quit|Don't trust|I accept|I trust|Yes, continue|Trust folder/i, keys: ["Enter"], description: "Generic confirmation dialog" },
|
|
3563
5438
|
{ pattern: /Resume Session/i, keys: ["Escape"], description: "Resume session picker — start fresh" },
|
|
3564
5439
|
];
|
|
3565
|
-
|
|
5440
|
+
const deadline = Date.now() + budgetMs;
|
|
5441
|
+
const remaining = () => deadline - Date.now();
|
|
5442
|
+
const sleep = async () => { if (remaining() > 0)
|
|
5443
|
+
await new Promise(r => setTimeout(r, Math.min(pollMs, Math.max(remaining(), 0)))); };
|
|
5444
|
+
let cleanReadyPolls = 0;
|
|
5445
|
+
let lastDialog = null;
|
|
5446
|
+
let captureFailures = 0;
|
|
5447
|
+
let attempts = 0;
|
|
5448
|
+
do {
|
|
5449
|
+
attempts++;
|
|
5450
|
+
let pane;
|
|
5451
|
+
try {
|
|
5452
|
+
pane = await this.tmux.capturePane();
|
|
5453
|
+
}
|
|
5454
|
+
catch (err) {
|
|
5455
|
+
// Transient tmux trouble is not evidence about the CLI. Returning false
|
|
5456
|
+
// here would clear the session; retry within the budget instead.
|
|
5457
|
+
captureFailures++;
|
|
5458
|
+
this.logger.warn({ err, captureFailures }, "capture-pane failed during the startup dialog scan — retrying");
|
|
5459
|
+
// Only a POSITIVE dead pane ends the scan; an unanswerable liveness
|
|
5460
|
+
// query is the same transient trouble and must not clear the session.
|
|
5461
|
+
if ((await this.paneLiveness()) === "dead")
|
|
5462
|
+
return false;
|
|
5463
|
+
await sleep();
|
|
5464
|
+
continue;
|
|
5465
|
+
}
|
|
3566
5466
|
try {
|
|
3567
|
-
const pane = await this.tmux.capturePane();
|
|
3568
5467
|
// Try each startup dialog pattern before checking ready state
|
|
3569
5468
|
let matched = false;
|
|
5469
|
+
lastDialog = null;
|
|
3570
5470
|
for (const dialog of startupDialogs) {
|
|
3571
|
-
if (
|
|
3572
|
-
|
|
5471
|
+
if (Daemon.dialogMatches(dialog, pane)) {
|
|
5472
|
+
lastDialog = dialog;
|
|
5473
|
+
cleanReadyPolls = 0;
|
|
5474
|
+
// Start the parked clock for every delivery-blocking dialog, exact
|
|
5475
|
+
// ones included: if the auto-dismiss keeps failing here, the human
|
|
5476
|
+
// report should count from first sight, not from the runtime scan.
|
|
5477
|
+
if (!dialog.fatal && this.deliveryBlockingDialogs().some(d => Daemon.dialogKey(d) === Daemon.dialogKey(dialog))) {
|
|
5478
|
+
this.trackDialogParked(dialog);
|
|
5479
|
+
}
|
|
5480
|
+
// A fatal startup screen cannot be dismissed by keypresses (e.g. a
|
|
5481
|
+
// corrupt claude.json modal offering only "exit" / "reset config").
|
|
5482
|
+
// It must be caught HERE and not fall through to the ready check:
|
|
5483
|
+
// the modal contains the `❯` selector the ready pattern matches, so
|
|
5484
|
+
// it would otherwise pass as ready and queued messages would be
|
|
5485
|
+
// typed into the modal. Report the error (pause stops delivery) and
|
|
5486
|
+
// stop the retry loop — like the sign-in screen case below, only a
|
|
5487
|
+
// human fixing the config ends this state.
|
|
5488
|
+
if (dialog.fatal) {
|
|
5489
|
+
// Block deliveries BEFORE the emit: the pty_error pause handler
|
|
5490
|
+
// is async, and a queued message would otherwise win the race
|
|
5491
|
+
// and be pasted into the modal.
|
|
5492
|
+
this.fatalStartupBlocked = true;
|
|
5493
|
+
if (!this.fatalStartupReported) {
|
|
5494
|
+
this.fatalStartupReported = true;
|
|
5495
|
+
this.logger.warn({ description: dialog.description }, "Fatal startup screen detected — reporting instead of dismissing");
|
|
5496
|
+
this.emit("pty_error", { name: this.name, ...dialog.fatal });
|
|
5497
|
+
}
|
|
5498
|
+
return true;
|
|
5499
|
+
}
|
|
5500
|
+
if (dialog.holdOnly) {
|
|
5501
|
+
// Recognised but deliberately not answered (see RuntimeDialog.holdOnly).
|
|
5502
|
+
if (this.dialogParkedKey !== Daemon.dialogKey(dialog)) {
|
|
5503
|
+
this.logger.warn({ description: dialog.description }, "Startup dialog held — not auto-answering, deliveries stay blocked");
|
|
5504
|
+
}
|
|
5505
|
+
this.trackDialogParked(dialog);
|
|
5506
|
+
matched = true;
|
|
5507
|
+
break;
|
|
5508
|
+
}
|
|
5509
|
+
this.logger.info(`Dismissing startup dialog: ${dialog.description}`);
|
|
3573
5510
|
// Restart is exactly when inbound messages pile up, and nothing gates
|
|
3574
5511
|
// delivery on `spawning`. Take the pane lock for the key sequence so a
|
|
3575
5512
|
// queued message cannot be pasted into a half-dismissed trust dialog.
|
|
@@ -3584,35 +5521,82 @@ export class Daemon extends EventEmitter {
|
|
|
3584
5521
|
await new Promise(r => setTimeout(r, 200));
|
|
3585
5522
|
}
|
|
3586
5523
|
});
|
|
3587
|
-
// Wait for next screen to render
|
|
5524
|
+
// Wait for next screen to render — bounded by what is left of the budget.
|
|
5525
|
+
const renderWait = Math.max(0, Math.min(10_000, remaining()));
|
|
3588
5526
|
if (this.controlClient) {
|
|
3589
5527
|
const wid = readFileSync(join(this.instanceDir, "window-id"), "utf-8").trim();
|
|
3590
|
-
await this.controlClient.waitForIdle(wid,
|
|
5528
|
+
await this.controlClient.waitForIdle(wid, renderWait);
|
|
3591
5529
|
}
|
|
3592
5530
|
else {
|
|
3593
|
-
await new Promise(r => setTimeout(r, 3_000));
|
|
5531
|
+
await new Promise(r => setTimeout(r, Math.min(3_000, renderWait)));
|
|
3594
5532
|
}
|
|
3595
|
-
if (
|
|
5533
|
+
if ((await this.paneLiveness()) === "dead")
|
|
3596
5534
|
return false;
|
|
3597
5535
|
matched = true;
|
|
3598
5536
|
break;
|
|
3599
5537
|
}
|
|
3600
5538
|
}
|
|
3601
|
-
if (matched)
|
|
5539
|
+
if (matched) {
|
|
5540
|
+
if (lastDialog?.holdOnly)
|
|
5541
|
+
await sleep();
|
|
3602
5542
|
continue;
|
|
3603
|
-
|
|
3604
|
-
if (this.
|
|
5543
|
+
}
|
|
5544
|
+
if (this.dialogParkedSince !== 0 && !this.deliveryBlockingDialogs().some(d => Daemon.dialogMatches(d, pane)))
|
|
5545
|
+
this.trackDialogParked(null);
|
|
5546
|
+
// CLI is ready (pattern defined by each backend). Require it on two
|
|
5547
|
+
// consecutive polls: the first ready frame is also the moment a late
|
|
5548
|
+
// dialog is about to be painted over it.
|
|
5549
|
+
if (this.backend.getReadyPattern().test(pane)) {
|
|
5550
|
+
cleanReadyPolls++;
|
|
5551
|
+
if (cleanReadyPolls >= 2 || remaining() <= 0) {
|
|
5552
|
+
// A real ready prompt (with no fatal dialog on screen — those are
|
|
5553
|
+
// matched above, before this check) means the fatal screen is gone,
|
|
5554
|
+
// e.g. the user fixed the config and the instance respawned.
|
|
5555
|
+
this.fatalStartupBlocked = false;
|
|
5556
|
+
return true;
|
|
5557
|
+
}
|
|
5558
|
+
await sleep();
|
|
5559
|
+
continue;
|
|
5560
|
+
}
|
|
5561
|
+
cleanReadyPolls = 0;
|
|
5562
|
+
// A CLI parked at its own sign-in screen is an AUTH incident: no dialog
|
|
5563
|
+
// key can dismiss it and no ready pattern will ever appear. Report it as
|
|
5564
|
+
// an auth error (the lifecycle double-checks with the token-free probe)
|
|
5565
|
+
// instead of letting it decay into crash/hang/MCP-died noise, and stop
|
|
5566
|
+
// the retry loop — only re-login (see /login) ends this state.
|
|
5567
|
+
const loginScreen = LOGIN_FLOWS[this.config.backend ?? "claude-code"]?.loginScreenPattern;
|
|
5568
|
+
if (loginScreen?.test(pane) && !this.loginScreenReported) {
|
|
5569
|
+
this.loginScreenReported = true;
|
|
5570
|
+
this.authFailureUnresolved = true;
|
|
5571
|
+
this.logger.warn({ backend: this.config.backend }, "CLI is waiting at its sign-in screen — reporting auth error");
|
|
5572
|
+
this.emit("pty_error", {
|
|
5573
|
+
name: this.name,
|
|
5574
|
+
type: "auth_error",
|
|
5575
|
+
action: "pause",
|
|
5576
|
+
message: "CLI is waiting at its sign-in screen — credentials are missing or expired",
|
|
5577
|
+
});
|
|
3605
5578
|
return true;
|
|
5579
|
+
}
|
|
3606
5580
|
// Fatal: command not found (must match full phrase to avoid false positives
|
|
3607
5581
|
// like Kiro's "agent X not found, using default")
|
|
3608
5582
|
if (/command not found|: not found$/m.test(pane))
|
|
3609
5583
|
return false;
|
|
3610
5584
|
}
|
|
3611
|
-
catch {
|
|
3612
|
-
|
|
5585
|
+
catch (err) {
|
|
5586
|
+
// Key sends / isWindowAlive failing: same rule — log, retry within the budget.
|
|
5587
|
+
this.logger.warn({ err }, "startup dialog scan step failed — retrying");
|
|
5588
|
+
if ((await this.paneLiveness()) === "dead")
|
|
5589
|
+
return false;
|
|
3613
5590
|
}
|
|
5591
|
+
await sleep();
|
|
5592
|
+
} while (remaining() > 0);
|
|
5593
|
+
// Budget exhausted. Never silently: say which way it went.
|
|
5594
|
+
if (lastDialog) {
|
|
5595
|
+
this.logger.warn({ description: lastDialog.description, attempts, budgetMs }, "Startup scan exhausted with a dialog still on screen — reporting the CLI alive; deliveries stay blocked until it is answered");
|
|
5596
|
+
}
|
|
5597
|
+
else {
|
|
5598
|
+
this.logger.warn({ attempts, budgetMs, captureFailures }, "Startup scan exhausted without a ready prompt or a known dialog — assuming ready (unknown CLI screen)");
|
|
3614
5599
|
}
|
|
3615
|
-
// Exhausted attempts — assume ok for unknown CLI prompts
|
|
3616
5600
|
return true;
|
|
3617
5601
|
}
|
|
3618
5602
|
/**
|