@songsid/agend 2.1.4-beta.4 → 2.1.4-beta.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/README.zh-TW.md +1 -1
- package/dist/agent-endpoint.js +1 -1
- package/dist/agent-endpoint.js.map +1 -1
- package/dist/backend/antigravity.d.ts +9 -5
- package/dist/backend/antigravity.js +156 -31
- package/dist/backend/antigravity.js.map +1 -1
- package/dist/backend/claude-code.d.ts +62 -23
- package/dist/backend/claude-code.js +296 -43
- package/dist/backend/claude-code.js.map +1 -1
- package/dist/backend/codex.d.ts +18 -0
- package/dist/backend/codex.js +148 -8
- package/dist/backend/codex.js.map +1 -1
- package/dist/backend/gemini-cli.js +2 -2
- package/dist/backend/gemini-cli.js.map +1 -1
- package/dist/backend/grok.js +2 -2
- package/dist/backend/grok.js.map +1 -1
- package/dist/backend/kiro.d.ts +2 -0
- package/dist/backend/kiro.js +24 -3
- package/dist/backend/kiro.js.map +1 -1
- package/dist/backend/opencode.d.ts +27 -11
- package/dist/backend/opencode.js +65 -41
- package/dist/backend/opencode.js.map +1 -1
- package/dist/backend/types.d.ts +45 -3
- package/dist/backend/types.js +12 -1
- package/dist/backend/types.js.map +1 -1
- package/dist/channel/adapters/discord.d.ts +44 -2
- package/dist/channel/adapters/discord.js +526 -78
- package/dist/channel/adapters/discord.js.map +1 -1
- package/dist/channel/adapters/telegram.d.ts +30 -0
- package/dist/channel/adapters/telegram.js +212 -18
- package/dist/channel/adapters/telegram.js.map +1 -1
- package/dist/channel/agy-mcp-launcher.d.ts +2 -0
- package/dist/channel/agy-mcp-launcher.js +28 -0
- package/dist/channel/agy-mcp-launcher.js.map +1 -0
- package/dist/channel/ipc-bridge.d.ts +2 -1
- package/dist/channel/ipc-bridge.js +33 -4
- package/dist/channel/ipc-bridge.js.map +1 -1
- package/dist/channel/mcp-server.js +28 -25
- package/dist/channel/mcp-server.js.map +1 -1
- package/dist/channel/mcp-tools.js +4 -2
- package/dist/channel/mcp-tools.js.map +1 -1
- package/dist/channel/message-queue.js +62 -1
- package/dist/channel/message-queue.js.map +1 -1
- package/dist/channel/types.d.ts +31 -1
- package/dist/classic-channel-manager.d.ts +40 -3
- package/dist/classic-channel-manager.js +218 -31
- package/dist/classic-channel-manager.js.map +1 -1
- package/dist/cli.js +109 -37
- package/dist/cli.js.map +1 -1
- package/dist/config-validator.js +48 -7
- package/dist/config-validator.js.map +1 -1
- package/dist/config.d.ts +4 -0
- package/dist/config.js +12 -1
- package/dist/config.js.map +1 -1
- package/dist/cross-instance-envelope.d.ts +6 -0
- package/dist/cross-instance-envelope.js +30 -0
- package/dist/cross-instance-envelope.js.map +1 -0
- package/dist/daemon.d.ts +319 -9
- package/dist/daemon.js +1402 -255
- package/dist/daemon.js.map +1 -1
- package/dist/doctor.d.ts +41 -0
- package/dist/doctor.js +267 -0
- package/dist/doctor.js.map +1 -0
- package/dist/fleet-context.d.ts +47 -0
- package/dist/fleet-manager.d.ts +265 -11
- package/dist/fleet-manager.js +2273 -455
- package/dist/fleet-manager.js.map +1 -1
- package/dist/fleet-yaml-slim.d.ts +10 -0
- package/dist/fleet-yaml-slim.js +59 -0
- package/dist/fleet-yaml-slim.js.map +1 -0
- package/dist/general-knowledge/skills/backend-providers/SKILL.md +114 -0
- package/dist/general-knowledge/skills/cross-instance-messaging/SKILL.md +3 -1
- package/dist/general-knowledge/skills/delegation-playbook/SKILL.md +52 -0
- package/dist/general-knowledge/skills/development-workflow/SKILL.md +28 -0
- package/dist/general-knowledge/skills/fleet-config/SKILL.md +1 -0
- package/dist/general-knowledge/skills/fleet-health/SKILL.md +1 -0
- package/dist/general-knowledge/skills/fleet-restart/SKILL.md +1 -0
- package/dist/general-knowledge/skills/instance-lifecycle/SKILL.md +1 -0
- package/dist/general-knowledge/skills/model-discovery/SKILL.md +64 -16
- package/dist/general-knowledge/skills/multi-channel/SKILL.md +1 -0
- package/dist/general-knowledge/skills/scheduling/SKILL.md +1 -0
- package/dist/general-knowledge/skills/session-management/SKILL.md +1 -0
- package/dist/general-knowledge/skills/tui-effort/SKILL.md +1 -0
- package/dist/general-knowledge/skills/worker-collaboration/SKILL.md +30 -0
- package/dist/instance-lifecycle.d.ts +67 -0
- package/dist/instance-lifecycle.js +273 -8
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/instructions.d.ts +12 -0
- package/dist/instructions.js +35 -1
- package/dist/instructions.js.map +1 -1
- package/dist/locale.js +966 -288
- package/dist/locale.js.map +1 -1
- package/dist/login-flows.d.ts +91 -0
- package/dist/login-flows.js +170 -0
- package/dist/login-flows.js.map +1 -0
- package/dist/login-manager.d.ts +63 -0
- package/dist/login-manager.js +134 -0
- package/dist/login-manager.js.map +1 -0
- package/dist/network-family.d.ts +18 -0
- package/dist/network-family.js +20 -0
- package/dist/network-family.js.map +1 -0
- package/dist/outbound-handlers.d.ts +8 -0
- package/dist/outbound-handlers.js +101 -18
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/outbound-schemas.d.ts +7 -1
- package/dist/outbound-schemas.js +9 -0
- package/dist/outbound-schemas.js.map +1 -1
- package/dist/reply-dedup.d.ts +7 -8
- package/dist/reply-dedup.js +0 -0
- package/dist/reply-dedup.js.map +1 -1
- package/dist/restart-progress.d.ts +2 -0
- package/dist/restart-progress.js +3 -0
- package/dist/restart-progress.js.map +1 -1
- package/dist/scheduler/db.d.ts +12 -0
- package/dist/scheduler/db.js +59 -0
- package/dist/scheduler/db.js.map +1 -1
- package/dist/service-installer.d.ts +11 -0
- package/dist/service-installer.js +84 -18
- package/dist/service-installer.js.map +1 -1
- package/dist/settings-api.js +1 -1
- package/dist/settings-api.js.map +1 -1
- package/dist/setup-wizard.js +2 -2
- package/dist/setup-wizard.js.map +1 -1
- package/dist/spawn-gate.d.ts +30 -0
- package/dist/spawn-gate.js +79 -0
- package/dist/spawn-gate.js.map +1 -0
- package/dist/storm-window.d.ts +83 -0
- package/dist/storm-window.js +251 -0
- package/dist/storm-window.js.map +1 -0
- package/dist/tips.d.ts +44 -0
- package/dist/tips.js +355 -0
- package/dist/tips.js.map +1 -0
- package/dist/tmux-manager.d.ts +16 -0
- package/dist/tmux-manager.js +110 -25
- package/dist/tmux-manager.js.map +1 -1
- package/dist/tool-progress.d.ts +40 -0
- package/dist/tool-progress.js +289 -0
- package/dist/tool-progress.js.map +1 -0
- package/dist/topic-commands.d.ts +42 -4
- package/dist/topic-commands.js +361 -67
- package/dist/topic-commands.js.map +1 -1
- package/dist/transcript-monitor.d.ts +15 -2
- package/dist/transcript-monitor.js +63 -17
- package/dist/transcript-monitor.js.map +1 -1
- package/dist/transcript-sources.d.ts +131 -0
- package/dist/transcript-sources.js +580 -0
- package/dist/transcript-sources.js.map +1 -0
- package/dist/types.d.ts +13 -0
- package/dist/ui/dashboard.html +55 -32
- package/dist/ui/settings.html +200 -60
- package/dist/ui/view.html +147 -30
- package/dist/usage/format-rich.js +24 -23
- package/dist/usage/format-rich.js.map +1 -1
- package/dist/usage/i18n-keys.d.ts +7 -0
- package/dist/usage/i18n-keys.js +34 -0
- package/dist/usage/i18n-keys.js.map +1 -0
- package/dist/usage/i18n.d.ts +5 -0
- package/dist/usage/i18n.js +27 -0
- package/dist/usage/i18n.js.map +1 -0
- package/dist/usage/providers.d.ts +19 -0
- package/dist/usage/providers.js +150 -74
- package/dist/usage/providers.js.map +1 -1
- package/dist/usage/usage-api.d.ts +8 -2
- package/dist/usage/usage-api.js +45 -23
- package/dist/usage/usage-api.js.map +1 -1
- package/dist/workflow-templates/default.md +2 -1
- package/package.json +2 -2
package/dist/daemon.js
CHANGED
|
@@ -8,6 +8,9 @@ import { mcpServerState } from "./mcp-liveness.js";
|
|
|
8
8
|
import { clearPausedMarker, writePausedMarker } from "./pause-marker.js";
|
|
9
9
|
import { TmuxManager, resolveTmuxLogicalSize } from "./tmux-manager.js";
|
|
10
10
|
import { TranscriptMonitor } from "./transcript-monitor.js";
|
|
11
|
+
import { createTranscriptSource } from "./transcript-sources.js";
|
|
12
|
+
import { LOGIN_FLOWS } from "./login-flows.js";
|
|
13
|
+
import { ProgressAccumulator, summarizeProgress } from "./tool-progress.js";
|
|
11
14
|
import { ContextGuardian } from "./context-guardian.js";
|
|
12
15
|
import { IpcServer } from "./channel/ipc-bridge.js";
|
|
13
16
|
import { daemonBudgetMs } from "./channel/ipc-timeouts.js";
|
|
@@ -19,6 +22,7 @@ import { HangDetector } from "./hang-detector.js";
|
|
|
19
22
|
import { writeSecretFile } from "./secret-file.js";
|
|
20
23
|
import { PaneWriteLock } from "./pane-write-lock.js";
|
|
21
24
|
import { buildFleetInstructions } from "./instructions.js";
|
|
25
|
+
import { formatCrossInstanceInboundMessage, renderCrossInstanceHandoffMetadata, } from "./cross-instance-envelope.js";
|
|
22
26
|
const __filename = fileURLToPath(import.meta.url);
|
|
23
27
|
const __dirname = dirname(__filename);
|
|
24
28
|
// Tool routing sets — module-level to avoid re-creation on every handleToolCall
|
|
@@ -45,6 +49,10 @@ export function buildInstructionReloadNotice(binaryName, instanceName, instanceD
|
|
|
45
49
|
export const DEFAULT_STUCK_TIMEOUT_MS = 10 * 60_000;
|
|
46
50
|
export const DEFAULT_STATE_IDLE_DEBOUNCE_MS = 2_000;
|
|
47
51
|
export const DEFAULT_STATE_SAFETY_SWEEP_MS = 60_000;
|
|
52
|
+
/** Coalesce one cosmetic-redraw burst before comparing the visible pane. */
|
|
53
|
+
export const PERIODIC_REDRAW_PROBE_MS = 25;
|
|
54
|
+
/** A foreground server/port-forward should hand control back or be acknowledged. */
|
|
55
|
+
export const DEFAULT_BLOCKING_PROCESS_GRACE_MS = 2 * 60_000;
|
|
48
56
|
const LAST_INBOUND_FILE = "last-inbound-at";
|
|
49
57
|
/** Minimum gap between "health check is failing" notifications for one instance. */
|
|
50
58
|
const HEALTH_ERROR_NOTIFY_INTERVAL_MS = 10 * 60_000;
|
|
@@ -131,19 +139,7 @@ export function writeLastInboundAt(instanceDir, timestamp) {
|
|
|
131
139
|
* message_id line.
|
|
132
140
|
*/
|
|
133
141
|
export function renderHandoffMetadata(meta) {
|
|
134
|
-
|
|
135
|
-
const add = (label, value) => {
|
|
136
|
-
if (value && value.trim())
|
|
137
|
-
rows.push(`${label}: ${value.trim()}`);
|
|
138
|
-
};
|
|
139
|
-
add("message_id", meta.message_id);
|
|
140
|
-
add("correlation_id", meta.correlation_id);
|
|
141
|
-
add("request_kind", meta.request_kind);
|
|
142
|
-
add("task_summary", meta.task_summary);
|
|
143
|
-
add("working_directory", meta.working_directory);
|
|
144
|
-
add("branch", meta.branch);
|
|
145
|
-
add("attachment_file_id", meta.attachment_file_id);
|
|
146
|
-
return rows.length ? `\n(${rows.join(" | ")})` : "";
|
|
142
|
+
return renderCrossInstanceHandoffMetadata(meta);
|
|
147
143
|
}
|
|
148
144
|
/** Headless inactivity timer used by the daemon and unit tests. */
|
|
149
145
|
export class AutoPauseController {
|
|
@@ -157,6 +153,11 @@ export class AutoPauseController {
|
|
|
157
153
|
recordActivity(now = Date.now()) {
|
|
158
154
|
this.lastActivityAt = now;
|
|
159
155
|
}
|
|
156
|
+
/** Apply a new idle threshold and start a fresh countdown from this update. */
|
|
157
|
+
reconfigure(thresholdMs, now = Date.now()) {
|
|
158
|
+
this.thresholdMs = Math.max(0, thresholdMs);
|
|
159
|
+
this.lastActivityAt = now;
|
|
160
|
+
}
|
|
160
161
|
observe(state, now = Date.now()) {
|
|
161
162
|
if (this.pausedAt !== null || this.thresholdMs <= 0)
|
|
162
163
|
return false;
|
|
@@ -279,9 +280,11 @@ export class PaneStateMachine {
|
|
|
279
280
|
* working" and a finished turn would not be seen as idle until the next 60s
|
|
280
281
|
* safety sweep. Output timestamps are the honest liveness signal here, and
|
|
281
282
|
* the caller has them.
|
|
283
|
+
* @param opts.forceBusy the backend has positively identified a still-running
|
|
284
|
+
* foreground tool even though its TUI keeps an old ready footer visible.
|
|
282
285
|
*/
|
|
283
286
|
observe(pane, now = Date.now(), opts = {}) {
|
|
284
|
-
const { settled = false, changeAt = now } = opts;
|
|
287
|
+
const { settled = false, changeAt = now, forceBusy = false } = opts;
|
|
285
288
|
const paneHash = createHash("sha256").update(pane).digest("hex");
|
|
286
289
|
const firstObservation = this.lastPaneHash === null;
|
|
287
290
|
const paneChanged = this.lastPaneHash !== paneHash;
|
|
@@ -290,7 +293,10 @@ export class PaneStateMachine {
|
|
|
290
293
|
this.lastPaneChangeAt = changeAt;
|
|
291
294
|
}
|
|
292
295
|
this.lastObservedAt = now;
|
|
293
|
-
|
|
296
|
+
// Some TUIs keep their ready footer visible while a foreground tool owns
|
|
297
|
+
// stdin. Backends can positively identify that tool from the pane; in that
|
|
298
|
+
// case the old prompt must not clear pending work or retire Cancel.
|
|
299
|
+
const ready = !forceBusy && this.isReady(pane);
|
|
294
300
|
const nextState = firstObservation
|
|
295
301
|
? ready ? "idle" : "working"
|
|
296
302
|
: paneChanged && !settled
|
|
@@ -529,6 +535,66 @@ export function extractProxyReplyText(pane, opts = {}) {
|
|
|
529
535
|
// The end of the turn is the answer — when over budget, keep the tail.
|
|
530
536
|
return text.length > maxChars ? text.slice(text.length - maxChars) : text;
|
|
531
537
|
}
|
|
538
|
+
/**
|
|
539
|
+
* Find the narrow Claude XML failure signature reported in #648.
|
|
540
|
+
*
|
|
541
|
+
* This intentionally runs only at the daemon's busy→idle edge (the caller
|
|
542
|
+
* enforces that) and only scans output after the current inbound marker. A
|
|
543
|
+
* partial tool call while Claude is still streaming is therefore never acted
|
|
544
|
+
* on. The extracted argument goes through the same credential redaction as the
|
|
545
|
+
* dead-MCP pane proxy before it can leave the host.
|
|
546
|
+
*/
|
|
547
|
+
export function detectMalformedClaudeToolCall(pane, opts = {}) {
|
|
548
|
+
let clean = pane
|
|
549
|
+
.replace(/\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])/g, "")
|
|
550
|
+
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "");
|
|
551
|
+
const marker = opts.inboundMarker?.trim();
|
|
552
|
+
if (marker) {
|
|
553
|
+
const markerAt = clean.lastIndexOf(marker);
|
|
554
|
+
if (markerAt >= 0)
|
|
555
|
+
clean = clean.slice(markerAt + marker.length);
|
|
556
|
+
}
|
|
557
|
+
const lines = clean.split(/\r?\n/).slice(-(opts.maxLines ?? 120));
|
|
558
|
+
const tail = lines.join("\n");
|
|
559
|
+
const closingTags = [...tail.matchAll(/<\/(?:parameter|invoke|function_calls)>/gi)];
|
|
560
|
+
if (closingTags.length === 0)
|
|
561
|
+
return null;
|
|
562
|
+
const parameterOpen = /<parameter\b[^>]*\bname\s*=\s*["']text["'][^>]*>/gi;
|
|
563
|
+
const parameterOpens = [...tail.matchAll(parameterOpen)];
|
|
564
|
+
const open = parameterOpens.at(-1);
|
|
565
|
+
if (!open || open.index === undefined) {
|
|
566
|
+
// Base stale suppression on the XML artifact itself, not unrelated pane
|
|
567
|
+
// redraws appended after it. The occurrence counts still distinguish a
|
|
568
|
+
// genuinely new identical malformed call while both remain visible.
|
|
569
|
+
const signature = createHash("sha256")
|
|
570
|
+
.update(`0:${closingTags.length}:${closingTags.map(tag => tag[0].toLowerCase()).join(":")}`)
|
|
571
|
+
.digest("hex");
|
|
572
|
+
return { signature, text: null };
|
|
573
|
+
}
|
|
574
|
+
const contentAt = open.index + open[0].length;
|
|
575
|
+
const closeAt = tail.slice(contentAt).search(/<\/parameter>/i);
|
|
576
|
+
if (closeAt < 0) {
|
|
577
|
+
const signature = createHash("sha256")
|
|
578
|
+
.update(`${parameterOpens.length}:${closingTags.length}:${open[0]}`)
|
|
579
|
+
.digest("hex");
|
|
580
|
+
return { signature, text: null };
|
|
581
|
+
}
|
|
582
|
+
const fragment = tail.slice(contentAt, contentAt + closeAt);
|
|
583
|
+
// Fingerprint only the malformed XML artifact; never put its contents in logs.
|
|
584
|
+
const signature = createHash("sha256")
|
|
585
|
+
.update(`${parameterOpens.length}:${closingTags.length}:${open[0]}${fragment}</parameter>`)
|
|
586
|
+
.digest("hex");
|
|
587
|
+
let text = sanitizePaneTail(fragment, opts.maxLines ?? 120)
|
|
588
|
+
.join("\n")
|
|
589
|
+
.replace(/\n{3,}/g, "\n\n")
|
|
590
|
+
.trim();
|
|
591
|
+
if (text.replace(/[^\p{L}\p{N}]/gu, "").length < 2)
|
|
592
|
+
return { signature, text: null };
|
|
593
|
+
const maxChars = opts.maxChars ?? 12_000;
|
|
594
|
+
if (text.length > maxChars)
|
|
595
|
+
text = text.slice(0, maxChars);
|
|
596
|
+
return { signature, text };
|
|
597
|
+
}
|
|
532
598
|
const INTERACTIVE_PROMPT_PATTERNS = [
|
|
533
599
|
{ kind: "sudo_password", pattern: /^\s*\[sudo\]\s+password\s+for\s+[^:\n]+:\s*$/im },
|
|
534
600
|
// macOS sudo and several package installers use this exact no-echo prompt.
|
|
@@ -591,6 +657,69 @@ export class InteractivePromptDetector {
|
|
|
591
657
|
this.notifiedSignature = null;
|
|
592
658
|
}
|
|
593
659
|
}
|
|
660
|
+
const BLOCKING_PROCESS_PATTERNS = [
|
|
661
|
+
// kubectl port-forward (including a backgrounded child whose inherited
|
|
662
|
+
// stdout keeps the parent shell tool open).
|
|
663
|
+
/^\s*(?:Forwarding from\s+(?:127\.0\.0\.1|localhost|\[::1\]):\d+\s+->\s+\d+|Handling connection for\s+\d+)\s*$/im,
|
|
664
|
+
// Common development servers. Keep these line-shaped and require an address
|
|
665
|
+
// or port so prose such as "the server is running" does not arm the detector.
|
|
666
|
+
/^\s*(?:(?:INFO:\s*)?Uvicorn running on|Serving HTTP on|Listening on|Server (?:is )?(?:listening|running) (?:at|on)|Application startup complete[^\n]*(?:port|https?:\/\/))[^\n]*(?:https?:\/\/|\b(?:localhost|127\.0\.0\.1|0\.0\.0\.0|\[::\]|port)\b)[^\n]*$/im,
|
|
667
|
+
];
|
|
668
|
+
/**
|
|
669
|
+
* Detect a foreground shell tool which has become a long-lived server.
|
|
670
|
+
*
|
|
671
|
+
* Unlike the generic pane-stuck clock, this clock deliberately survives new
|
|
672
|
+
* output: access logs and `Handling connection` lines are exactly why a
|
|
673
|
+
* foreground server can remain blocked forever without ever looking silent.
|
|
674
|
+
* Once a server marker is seen, the still-running tool identity is the stable
|
|
675
|
+
* signal; the marker may scroll out of the 32-row pane.
|
|
676
|
+
*/
|
|
677
|
+
export class BlockingProcessDetector {
|
|
678
|
+
graceMs;
|
|
679
|
+
activity = null;
|
|
680
|
+
evidence = null;
|
|
681
|
+
detectedAt = 0;
|
|
682
|
+
notified = false;
|
|
683
|
+
constructor(graceMs = DEFAULT_BLOCKING_PROCESS_GRACE_MS) {
|
|
684
|
+
this.graceMs = graceMs;
|
|
685
|
+
}
|
|
686
|
+
observe(pane, activity, now = Date.now()) {
|
|
687
|
+
const shellActivity = activity && /^(?:shell|bash|exec_command|terminal|command)(?::|$)/i.test(activity);
|
|
688
|
+
if (!shellActivity) {
|
|
689
|
+
this.reset();
|
|
690
|
+
return null;
|
|
691
|
+
}
|
|
692
|
+
if (activity !== this.activity) {
|
|
693
|
+
this.reset();
|
|
694
|
+
this.activity = activity;
|
|
695
|
+
}
|
|
696
|
+
if (!this.evidence) {
|
|
697
|
+
const tail = sanitizePaneTail(pane, 40).join("\n");
|
|
698
|
+
for (const pattern of BLOCKING_PROCESS_PATTERNS) {
|
|
699
|
+
const match = tail.match(pattern);
|
|
700
|
+
if (!match)
|
|
701
|
+
continue;
|
|
702
|
+
this.evidence = match[0].trim().slice(0, 200);
|
|
703
|
+
this.detectedAt = now;
|
|
704
|
+
break;
|
|
705
|
+
}
|
|
706
|
+
}
|
|
707
|
+
if (!this.evidence || this.notified || now - this.detectedAt < this.graceMs)
|
|
708
|
+
return null;
|
|
709
|
+
this.notified = true;
|
|
710
|
+
return {
|
|
711
|
+
activity,
|
|
712
|
+
evidence: this.evidence,
|
|
713
|
+
blockedForMs: now - this.detectedAt,
|
|
714
|
+
};
|
|
715
|
+
}
|
|
716
|
+
reset() {
|
|
717
|
+
this.activity = null;
|
|
718
|
+
this.evidence = null;
|
|
719
|
+
this.detectedAt = 0;
|
|
720
|
+
this.notified = false;
|
|
721
|
+
}
|
|
722
|
+
}
|
|
594
723
|
export class Daemon extends EventEmitter {
|
|
595
724
|
name;
|
|
596
725
|
config;
|
|
@@ -599,6 +728,8 @@ export class Daemon extends EventEmitter {
|
|
|
599
728
|
backend;
|
|
600
729
|
controlClient;
|
|
601
730
|
runtimeIdentity;
|
|
731
|
+
spawnGate;
|
|
732
|
+
stormWindow;
|
|
602
733
|
logger;
|
|
603
734
|
tmuxSessionName;
|
|
604
735
|
tmux = null;
|
|
@@ -608,6 +739,13 @@ export class Daemon extends EventEmitter {
|
|
|
608
739
|
guardian = null;
|
|
609
740
|
adapter = null;
|
|
610
741
|
pendingIpcRequests = new Map();
|
|
742
|
+
/**
|
|
743
|
+
* Daemon-global request sequence. MCP request ids are only process-local:
|
|
744
|
+
* Claude can have more than one MCP child connected to the same daemon and
|
|
745
|
+
* each starts at 1, so using requestId alone lets one caller steal another's
|
|
746
|
+
* response.
|
|
747
|
+
*/
|
|
748
|
+
fleetRequestSeq = 0;
|
|
611
749
|
// Track chatId/threadId from inbound messages for automatic outbound routing
|
|
612
750
|
lastChatId;
|
|
613
751
|
lastThreadId;
|
|
@@ -618,10 +756,8 @@ export class Daemon extends EventEmitter {
|
|
|
618
756
|
currentActivity = null;
|
|
619
757
|
// Session identity: map IPC socket → sessionName (from mcp_ready)
|
|
620
758
|
socketSessionNames = new Map();
|
|
621
|
-
// Crash recovery
|
|
622
|
-
|
|
623
|
-
static tmuxServerPaused = false;
|
|
624
|
-
static tmuxServerRecoveryTimer = null;
|
|
759
|
+
// Crash recovery is fleet-owned; all daemon objects share the coordinators
|
|
760
|
+
// injected by FleetManager below.
|
|
625
761
|
healthCheckTimer = null;
|
|
626
762
|
crashCount = 0;
|
|
627
763
|
lastCrashAt = 0;
|
|
@@ -655,6 +791,7 @@ export class Daemon extends EventEmitter {
|
|
|
655
791
|
/** Fallback safety sweep used only when no shared tmux control client exists. */
|
|
656
792
|
instanceStateMonitorTimer = null;
|
|
657
793
|
instanceStateIdleTimer = null;
|
|
794
|
+
instanceStateOutputProbeTimer = null;
|
|
658
795
|
instanceStateStuckTimer = null;
|
|
659
796
|
instanceStateOutputListener = null;
|
|
660
797
|
instanceStateOutputEventName = null;
|
|
@@ -665,6 +802,7 @@ export class Daemon extends EventEmitter {
|
|
|
665
802
|
instanceStateReadyPattern = null;
|
|
666
803
|
instanceStateBusyPattern = null;
|
|
667
804
|
instanceStateMonitorActive = false;
|
|
805
|
+
sessionCheckpointWarningEmitted = false;
|
|
668
806
|
statePollInFlight = false;
|
|
669
807
|
// ── Dead-MCP proxy reply: per-turn evidence ─────────────────────────────
|
|
670
808
|
// Set when an inbound message lands in the CLI; cleared on the idle edge that
|
|
@@ -672,8 +810,12 @@ export class Daemon extends EventEmitter {
|
|
|
672
810
|
// between, the agent's answer exists only on screen — the daemon relays it.
|
|
673
811
|
turnHadInbound = false;
|
|
674
812
|
turnOutboundDelivered = false;
|
|
813
|
+
/** Successful `reply` specifically; cross-instance tools do not satisfy #648. */
|
|
814
|
+
turnReplyDelivered = false;
|
|
675
815
|
turnCorrelationId;
|
|
676
816
|
turnInboundMarker;
|
|
817
|
+
/** Prevent a visible stale XML fragment from being recovered on later turns. */
|
|
818
|
+
lastMalformedToolCallSignature;
|
|
677
819
|
proxyReplySeq = 0;
|
|
678
820
|
autoPauseController;
|
|
679
821
|
pauseRequested = false;
|
|
@@ -684,6 +826,35 @@ export class Daemon extends EventEmitter {
|
|
|
684
826
|
* would silently no-op exactly when we most need to stop feeding the CLI.
|
|
685
827
|
*/
|
|
686
828
|
pausePending = false;
|
|
829
|
+
/**
|
|
830
|
+
* An auth failure was detected and nothing has resolved it yet.
|
|
831
|
+
*
|
|
832
|
+
* Separate from `pausePending` because it must outlive it: the deferred pause
|
|
833
|
+
* clears `pausePending` as soon as it fires, but if that pause is refused the
|
|
834
|
+
* instance is still auth-broken and must not resume raising alarms.
|
|
835
|
+
*/
|
|
836
|
+
authFailureUnresolved = false;
|
|
837
|
+
/** One login-screen auth report per spawn — the screen persists across polls. */
|
|
838
|
+
loginScreenReported = false;
|
|
839
|
+
/** A fatal startup screen (see StartupDialog.fatal) was already reported this run. */
|
|
840
|
+
fatalStartupReported = false;
|
|
841
|
+
/**
|
|
842
|
+
* The CLI is parked on a fatal startup screen and every pane write must be
|
|
843
|
+
* refused. This is deliberately separate from the pause that the reported
|
|
844
|
+
* pty_error triggers: the pause is applied by an async handler, and a queued
|
|
845
|
+
* delivery would race it into the modal (where paste+Enter can confirm
|
|
846
|
+
* "Exit and fix manually"). Set synchronously at dialog-match time; cleared
|
|
847
|
+
* when a startup scan reaches a real ready prompt, or on wake().
|
|
848
|
+
*/
|
|
849
|
+
fatalStartupBlocked = false;
|
|
850
|
+
/** model_error seen mid-turn: notify only if it survives to the idle screen. */
|
|
851
|
+
pendingModelErrorKey = null;
|
|
852
|
+
/**
|
|
853
|
+
* Let the next pause() proceed from a stuck pane. Set only for the auth-deferred
|
|
854
|
+
* pause — see the pausePending consumption site for why waiting for idle there
|
|
855
|
+
* is waiting forever.
|
|
856
|
+
*/
|
|
857
|
+
pauseAllowStuck = false;
|
|
687
858
|
pauseWakeState = "active";
|
|
688
859
|
pauseWakeTransition = null;
|
|
689
860
|
// Model failover: override model on next spawn when rate-limited
|
|
@@ -697,6 +868,12 @@ export class Daemon extends EventEmitter {
|
|
|
697
868
|
* and ⏳/👀/✅ reactions live here). It does NOT cover the other writers that
|
|
698
869
|
* reach the pane — {@link paneWriteLock} does. */
|
|
699
870
|
pasteLock = Promise.resolve();
|
|
871
|
+
/**
|
|
872
|
+
* Fleet cancellation epoch. Queue entries retain the epoch they arrived with;
|
|
873
|
+
* advancing it invalidates only work received before the cancel, never a new
|
|
874
|
+
* user message which arrives immediately afterwards.
|
|
875
|
+
*/
|
|
876
|
+
deliveryEpoch = 0;
|
|
700
877
|
/** Mutual exclusion for *every* write into the pane, whichever subsystem it
|
|
701
878
|
* comes from. See PaneWriteLock for why interleaving is destructive. */
|
|
702
879
|
paneWriteLock = new PaneWriteLock();
|
|
@@ -710,9 +887,21 @@ export class Daemon extends EventEmitter {
|
|
|
710
887
|
lastBuiltInstructions = "";
|
|
711
888
|
pasteQueueDepth = 0;
|
|
712
889
|
firstDeliveryDelay = new FirstDeliveryDelay();
|
|
890
|
+
/** Orders /steer pastes against each other. Deliberately NOT pasteLock:
|
|
891
|
+
* a steer must not queue behind the normal deliveries it exists to
|
|
892
|
+
* overtake. Pane-level exclusion still comes from paneWriteLock inside
|
|
893
|
+
* deliverMessage, so a steer and a normal delivery can never interleave
|
|
894
|
+
* their PTY writes — the steer just doesn't wait for idle. */
|
|
895
|
+
steerLock = Promise.resolve();
|
|
896
|
+
/** Tool lines shown in the channel processing bubble for the current turn. */
|
|
897
|
+
turnProgress = new ProgressAccumulator();
|
|
898
|
+
lastProgressBroadcast = "";
|
|
899
|
+
progressBroadcastTimer = null;
|
|
900
|
+
lastProgressBroadcastAt = 0;
|
|
713
901
|
// PTY error pattern monitoring
|
|
714
902
|
errorMonitorTimer = null;
|
|
715
903
|
interactivePromptDetector = new InteractivePromptDetector();
|
|
904
|
+
blockingProcessDetector = new BlockingProcessDetector();
|
|
716
905
|
/** Same 5-min gate the error monitor uses, so a dead MCP server alerts once. */
|
|
717
906
|
static MCP_DEATH_COOLDOWN_MS = 5 * 60_000;
|
|
718
907
|
lastMcpDeathNotifiedAt = 0;
|
|
@@ -724,6 +913,16 @@ export class Daemon extends EventEmitter {
|
|
|
724
913
|
* past this point interrupting the turn costs less than staying mute.
|
|
725
914
|
*/
|
|
726
915
|
static MCP_RESTART_MAX_WAIT_MS = 30 * 60_000;
|
|
916
|
+
/**
|
|
917
|
+
* How long the revival restart waits, at the point of firing, for a
|
|
918
|
+
* CLI-respawned MCP server to announce itself.
|
|
919
|
+
*
|
|
920
|
+
* Codex <0.146 tears down and recreates ALL MCP connections on an auth/config
|
|
921
|
+
* change — healthy ones included — so the daemon sees the old pid die while
|
|
922
|
+
* the CLI is fine and a replacement is seconds away. Module load measured
|
|
923
|
+
* ~0.36s, so 2s is roughly 5x headroom without making a genuine loss wait.
|
|
924
|
+
*/
|
|
925
|
+
static MCP_REPLACEMENT_GRACE_MS = 2_000;
|
|
727
926
|
/**
|
|
728
927
|
* Sticky "restart to revive the MCP server as soon as the pane idles" — same
|
|
729
928
|
* shape as pausePending, because the death is usually detected mid-turn when
|
|
@@ -732,6 +931,25 @@ export class Daemon extends EventEmitter {
|
|
|
732
931
|
*/
|
|
733
932
|
mcpRestartPending = false;
|
|
734
933
|
mcpRestartStaleTimer = null;
|
|
934
|
+
/** In-flight replacement grace; also the guard against three triggers stacking. */
|
|
935
|
+
mcpRestartGraceTimer = null;
|
|
936
|
+
/**
|
|
937
|
+
* Which trigger the in-flight grace will commit under.
|
|
938
|
+
*
|
|
939
|
+
* The single-timer guard would otherwise SWALLOW a stale_timeout that lands
|
|
940
|
+
* during an idle-gated grace — and because the stale timer is one-shot and
|
|
941
|
+
* already cleared itself, the 30-minute backstop would be gone for good on an
|
|
942
|
+
* instance that never idles, which is the exact case it exists for.
|
|
943
|
+
*/
|
|
944
|
+
mcpRestartGraceTrigger = null;
|
|
945
|
+
/**
|
|
946
|
+
* A dead-MCP sighting held back once, pending a possible replacement.
|
|
947
|
+
*
|
|
948
|
+
* Deliberately NOT the same field as `mcpDeathNotifiedForPid`: deferring must
|
|
949
|
+
* not consume the once-per-pid budget, or a suspicion that turns out to be a
|
|
950
|
+
* real loss could never be reported at all.
|
|
951
|
+
*/
|
|
952
|
+
mcpDeathDeferredForPid = null;
|
|
735
953
|
/** Prevent in-flight monitor callbacks from re-arming after a pause. */
|
|
736
954
|
runtimeMonitorsFrozen = false;
|
|
737
955
|
errorWaitingForRecovery = false; // true = error detected, waiting for ready pattern
|
|
@@ -740,7 +958,7 @@ export class Daemon extends EventEmitter {
|
|
|
740
958
|
activeErrorPatternKey = null;
|
|
741
959
|
/** Whether this instance is in an abnormal error state (auto-pause is normal). */
|
|
742
960
|
get isErrorState() {
|
|
743
|
-
return this.errorWaitingForRecovery || (this.healthCheckPaused && !this.isPaused) ||
|
|
961
|
+
return this.errorWaitingForRecovery || (this.healthCheckPaused && !this.isPaused) || this.stormWindow?.isActive() === true;
|
|
744
962
|
}
|
|
745
963
|
get isPaused() { return this.pauseWakeState !== "active"; }
|
|
746
964
|
/** Current CLI process state for status surfaces and recovery commands. */
|
|
@@ -780,7 +998,7 @@ export class Daemon extends EventEmitter {
|
|
|
780
998
|
this.errorRecoveryDeadlineAt = 0;
|
|
781
999
|
this.activeErrorPatternKey = null;
|
|
782
1000
|
}
|
|
783
|
-
constructor(name, config, instanceDir, topicMode = false, backend, controlClient, rootLogger, runtimeIdentity) {
|
|
1001
|
+
constructor(name, config, instanceDir, topicMode = false, backend, controlClient, rootLogger, runtimeIdentity, spawnGate, stormWindow) {
|
|
784
1002
|
super();
|
|
785
1003
|
this.name = name;
|
|
786
1004
|
this.config = config;
|
|
@@ -789,6 +1007,8 @@ export class Daemon extends EventEmitter {
|
|
|
789
1007
|
this.backend = backend;
|
|
790
1008
|
this.controlClient = controlClient;
|
|
791
1009
|
this.runtimeIdentity = runtimeIdentity;
|
|
1010
|
+
this.spawnGate = spawnGate;
|
|
1011
|
+
this.stormWindow = stormWindow;
|
|
792
1012
|
if (!rootLogger)
|
|
793
1013
|
throw new Error("Daemon requires a shared root logger");
|
|
794
1014
|
this.runtimeIdentity ??= {
|
|
@@ -877,20 +1097,7 @@ export class Daemon extends EventEmitter {
|
|
|
877
1097
|
this.handleToolCall(msg, socket);
|
|
878
1098
|
}
|
|
879
1099
|
else if (msg.type === "mcp_ready") {
|
|
880
|
-
|
|
881
|
-
if (sessionName) {
|
|
882
|
-
this.socketSessionNames.set(socket, sessionName);
|
|
883
|
-
socket.on("close", () => {
|
|
884
|
-
this.socketSessionNames.delete(socket);
|
|
885
|
-
// Notify fleet manager so it can clean up sessionRegistry
|
|
886
|
-
if (sessionName !== this.name) {
|
|
887
|
-
this.ipcServer?.broadcast({ type: "session_disconnected", sessionName });
|
|
888
|
-
}
|
|
889
|
-
});
|
|
890
|
-
}
|
|
891
|
-
this.logger.debug({ sessionName }, "MCP channel server connected and ready");
|
|
892
|
-
// Notify FleetManager's IPC client that MCP is ready
|
|
893
|
-
this.ipcServer?.broadcast({ type: "mcp_ready", sessionName });
|
|
1100
|
+
this.handleMcpReady(msg, socket);
|
|
894
1101
|
}
|
|
895
1102
|
else if (msg.type === "query_sessions") {
|
|
896
1103
|
// Fleet manager asks for all registered session names (catches sessions
|
|
@@ -910,43 +1117,38 @@ export class Daemon extends EventEmitter {
|
|
|
910
1117
|
// Fleet manager routed a message to us (topic mode)
|
|
911
1118
|
const meta = msg.meta;
|
|
912
1119
|
const targetSession = msg.targetSession;
|
|
1120
|
+
const deliveryEpoch = this.captureDeliveryEpoch(msg.delivery_epoch);
|
|
913
1121
|
void this.wake().then(() => {
|
|
914
|
-
this.pushChannelMessage(msg.content, meta, targetSession);
|
|
1122
|
+
this.pushChannelMessage(msg.content, meta, targetSession, deliveryEpoch);
|
|
915
1123
|
}).catch(err => {
|
|
916
1124
|
this.logger.error({ err: err.message }, "Wake failed for inbound delivery");
|
|
917
1125
|
});
|
|
918
1126
|
}
|
|
919
1127
|
else if (msg.type === "raw_paste") {
|
|
920
1128
|
// Paste raw text directly to CLI without [user:] wrapping.
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
1129
|
+
this.queueRawPaste(msg.content, this.captureDeliveryEpoch(msg.delivery_epoch), msg.confirm_clear === true);
|
|
1130
|
+
}
|
|
1131
|
+
else if (msg.type === "config_update") {
|
|
1132
|
+
this.applyConfigUpdate(msg.config);
|
|
1133
|
+
}
|
|
1134
|
+
else if (msg.type === "steer") {
|
|
1135
|
+
const meta = (msg.meta ?? {});
|
|
1136
|
+
this.steerMessage(msg.content, meta, this.captureDeliveryEpoch(msg.delivery_epoch));
|
|
1137
|
+
}
|
|
1138
|
+
else if (msg.type === "btw") {
|
|
1139
|
+
const meta = (msg.meta ?? {});
|
|
1140
|
+
this.btwMessage(msg.content, meta, this.captureDeliveryEpoch(msg.delivery_epoch));
|
|
931
1141
|
}
|
|
932
1142
|
else if (msg.type === "fleet_schedule_trigger") {
|
|
933
1143
|
const payload = msg.payload;
|
|
934
1144
|
const meta = msg.meta;
|
|
935
|
-
|
|
1145
|
+
const deliveryEpoch = this.captureDeliveryEpoch(msg.delivery_epoch);
|
|
1146
|
+
void this.wake().then(() => this.pushChannelMessage(payload.message, meta, undefined, deliveryEpoch)).catch(err => {
|
|
936
1147
|
this.logger.error({ err: err.message }, "Wake failed for scheduled delivery");
|
|
937
1148
|
});
|
|
938
1149
|
}
|
|
939
1150
|
else if (msg.type === "query_instance_state") {
|
|
940
|
-
|
|
941
|
-
this.ipcServer?.send(socket, {
|
|
942
|
-
type: "instance_state_response",
|
|
943
|
-
requestId: msg.requestId,
|
|
944
|
-
instanceName: this.name,
|
|
945
|
-
...snapshot,
|
|
946
|
-
state: this.isPaused ? "paused" : snapshot.state,
|
|
947
|
-
processStatus: this.processStatus,
|
|
948
|
-
pausedAt: this.lastPausedAt,
|
|
949
|
-
});
|
|
1151
|
+
void this.respondToInstanceStateQuery(msg, socket);
|
|
950
1152
|
}
|
|
951
1153
|
});
|
|
952
1154
|
// 2. Tmux — ensure session, create window if not alive
|
|
@@ -1032,8 +1234,11 @@ export class Daemon extends EventEmitter {
|
|
|
1032
1234
|
const outputLog = join(this.instanceDir, "output.log");
|
|
1033
1235
|
rotateLogIfNeeded(outputLog);
|
|
1034
1236
|
await this.tmux.pipeOutput(outputLog).catch(() => { });
|
|
1035
|
-
// 4. Transcript monitor
|
|
1036
|
-
|
|
1237
|
+
// 4. Transcript monitor. claude-code is handled inside the monitor
|
|
1238
|
+
// (statusline transcript); codex/kiro/opencode read their CLI's own
|
|
1239
|
+
// conversation store via a pluggable source. Backends with no known
|
|
1240
|
+
// source stay inert exactly as before.
|
|
1241
|
+
this.transcriptMonitor = new TranscriptMonitor(this.instanceDir, this.logger, createTranscriptSource(this.config.backend ?? "claude-code", this.config.working_directory));
|
|
1037
1242
|
// 5. Wire transcript events
|
|
1038
1243
|
const ackIfPending = () => {
|
|
1039
1244
|
if (!this.pendingAckMessage || !this.adapter)
|
|
@@ -1049,6 +1254,7 @@ export class Daemon extends EventEmitter {
|
|
|
1049
1254
|
this.recordRecentEvent({ type: "tool_use", name, preview: this.summarizeTool(name, input) });
|
|
1050
1255
|
this.recordRecentToolActivity(this.summarizeTool(name, input));
|
|
1051
1256
|
this.publishActivity(this.summarizeTool(name, input));
|
|
1257
|
+
this.recordToolProgress(name, input);
|
|
1052
1258
|
});
|
|
1053
1259
|
this.transcriptMonitor.on("tool_result", (name, _output) => {
|
|
1054
1260
|
this.recordRecentEvent({ type: "tool_result", name });
|
|
@@ -1111,6 +1317,42 @@ export class Daemon extends EventEmitter {
|
|
|
1111
1317
|
* process would have no client reading it. Only the CLI can restore its own
|
|
1112
1318
|
* tools — hence notify, and let the operator decide about restarting.
|
|
1113
1319
|
*/
|
|
1320
|
+
/**
|
|
1321
|
+
* An MCP server has connected and announced itself.
|
|
1322
|
+
*
|
|
1323
|
+
* Beyond the existing session bookkeeping this is the earliest evidence that
|
|
1324
|
+
* tools are back — roughly 30s ahead of the next health tick, which is
|
|
1325
|
+
* exactly the window #663's false restarts fired in.
|
|
1326
|
+
*/
|
|
1327
|
+
handleMcpReady(msg, socket) {
|
|
1328
|
+
const sessionName = msg.sessionName;
|
|
1329
|
+
if (sessionName) {
|
|
1330
|
+
this.socketSessionNames.set(socket, sessionName);
|
|
1331
|
+
socket.on("close", () => {
|
|
1332
|
+
this.socketSessionNames.delete(socket);
|
|
1333
|
+
// Notify fleet manager so it can clean up sessionRegistry
|
|
1334
|
+
if (sessionName !== this.name) {
|
|
1335
|
+
this.ipcServer?.broadcast({ type: "session_disconnected", sessionName });
|
|
1336
|
+
}
|
|
1337
|
+
});
|
|
1338
|
+
}
|
|
1339
|
+
this.logger.debug({ sessionName, pid: msg.pid }, "MCP channel server connected and ready");
|
|
1340
|
+
// Gated on ownership: this ipcServer also accepts EXTERNAL sessions (a
|
|
1341
|
+
// `claude` run in another directory registers as `external-<cwd>-<pid>`).
|
|
1342
|
+
// Without the gate an unrelated session could stand down a restart armed
|
|
1343
|
+
// for THIS instance's genuinely dead server. The liveness re-read below is
|
|
1344
|
+
// a second guard, but ownership is the one that is semantically right.
|
|
1345
|
+
if (sessionName === this.name && this.mcpRestartPending) {
|
|
1346
|
+
const live = mcpServerState(this.instanceDir);
|
|
1347
|
+
if (live.state === "alive") {
|
|
1348
|
+
this.logger.info({ pid: live.pid }, "MCP server replaced — standing down the pending revival restart");
|
|
1349
|
+
this.mcpDeathNotifiedForPid = null;
|
|
1350
|
+
this.mcpDeathDeferredForPid = null;
|
|
1351
|
+
this.clearMcpRestartRequest();
|
|
1352
|
+
}
|
|
1353
|
+
}
|
|
1354
|
+
this.ipcServer?.broadcast({ type: "mcp_ready", sessionName });
|
|
1355
|
+
}
|
|
1114
1356
|
checkMcpServerAlive() {
|
|
1115
1357
|
if (this.isPaused)
|
|
1116
1358
|
return;
|
|
@@ -1119,22 +1361,53 @@ export class Daemon extends EventEmitter {
|
|
|
1119
1361
|
return;
|
|
1120
1362
|
if (status.state === "alive") {
|
|
1121
1363
|
this.mcpDeathNotifiedForPid = null; // the CLI respawned it — re-arm
|
|
1364
|
+
this.mcpDeathDeferredForPid = null;
|
|
1122
1365
|
this.clearMcpRestartRequest(); // tools are back — stand down a pending auto-restart
|
|
1123
1366
|
return;
|
|
1124
1367
|
}
|
|
1125
1368
|
// Dead: report once per pid, and at most once per cooldown window.
|
|
1126
1369
|
if (this.mcpDeathNotifiedForPid === status.pid)
|
|
1127
1370
|
return;
|
|
1371
|
+
// An auth-broken CLI cannot keep an MCP server alive, and restarting it
|
|
1372
|
+
// only loops back to the sign-in screen — hold the auto-restart until a
|
|
1373
|
+
// re-login (which restarts the instance and replaces this daemon anyway).
|
|
1374
|
+
const authSuspected = this.authFailureUnresolved;
|
|
1375
|
+
const autoRestart = this.config.mcp_auto_restart !== false && !authSuspected;
|
|
1376
|
+
// Codex <0.146 kills and recreates its MCP connections wholesale, so the
|
|
1377
|
+
// FIRST sighting of a dead pid under a live Codex is more likely a swap
|
|
1378
|
+
// than a loss. Hold the alarm for one tick — only the alarm: the restart is
|
|
1379
|
+
// still armed, and the chokepoint's re-read cancels it if the replacement
|
|
1380
|
+
// arrives. Not generalised to other backends, where a dead MCP under a live
|
|
1381
|
+
// CLI is a genuine incident that silence would mask.
|
|
1382
|
+
//
|
|
1383
|
+
// Crucially this does NOT touch mcpDeathNotifiedForPid or the cooldown
|
|
1384
|
+
// clock: if the suspicion is wrong the incident must still be reportable.
|
|
1385
|
+
if (this.suspectsTransientMcpReplacement() && this.mcpDeathDeferredForPid !== status.pid) {
|
|
1386
|
+
this.mcpDeathDeferredForPid = status.pid;
|
|
1387
|
+
this.logger.warn({ pid: status.pid }, "MCP server pid is gone under a live Codex — holding the alarm one tick for a possible replacement");
|
|
1388
|
+
if (autoRestart)
|
|
1389
|
+
this.armMcpRestartWhenIdle();
|
|
1390
|
+
return;
|
|
1391
|
+
}
|
|
1128
1392
|
if (Date.now() - this.lastMcpDeathNotifiedAt < Daemon.MCP_DEATH_COOLDOWN_MS)
|
|
1129
1393
|
return;
|
|
1130
1394
|
this.mcpDeathNotifiedForPid = status.pid;
|
|
1395
|
+
this.mcpDeathDeferredForPid = null;
|
|
1131
1396
|
this.lastMcpDeathNotifiedAt = Date.now();
|
|
1132
|
-
|
|
1133
|
-
this.
|
|
1134
|
-
this.emit("mcp_died", { name: this.name, pid: status.pid, autoRestart });
|
|
1397
|
+
this.logger.error({ pid: status.pid, autoRestart, authSuspected }, "MCP server process is gone — instance has no agend tools");
|
|
1398
|
+
this.emit("mcp_died", { name: this.name, pid: status.pid, autoRestart, authSuspected });
|
|
1135
1399
|
if (autoRestart)
|
|
1136
1400
|
this.armMcpRestartWhenIdle();
|
|
1137
1401
|
}
|
|
1402
|
+
/**
|
|
1403
|
+
* Whether a dead MCP pid on this backend is more likely a CLI-driven
|
|
1404
|
+
* replacement than a real loss. Codex only, and deliberately narrow: it is
|
|
1405
|
+
* the backend whose released fix (0.146) confirms it tore down healthy MCP
|
|
1406
|
+
* connections alongside the one it meant to restart.
|
|
1407
|
+
*/
|
|
1408
|
+
suspectsTransientMcpReplacement() {
|
|
1409
|
+
return (this.config.backend ?? this.backend?.binaryName) === "codex";
|
|
1410
|
+
}
|
|
1138
1411
|
/**
|
|
1139
1412
|
* Arm an automatic instance restart to revive a dead MCP server. Fires on the
|
|
1140
1413
|
* next busy→idle edge — or immediately when the pane is already idle, which is
|
|
@@ -1157,21 +1430,110 @@ export class Daemon extends EventEmitter {
|
|
|
1157
1430
|
this.fireMcpRestartRequest("already_idle");
|
|
1158
1431
|
}
|
|
1159
1432
|
}
|
|
1433
|
+
/**
|
|
1434
|
+
* Single chokepoint for all three revival triggers (already_idle, idle_edge,
|
|
1435
|
+
* stale_timeout) — and therefore the one place worth re-checking reality.
|
|
1436
|
+
*
|
|
1437
|
+
* The pending flag is armed by a health tick that saw the OLD pid die, and
|
|
1438
|
+
* was previously fired without ever looking again. When a CLI legitimately
|
|
1439
|
+
* respawns its own MCP child, the replacement is already serving by the time
|
|
1440
|
+
* the turn ends — but the idle edge fires inside the ~30s before the next
|
|
1441
|
+
* health tick can stand the request down, restarting a healthy instance. One
|
|
1442
|
+
* external user was restarted 18 times in two days this way (#663).
|
|
1443
|
+
*
|
|
1444
|
+
* Re-reading here is safe for every backend: worst case it clears a pending
|
|
1445
|
+
* restart that should not have existed.
|
|
1446
|
+
*/
|
|
1160
1447
|
fireMcpRestartRequest(trigger) {
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
//
|
|
1164
|
-
//
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
if (this.healthCheckPaused || this.spawning || this.isPaused || this.runtimeMonitorsFrozen) {
|
|
1168
|
-
this.logger.warn({ trigger }, "MCP revival restart cancelled — instance is already pausing/spawning/recovering");
|
|
1448
|
+
if (this.mcpRestartCancelledByRecovery(trigger))
|
|
1449
|
+
return;
|
|
1450
|
+
// Authoritative re-read — the CLI may have replaced the server since the
|
|
1451
|
+
// health tick that armed this.
|
|
1452
|
+
if (mcpServerState(this.instanceDir).state === "alive") {
|
|
1453
|
+
this.logger.info({ trigger }, "MCP revival restart stood down — a live MCP server is already serving");
|
|
1169
1454
|
this.clearMcpRestartRequest();
|
|
1170
1455
|
return;
|
|
1171
1456
|
}
|
|
1457
|
+
// Not back yet: give a CLI-driven replacement a bounded moment rather than
|
|
1458
|
+
// restarting the instance out from under it. One timer covers all three
|
|
1459
|
+
// triggers; a second trigger during the wait is a no-op.
|
|
1460
|
+
if (this.mcpRestartGraceTimer) {
|
|
1461
|
+
// stale_timeout outranks the idle-gated triggers: it is the backstop for
|
|
1462
|
+
// a turn that never ends, so it must not be lost to an in-flight grace.
|
|
1463
|
+
if (trigger === "stale_timeout")
|
|
1464
|
+
this.mcpRestartGraceTrigger = "stale_timeout";
|
|
1465
|
+
return;
|
|
1466
|
+
}
|
|
1467
|
+
this.mcpRestartGraceTrigger = trigger;
|
|
1468
|
+
this.mcpRestartGraceTimer = setTimeout(() => {
|
|
1469
|
+
this.mcpRestartGraceTimer = null;
|
|
1470
|
+
const committed = this.mcpRestartGraceTrigger ?? trigger;
|
|
1471
|
+
this.mcpRestartGraceTrigger = null;
|
|
1472
|
+
if (!this.mcpRestartPending || this.runtimeMonitorsFrozen)
|
|
1473
|
+
return;
|
|
1474
|
+
if (this.mcpRestartCancelledByRecovery(committed))
|
|
1475
|
+
return;
|
|
1476
|
+
if (mcpServerState(this.instanceDir).state === "alive") {
|
|
1477
|
+
this.logger.info({ trigger: committed }, "MCP revival restart stood down — replacement came up during the grace window");
|
|
1478
|
+
this.clearMcpRestartRequest();
|
|
1479
|
+
return;
|
|
1480
|
+
}
|
|
1481
|
+
// Re-check the idle gate. The grace turned what used to be a synchronous
|
|
1482
|
+
// decision into a 2s window, and "idle edge, then a queued message lands"
|
|
1483
|
+
// is the NORMAL ordering for queued delivery — so without this the
|
|
1484
|
+
// restart tears down a turn that started during the wait. Worse than the
|
|
1485
|
+
// race it replaced: the #485 proxy reply only runs at turn end, so an
|
|
1486
|
+
// in-flight answer would be swallowed with no fallback. Only the
|
|
1487
|
+
// 30-minute stale timer may interrupt a live turn, by design.
|
|
1488
|
+
//
|
|
1489
|
+
// Deliberately does NOT clear pending: the MCP really is gone, so the
|
|
1490
|
+
// request must survive to the next idle edge (which re-enters here), and
|
|
1491
|
+
// the stale timer keeps running underneath as the backstop.
|
|
1492
|
+
if (committed !== "stale_timeout" && !(this.instanceState === "idle" && this.pasteQueueDepth === 0)) {
|
|
1493
|
+
this.logger.info({ trigger: committed, instanceState: this.instanceState, pasteQueueDepth: this.pasteQueueDepth }, "MCP revival restart deferred — the pane went busy during the grace window");
|
|
1494
|
+
return;
|
|
1495
|
+
}
|
|
1496
|
+
// The suspicion was wrong: a real loss. Report it BEFORE asking for the
|
|
1497
|
+
// restart — instance-lifecycle's suppression path documents that it
|
|
1498
|
+
// assumes the mcp_died notification has already gone out, and this daemon
|
|
1499
|
+
// does not survive the restart to send it afterwards.
|
|
1500
|
+
this.reportDeferredMcpDeath();
|
|
1501
|
+
this.clearMcpRestartRequest();
|
|
1502
|
+
this.logger.warn({ trigger: committed }, "Requesting instance restart to revive its MCP server");
|
|
1503
|
+
this.emit("mcp_restart_requested", { name: this.name, trigger: committed });
|
|
1504
|
+
}, Daemon.MCP_REPLACEMENT_GRACE_MS);
|
|
1505
|
+
this.mcpRestartGraceTimer.unref?.();
|
|
1506
|
+
}
|
|
1507
|
+
/**
|
|
1508
|
+
* A recovery is already reshaping the CLI: crash-loop handling
|
|
1509
|
+
* (healthCheckPaused), a respawn in flight (spawning), or a pause
|
|
1510
|
+
* (isPaused / frozen monitors). Each ends in a fresh CLI — and with it a
|
|
1511
|
+
* fresh MCP server — or in supervision ending, where a restart on top would
|
|
1512
|
+
* fight the recovery. Either way the revival restart is moot.
|
|
1513
|
+
*/
|
|
1514
|
+
mcpRestartCancelledByRecovery(trigger) {
|
|
1515
|
+
if (!(this.healthCheckPaused || this.spawning || this.isPaused || this.runtimeMonitorsFrozen || this.stormWindow?.isActive()))
|
|
1516
|
+
return false;
|
|
1517
|
+
this.logger.warn({ trigger }, "MCP revival restart cancelled — instance is already pausing/spawning/recovering");
|
|
1172
1518
|
this.clearMcpRestartRequest();
|
|
1173
|
-
|
|
1174
|
-
|
|
1519
|
+
return true;
|
|
1520
|
+
}
|
|
1521
|
+
/**
|
|
1522
|
+
* Emit a dead-MCP report that was held back as a suspected replacement, once
|
|
1523
|
+
* it is clear no replacement is coming. Consumes the once-per-pid budget
|
|
1524
|
+
* here — deferring was to avoid spending it on a false alarm, not to drop
|
|
1525
|
+
* the incident.
|
|
1526
|
+
*/
|
|
1527
|
+
reportDeferredMcpDeath() {
|
|
1528
|
+
const pid = this.mcpDeathDeferredForPid;
|
|
1529
|
+
if (pid == null || this.mcpDeathNotifiedForPid === pid)
|
|
1530
|
+
return;
|
|
1531
|
+
const authSuspected = this.authFailureUnresolved;
|
|
1532
|
+
const autoRestart = this.config.mcp_auto_restart !== false && !authSuspected;
|
|
1533
|
+
this.mcpDeathNotifiedForPid = pid;
|
|
1534
|
+
this.lastMcpDeathNotifiedAt = Date.now();
|
|
1535
|
+
this.logger.error({ pid, autoRestart, authSuspected }, "MCP server process is gone — no replacement arrived, instance has no agend tools");
|
|
1536
|
+
this.emit("mcp_died", { name: this.name, pid, autoRestart, authSuspected });
|
|
1175
1537
|
}
|
|
1176
1538
|
clearMcpRestartRequest() {
|
|
1177
1539
|
this.mcpRestartPending = false;
|
|
@@ -1179,6 +1541,11 @@ export class Daemon extends EventEmitter {
|
|
|
1179
1541
|
clearTimeout(this.mcpRestartStaleTimer);
|
|
1180
1542
|
this.mcpRestartStaleTimer = null;
|
|
1181
1543
|
}
|
|
1544
|
+
if (this.mcpRestartGraceTimer) {
|
|
1545
|
+
clearTimeout(this.mcpRestartGraceTimer);
|
|
1546
|
+
this.mcpRestartGraceTimer = null;
|
|
1547
|
+
}
|
|
1548
|
+
this.mcpRestartGraceTrigger = null;
|
|
1182
1549
|
}
|
|
1183
1550
|
startHealthCheck() {
|
|
1184
1551
|
if (this.runtimeMonitorsFrozen || this.healthCheckTimer)
|
|
@@ -1212,7 +1579,7 @@ export class Daemon extends EventEmitter {
|
|
|
1212
1579
|
this.emitSupervisionEnded("instance directory was removed from disk", "Recreate the instance, or delete it from fleet.yaml.");
|
|
1213
1580
|
return;
|
|
1214
1581
|
}
|
|
1215
|
-
if (!this.tmux || this.spawning || this.healthCheckPaused
|
|
1582
|
+
if (!this.tmux || this.spawning || this.healthCheckPaused) {
|
|
1216
1583
|
scheduleNext();
|
|
1217
1584
|
return;
|
|
1218
1585
|
}
|
|
@@ -1270,28 +1637,29 @@ export class Daemon extends EventEmitter {
|
|
|
1270
1637
|
let nullReason;
|
|
1271
1638
|
if (!paneStatus) {
|
|
1272
1639
|
const serverAlive = await TmuxManager.sessionExists(this.tmuxSessionName);
|
|
1273
|
-
|
|
1640
|
+
// A server may have already restarted before this daemon's health
|
|
1641
|
+
// tick. The new PID is the durable generation boundary: treat it as
|
|
1642
|
+
// a fleet storm even though `has-session` is true again.
|
|
1643
|
+
const generationChanged = serverAlive
|
|
1644
|
+
? this.stormWindow?.observeServerAlive(await TmuxManager.getServerPid(this.tmuxSessionName)) === true
|
|
1645
|
+
: false;
|
|
1646
|
+
if (generationChanged)
|
|
1647
|
+
this.emit("tmux_server_crash", this.name);
|
|
1648
|
+
if (!serverAlive || generationChanged || this.stormWindow?.needsRecovery(this.name)) {
|
|
1274
1649
|
crashType = "server";
|
|
1275
|
-
nullReason = "server_gone";
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
this.
|
|
1284
|
-
this.emit("tmux_server_crash", this.name);
|
|
1285
|
-
if (!Daemon.tmuxServerRecoveryTimer) {
|
|
1286
|
-
Daemon.tmuxServerRecoveryTimer = setTimeout(() => {
|
|
1287
|
-
Daemon.tmuxServerRecoveryTimer = null;
|
|
1288
|
-
Daemon.tmuxServerPaused = false;
|
|
1289
|
-
}, 30_000);
|
|
1290
|
-
}
|
|
1291
|
-
scheduleNext();
|
|
1292
|
-
return;
|
|
1650
|
+
nullReason = serverAlive ? "server_storm_window_loss" : "server_gone";
|
|
1651
|
+
if (!serverAlive) {
|
|
1652
|
+
this.logger.error(`tmux server died — all ${cliLabel} windows lost`);
|
|
1653
|
+
const distinct = this.stormWindow?.recordServerDead(this.name, [this.name]) ?? true;
|
|
1654
|
+
if (distinct)
|
|
1655
|
+
this.emit("tmux_server_crash", this.name);
|
|
1656
|
+
}
|
|
1657
|
+
else {
|
|
1658
|
+
this.stormWindow?.addAffected(this.name);
|
|
1293
1659
|
}
|
|
1294
|
-
|
|
1660
|
+
// The fleet breaker owns the delay and extends it on every new
|
|
1661
|
+
// server generation. Do not schedule a competing fixed timer.
|
|
1662
|
+
await this.stormWindow?.waitForSpawnAllowed();
|
|
1295
1663
|
}
|
|
1296
1664
|
else {
|
|
1297
1665
|
// null but server alive: window-level disappearance. Probe whether
|
|
@@ -1357,6 +1725,17 @@ export class Daemon extends EventEmitter {
|
|
|
1357
1725
|
catch (err) {
|
|
1358
1726
|
this.logger.error({ err: err.message }, "Recovery from background session conflict failed");
|
|
1359
1727
|
}
|
|
1728
|
+
// Keep the loop alive: every health-check exit must either
|
|
1729
|
+
// scheduleNext() or deliberately set healthCheckPaused. This one
|
|
1730
|
+
// did neither, so a recovered instance ran UNMONITORED (while
|
|
1731
|
+
// isHealthCheckEffectivelyPaused still said monitoring was on)
|
|
1732
|
+
// until the next pause→wake cycle or fleet restart. The next tick
|
|
1733
|
+
// does the right thing in both outcomes: a healthy new window
|
|
1734
|
+
// passes the alive check, and a failed recovery hits the dead
|
|
1735
|
+
// window again with backgroundSessionRecoveryAttempted set — so
|
|
1736
|
+
// it falls through to the normal crash handling (crash counting,
|
|
1737
|
+
// backoff, supervision_ended when retries are exhausted).
|
|
1738
|
+
scheduleNext();
|
|
1360
1739
|
return; // Don't count as crash
|
|
1361
1740
|
}
|
|
1362
1741
|
// Already attempted recovery — fall through to normal crash handling
|
|
@@ -1376,6 +1755,14 @@ export class Daemon extends EventEmitter {
|
|
|
1376
1755
|
}
|
|
1377
1756
|
// Append to crash history
|
|
1378
1757
|
this.appendCrashHistory({ exitCode, lastOutput, crashType, reason: nullReason });
|
|
1758
|
+
// A fleet-wide server loss is one infrastructure incident, not N
|
|
1759
|
+
// independent CLI crash loops. StormWindow owns its escalation; do
|
|
1760
|
+
// not let the same three server generations permanently trip every
|
|
1761
|
+
// daemon's per-instance 3-in-5m breaker.
|
|
1762
|
+
if (crashType === "server") {
|
|
1763
|
+
this.crashTimestamps = [];
|
|
1764
|
+
this.crashCount = 0;
|
|
1765
|
+
}
|
|
1379
1766
|
if (max_retries <= 0) {
|
|
1380
1767
|
this.healthCheckPaused = true;
|
|
1381
1768
|
this.logger.warn(`${cliLabel} window died — automatic restart is disabled`);
|
|
@@ -1462,6 +1849,10 @@ export class Daemon extends EventEmitter {
|
|
|
1462
1849
|
this.setProcessStatus("running");
|
|
1463
1850
|
this.logger.info({ resumed }, `Respawned ${cliLabel} window after crash`);
|
|
1464
1851
|
this.emit("crash_respawn", this.name);
|
|
1852
|
+
// Emit the per-instance audit event while the storm is still active
|
|
1853
|
+
// so its chat notification is suppressed into the fleet summary.
|
|
1854
|
+
if (crashType === "server")
|
|
1855
|
+
this.stormWindow?.markRecovered(this.name);
|
|
1465
1856
|
}
|
|
1466
1857
|
catch (err) {
|
|
1467
1858
|
this.logger.error({ err }, `Failed to respawn ${cliLabel} window`);
|
|
@@ -1573,6 +1964,25 @@ export class Daemon extends EventEmitter {
|
|
|
1573
1964
|
// A prompt is not an error and must not enter the PTY recovery gate.
|
|
1574
1965
|
// Continue scanning real errors in this same snapshot.
|
|
1575
1966
|
}
|
|
1967
|
+
// If a backend can derive activity from this exact pane, trust its
|
|
1968
|
+
// explicit null (tool completed) instead of falling back to a possibly
|
|
1969
|
+
// stale transcript activity from the previous scan.
|
|
1970
|
+
const paneActivity = this.backend?.getPaneActivity
|
|
1971
|
+
? this.backend.getPaneActivity(pane)
|
|
1972
|
+
: this.currentActivity;
|
|
1973
|
+
const hasPendingWork = this.pendingWork.hasPendingWork();
|
|
1974
|
+
if (!hasPendingWork)
|
|
1975
|
+
this.blockingProcessDetector.reset();
|
|
1976
|
+
const blockingProcess = hasPendingWork
|
|
1977
|
+
? this.blockingProcessDetector.observe(pane, paneActivity, Date.now())
|
|
1978
|
+
: null;
|
|
1979
|
+
if (blockingProcess) {
|
|
1980
|
+
// Reuse the hang-notification bridge: it offers explicit Restart/Wait
|
|
1981
|
+
// choices and, unlike pty_error, does not retire a still-useful Cancel
|
|
1982
|
+
// button merely because a foreground process owns stdin.
|
|
1983
|
+
this.logger.warn(blockingProcess, "Foreground process is blocking the agent input loop");
|
|
1984
|
+
this.hangDetector?.emit("hang", { unchangedForMs: blockingProcess.blockedForMs });
|
|
1985
|
+
}
|
|
1576
1986
|
// Auto-dismiss runtime dialogs (e.g. Codex rate limit model switch)
|
|
1577
1987
|
for (const dialog of dialogs) {
|
|
1578
1988
|
if (!dialog.pattern.test(pane))
|
|
@@ -1645,6 +2055,7 @@ export class Daemon extends EventEmitter {
|
|
|
1645
2055
|
if (seen > 0)
|
|
1646
2056
|
this.lastErrorCount.set(Daemon.errorPatternKey(ep), seen);
|
|
1647
2057
|
}
|
|
2058
|
+
this.authFailureUnresolved = false;
|
|
1648
2059
|
this.clearErrorRecoveryGate();
|
|
1649
2060
|
this.logger.info({ downtime_s: downtime }, "PTY error recovered — agent is ready again");
|
|
1650
2061
|
this.emit("pty_recovered", { name: this.name, downtime_s: downtime });
|
|
@@ -1665,6 +2076,19 @@ export class Daemon extends EventEmitter {
|
|
|
1665
2076
|
this.logger.warn({ errorType: this.lastDetectedErrorType }, "PTY error recovery deadline expired — resuming error scan");
|
|
1666
2077
|
this.clearErrorRecoveryGate();
|
|
1667
2078
|
}
|
|
2079
|
+
// A model_error deferred mid-turn: the turn is over (or the pane left
|
|
2080
|
+
// "working"), so decide now — still on screen means the CLI gave up and
|
|
2081
|
+
// the user must act; gone means a retry got past it and nobody needs to know.
|
|
2082
|
+
if (this.pendingModelErrorKey && this.instanceState !== "working") {
|
|
2083
|
+
const key = this.pendingModelErrorKey;
|
|
2084
|
+
this.pendingModelErrorKey = null;
|
|
2085
|
+
const ep = patterns.find(candidate => Daemon.errorPatternKey(candidate) === key);
|
|
2086
|
+
if (ep && ep.pattern.test(pane)) {
|
|
2087
|
+
this.emitErrorPattern(ep, key, pane, now);
|
|
2088
|
+
return;
|
|
2089
|
+
}
|
|
2090
|
+
this.logger.debug({ errorKey: key }, "Deferred model_error cleared before idle — dropped");
|
|
2091
|
+
}
|
|
1668
2092
|
// State: monitoring — count-based new-error detection over the full pane.
|
|
1669
2093
|
for (const ep of patterns) {
|
|
1670
2094
|
const key = Daemon.errorPatternKey(ep);
|
|
@@ -1692,25 +2116,49 @@ export class Daemon extends EventEmitter {
|
|
|
1692
2116
|
continue;
|
|
1693
2117
|
}
|
|
1694
2118
|
this.lastErrorCount.set(key, count);
|
|
1695
|
-
//
|
|
1696
|
-
//
|
|
1697
|
-
//
|
|
1698
|
-
if (
|
|
1699
|
-
this.
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
this.
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
this.
|
|
2119
|
+
// Kiro's `auto` model picks a concrete model per turn and reroutes around
|
|
2120
|
+
// a temporarily unavailable one on its own — telling the user to run
|
|
2121
|
+
// /model would ask them to fix something the CLI is already handling.
|
|
2122
|
+
if (ep.type === "model_error" && ep.action === "notify" && this.config.model === "auto") {
|
|
2123
|
+
this.logger.debug({ errorType: ep.type }, "model_error suppressed — model 'auto' reroutes on its own");
|
|
2124
|
+
continue;
|
|
2125
|
+
}
|
|
2126
|
+
// A mid-turn model error is often a transient the CLI retries past. Hold
|
|
2127
|
+
// the notification until the turn ends and notify only if the error is
|
|
2128
|
+
// still on the idle screen (see the deferred check above).
|
|
2129
|
+
if (ep.type === "model_error" && ep.action === "notify" && this.instanceState === "working") {
|
|
2130
|
+
this.pendingModelErrorKey = key;
|
|
2131
|
+
this.logger.debug({ errorType: ep.type }, "model_error deferred — instance is mid-turn, re-checking at idle");
|
|
2132
|
+
continue;
|
|
2133
|
+
}
|
|
2134
|
+
this.emitErrorPattern(ep, key, pane, now);
|
|
1711
2135
|
break; // Only handle first unsuppressed new error per scan
|
|
1712
2136
|
}
|
|
1713
2137
|
}
|
|
2138
|
+
/** Arm the recovery gates and emit one detected error pattern. */
|
|
2139
|
+
emitErrorPattern(ep, key, pane, now) {
|
|
2140
|
+
// skipRecoveryWait: this error self-recovers (e.g. a timeout — Kiro is
|
|
2141
|
+
// back at its prompt immediately). Keep monitoring so the next occurrence
|
|
2142
|
+
// can fire without waiting for a possibly startup-only ready pattern.
|
|
2143
|
+
if (!ep.skipRecoveryWait) {
|
|
2144
|
+
this.errorWaitingForRecovery = true;
|
|
2145
|
+
this.errorDetectedAt = now;
|
|
2146
|
+
this.errorRecoveryDeadlineAt = now + Daemon.ERROR_RECOVERY_TIMEOUT_MS;
|
|
2147
|
+
this.activeErrorPatternKey = key;
|
|
2148
|
+
this.lastDetectedErrorType = ep.type;
|
|
2149
|
+
}
|
|
2150
|
+
this.lastErrorNotifiedAt.set(key, now);
|
|
2151
|
+
if (ep.action === "failover")
|
|
2152
|
+
this.lastFailoverAt = now;
|
|
2153
|
+
const message = this.resolveErrorMessage(pane, ep);
|
|
2154
|
+
// Remember an auth failure for as long as it is unresolved. An auth-broken
|
|
2155
|
+
// CLI cannot finish the turn it is holding, so every other detector in this
|
|
2156
|
+
// scan would otherwise keep reporting the same dead instance forever.
|
|
2157
|
+
if (ep.type === "auth_error")
|
|
2158
|
+
this.authFailureUnresolved = true;
|
|
2159
|
+
this.logger.warn({ errorType: ep.type, action: ep.action }, `PTY error detected: ${message}`);
|
|
2160
|
+
this.emit("pty_error", { name: this.name, ...ep, message });
|
|
2161
|
+
}
|
|
1714
2162
|
/**
|
|
1715
2163
|
* Notification text for a detected pattern. `formatMessage` patterns build it
|
|
1716
2164
|
* from the match so the user gets the specifics (e.g. which period and what
|
|
@@ -1749,6 +2197,97 @@ export class Daemon extends EventEmitter {
|
|
|
1749
2197
|
const cancelKey = this.backend?.getCancelKey() ?? "Escape";
|
|
1750
2198
|
await this.tmux?.sendSpecialKey(cancelKey);
|
|
1751
2199
|
}
|
|
2200
|
+
/**
|
|
2201
|
+
* Drop every not-yet-started pane delivery which predates a user cancel.
|
|
2202
|
+
* Promise chains cannot be removed, so entries become generation-checked
|
|
2203
|
+
* no-ops. An entry already past its check is allowed to finish its current PTY
|
|
2204
|
+
* transaction; the interrupt key still cancels the CLI generation.
|
|
2205
|
+
*/
|
|
2206
|
+
clearPendingDeliveries(fleetEpoch) {
|
|
2207
|
+
this.deliveryEpoch = fleetEpoch === undefined
|
|
2208
|
+
? this.deliveryEpoch + 1
|
|
2209
|
+
: Math.max(this.deliveryEpoch, fleetEpoch);
|
|
2210
|
+
this.logger.info({ deliveryEpoch: this.deliveryEpoch }, "Pending delivery queue cleared by user cancel");
|
|
2211
|
+
}
|
|
2212
|
+
captureDeliveryEpoch(value) {
|
|
2213
|
+
const epoch = typeof value === "number" && Number.isSafeInteger(value) && value >= 0
|
|
2214
|
+
? value
|
|
2215
|
+
: this.deliveryEpoch;
|
|
2216
|
+
if (epoch > this.deliveryEpoch)
|
|
2217
|
+
this.deliveryEpoch = epoch;
|
|
2218
|
+
return epoch;
|
|
2219
|
+
}
|
|
2220
|
+
isDeliveryEpochCurrent(epoch) {
|
|
2221
|
+
return epoch === this.deliveryEpoch;
|
|
2222
|
+
}
|
|
2223
|
+
queueRawPaste(rawText, deliveryEpoch = this.deliveryEpoch, confirmClear = false) {
|
|
2224
|
+
if (!this.tmux || !this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
2225
|
+
return;
|
|
2226
|
+
this.pasteLock = this.pasteLock.then(async () => {
|
|
2227
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch)) {
|
|
2228
|
+
this.logger.info("Pending raw delivery dropped by user cancel");
|
|
2229
|
+
return;
|
|
2230
|
+
}
|
|
2231
|
+
await this.wake();
|
|
2232
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
2233
|
+
return;
|
|
2234
|
+
await this.deliverMessage(rawText, undefined, { deliveryEpoch });
|
|
2235
|
+
if (confirmClear)
|
|
2236
|
+
await this.confirmBackendClearDialog();
|
|
2237
|
+
this.logger.debug({ text: rawText.slice(0, 100) }, "Raw paste delivered");
|
|
2238
|
+
}).catch(err => {
|
|
2239
|
+
this.logger.warn({ err: err.message }, "raw_paste delivery error");
|
|
2240
|
+
});
|
|
2241
|
+
}
|
|
2242
|
+
/**
|
|
2243
|
+
* Confirm a backend's terminal-level /clear prompt after the channel-level
|
|
2244
|
+
* admin confirmation has already succeeded. Poll instead of pasting `/clear\ny`
|
|
2245
|
+
* as one buffer: Kiro versions that clear immediately interpret that queued
|
|
2246
|
+
* `y` as a brand-new user message.
|
|
2247
|
+
*/
|
|
2248
|
+
async confirmBackendClearDialog() {
|
|
2249
|
+
const dialog = this.backend?.getClearConfirmationDialog?.();
|
|
2250
|
+
if (!dialog || !this.tmux)
|
|
2251
|
+
return;
|
|
2252
|
+
const matches = (pane) => {
|
|
2253
|
+
dialog.pattern.lastIndex = 0;
|
|
2254
|
+
return dialog.pattern.test(pane);
|
|
2255
|
+
};
|
|
2256
|
+
const deadline = Date.now() + 3_000;
|
|
2257
|
+
while (Date.now() <= deadline) {
|
|
2258
|
+
const pane = await this.tmux.capturePane().catch(() => "");
|
|
2259
|
+
if (matches(pane)) {
|
|
2260
|
+
const sent = await this.paneWriteLock.run(async () => {
|
|
2261
|
+
// Re-check under the write lock so a prompt that disappeared between
|
|
2262
|
+
// capture and lock acquisition never receives a stray literal `y`.
|
|
2263
|
+
const currentPane = await this.tmux.capturePane().catch(() => "");
|
|
2264
|
+
if (!matches(currentPane))
|
|
2265
|
+
return false;
|
|
2266
|
+
const specialKeys = new Set(["Up", "Down", "Enter", "Escape", "Right", "Left"]);
|
|
2267
|
+
for (const key of dialog.keys) {
|
|
2268
|
+
const ok = specialKeys.has(key)
|
|
2269
|
+
? await this.tmux.sendSpecialKey(key)
|
|
2270
|
+
: await this.tmux.sendKeys(key);
|
|
2271
|
+
if (!ok)
|
|
2272
|
+
return false;
|
|
2273
|
+
await new Promise(resolve => setTimeout(resolve, 100));
|
|
2274
|
+
}
|
|
2275
|
+
return true;
|
|
2276
|
+
});
|
|
2277
|
+
if (sent) {
|
|
2278
|
+
this.logger.info({ dialog: dialog.description }, "Auto-confirmed backend clear dialog");
|
|
2279
|
+
}
|
|
2280
|
+
else {
|
|
2281
|
+
this.logger.warn({ dialog: dialog.description }, "Backend clear dialog disappeared or confirmation keys failed");
|
|
2282
|
+
}
|
|
2283
|
+
return;
|
|
2284
|
+
}
|
|
2285
|
+
await new Promise(resolve => setTimeout(resolve, 100));
|
|
2286
|
+
}
|
|
2287
|
+
// Recent Kiro releases may clear immediately without a second prompt. That
|
|
2288
|
+
// is a successful no-op here, not an error.
|
|
2289
|
+
this.logger.debug({ dialog: dialog.description }, "Backend clear dialog did not appear");
|
|
2290
|
+
}
|
|
1752
2291
|
/** Send the backend-specific graceful quit command/key sequence. */
|
|
1753
2292
|
async sendQuitSequence() {
|
|
1754
2293
|
if (!this.tmux || !this.backend)
|
|
@@ -1756,18 +2295,20 @@ export class Daemon extends EventEmitter {
|
|
|
1756
2295
|
const quitCmd = this.backend.getQuitCommand();
|
|
1757
2296
|
const quitKey = this.backend.getQuitKey?.();
|
|
1758
2297
|
if (quitCmd) {
|
|
1759
|
-
await this.tmux.sendKeys(quitCmd)
|
|
2298
|
+
if (!await this.tmux.sendKeys(quitCmd))
|
|
2299
|
+
return false;
|
|
1760
2300
|
// Delay before Enter to prevent tmux server races when instances stop in
|
|
1761
2301
|
// parallel (same pattern as pasteText).
|
|
1762
2302
|
await new Promise(r => setTimeout(r, 150));
|
|
1763
|
-
|
|
1764
|
-
return true;
|
|
2303
|
+
return this.tmux.sendSpecialKey("Enter");
|
|
1765
2304
|
}
|
|
1766
2305
|
if (!quitKey)
|
|
1767
2306
|
return false;
|
|
1768
2307
|
const presses = Math.max(1, Math.floor(this.backend.getQuitKeyPresses?.() ?? 1));
|
|
1769
2308
|
for (let i = 0; i < presses; i++) {
|
|
1770
|
-
await this.tmux.sendSpecialKey(quitKey);
|
|
2309
|
+
const sent = await this.tmux.sendSpecialKey(quitKey);
|
|
2310
|
+
if (!sent)
|
|
2311
|
+
return false;
|
|
1771
2312
|
if (i + 1 < presses)
|
|
1772
2313
|
await new Promise(r => setTimeout(r, 250));
|
|
1773
2314
|
}
|
|
@@ -1800,8 +2341,23 @@ export class Daemon extends EventEmitter {
|
|
|
1800
2341
|
}
|
|
1801
2342
|
}
|
|
1802
2343
|
}
|
|
1803
|
-
if (!killed)
|
|
1804
|
-
this.logger.warn("CLI did not exit gracefully within 3s
|
|
2344
|
+
if (!killed) {
|
|
2345
|
+
this.logger.warn({ quitSent }, "CLI did not exit gracefully within 3s — falling back to SIGTERM");
|
|
2346
|
+
await this.killProcessTree("SIGTERM");
|
|
2347
|
+
for (let i = 0; i < 5; i++) {
|
|
2348
|
+
await new Promise(r => setTimeout(r, 200));
|
|
2349
|
+
const status = await this.tmux.getPaneStatus();
|
|
2350
|
+
if (!status || !status.alive) {
|
|
2351
|
+
killed = true;
|
|
2352
|
+
break;
|
|
2353
|
+
}
|
|
2354
|
+
}
|
|
2355
|
+
}
|
|
2356
|
+
if (!killed) {
|
|
2357
|
+
this.logger.warn("CLI process tree survived SIGTERM — falling back to SIGKILL");
|
|
2358
|
+
await this.killProcessTree("SIGKILL");
|
|
2359
|
+
await new Promise(r => setTimeout(r, 200));
|
|
2360
|
+
}
|
|
1805
2361
|
// Always kill window — remain-on-exit keeps dead panes around after CLI exits
|
|
1806
2362
|
const stoppedWindowId = this.tmux.getWindowId();
|
|
1807
2363
|
await this.tmux.killWindow();
|
|
@@ -1850,6 +2406,33 @@ export class Daemon extends EventEmitter {
|
|
|
1850
2406
|
stateChangedAt: Date.now(),
|
|
1851
2407
|
};
|
|
1852
2408
|
}
|
|
2409
|
+
/**
|
|
2410
|
+
* Answer a FleetManager state query. Most callers only need the cached state,
|
|
2411
|
+
* but lifecycle decisions such as post-reply Cancel retirement need an
|
|
2412
|
+
* authoritative pane observation: after startup the last transition can stay
|
|
2413
|
+
* "working" even though the CLI has since settled without emitting another
|
|
2414
|
+
* control-mode output record.
|
|
2415
|
+
*/
|
|
2416
|
+
async respondToInstanceStateQuery(msg, socket) {
|
|
2417
|
+
try {
|
|
2418
|
+
if (msg.refresh === true) {
|
|
2419
|
+
await this.captureAndEvaluateInstanceState("state_query");
|
|
2420
|
+
}
|
|
2421
|
+
const snapshot = this.getInstanceStateSnapshot();
|
|
2422
|
+
this.ipcServer?.send(socket, {
|
|
2423
|
+
type: "instance_state_response",
|
|
2424
|
+
requestId: msg.requestId,
|
|
2425
|
+
instanceName: this.name,
|
|
2426
|
+
...snapshot,
|
|
2427
|
+
state: this.isPaused ? "paused" : snapshot.state,
|
|
2428
|
+
processStatus: this.processStatus,
|
|
2429
|
+
pausedAt: this.lastPausedAt,
|
|
2430
|
+
});
|
|
2431
|
+
}
|
|
2432
|
+
catch (err) {
|
|
2433
|
+
this.logger.debug({ err: err.message }, "Instance state query failed");
|
|
2434
|
+
}
|
|
2435
|
+
}
|
|
1853
2436
|
/** Gracefully stop the CLI while keeping its remain-on-exit tmux window. */
|
|
1854
2437
|
/**
|
|
1855
2438
|
* Mark the instance to pause as soon as its pane is idle. Used for auth
|
|
@@ -1869,7 +2452,14 @@ export class Daemon extends EventEmitter {
|
|
|
1869
2452
|
if (this.getPauseWakeState() !== "active")
|
|
1870
2453
|
return;
|
|
1871
2454
|
}
|
|
1872
|
-
|
|
2455
|
+
// `pauseAllowStuck` is consumed here, not merely read: it authorises exactly
|
|
2456
|
+
// one pause from a stuck pane (the auth-deferred one) and must never leak
|
|
2457
|
+
// into an ordinary idle-timeout pause of a busy instance.
|
|
2458
|
+
const allowStuck = this.pauseAllowStuck;
|
|
2459
|
+
this.pauseAllowStuck = false;
|
|
2460
|
+
const pausableState = this.instanceState === "idle"
|
|
2461
|
+
|| (allowStuck && this.instanceState === "stuck");
|
|
2462
|
+
if (!pausableState || this.pasteQueueDepth > 0) {
|
|
1873
2463
|
this.pauseRequested = false;
|
|
1874
2464
|
return;
|
|
1875
2465
|
}
|
|
@@ -1933,6 +2523,10 @@ export class Daemon extends EventEmitter {
|
|
|
1933
2523
|
// An explicit wake (e.g. after the user re-logs in) cancels a deferred
|
|
1934
2524
|
// auth pause — otherwise the instance would pause again the moment it idles.
|
|
1935
2525
|
this.pausePending = false;
|
|
2526
|
+
this.authFailureUnresolved = false;
|
|
2527
|
+
this.loginScreenReported = false;
|
|
2528
|
+
this.fatalStartupReported = false;
|
|
2529
|
+
this.fatalStartupBlocked = false;
|
|
1936
2530
|
if (this.pauseWakeState === "active")
|
|
1937
2531
|
return;
|
|
1938
2532
|
if (this.pauseWakeState === "pausing")
|
|
@@ -1993,6 +2587,14 @@ export class Daemon extends EventEmitter {
|
|
|
1993
2587
|
applyInstanceStateSnapshot(snapshot, pane) {
|
|
1994
2588
|
const previous = this.instanceState;
|
|
1995
2589
|
this.instanceState = snapshot.state;
|
|
2590
|
+
// OpenCode creates its session lazily on the first submitted message.
|
|
2591
|
+
// Waiting until stop/pause to persist that id loses resume state when the
|
|
2592
|
+
// fleet is SIGKILLed or the host reboots.
|
|
2593
|
+
// Idle observations are safe checkpoints; the 60s safety sweep also gives
|
|
2594
|
+
// us a bounded retry if session creation produced no visible state edge.
|
|
2595
|
+
if (snapshot.state === "idle" && this.backend?.binaryName === "opencode") {
|
|
2596
|
+
this.saveSessionId();
|
|
2597
|
+
}
|
|
1996
2598
|
// Only a transition back to idle completes pending work. Repeated idle
|
|
1997
2599
|
// observations between enqueue and paste must not clear a newer inbound.
|
|
1998
2600
|
if (snapshot.state === "idle" && previous !== "idle") {
|
|
@@ -2001,6 +2603,7 @@ export class Daemon extends EventEmitter {
|
|
|
2001
2603
|
// tool_result (interrupted, crashed, cancelled), which would otherwise leave
|
|
2002
2604
|
// the last tool pinned to the progress line for the rest of the session.
|
|
2003
2605
|
this.publishActivity(null);
|
|
2606
|
+
this.resetToolProgress();
|
|
2004
2607
|
// Must run before the mcpRestartPending branch below: the pane text is the
|
|
2005
2608
|
// only copy of the answer, and the revival restart is about to clear it.
|
|
2006
2609
|
this.maybeProxyReplyOnTurnEnd(pane);
|
|
@@ -2020,8 +2623,13 @@ export class Daemon extends EventEmitter {
|
|
|
2020
2623
|
if (snapshot.state !== "idle")
|
|
2021
2624
|
this.pauseRequested = false;
|
|
2022
2625
|
// A pause deferred by an auth failure: retry the moment the pane settles.
|
|
2023
|
-
|
|
2626
|
+
// "stuck" counts as settled here, and that is the whole point — an expired
|
|
2627
|
+
// session cannot complete the turn it is holding, so the pane never returns
|
|
2628
|
+
// to idle and waiting for idle meant waiting forever. The instance stayed
|
|
2629
|
+
// live, kept its warm slot, and re-raised a hang alert on every scan.
|
|
2630
|
+
if (this.pausePending && (snapshot.state === "idle" || snapshot.state === "stuck") && this.pasteQueueDepth === 0) {
|
|
2024
2631
|
this.pausePending = false;
|
|
2632
|
+
this.pauseAllowStuck = snapshot.state === "stuck";
|
|
2025
2633
|
this.emit("auto_pause_requested", { name: this.name, idleSince: snapshot.stateChangedAt });
|
|
2026
2634
|
return;
|
|
2027
2635
|
}
|
|
@@ -2051,6 +2659,7 @@ export class Daemon extends EventEmitter {
|
|
|
2051
2659
|
return;
|
|
2052
2660
|
this.turnHadInbound = true;
|
|
2053
2661
|
this.turnOutboundDelivered = false;
|
|
2662
|
+
this.turnReplyDelivered = false;
|
|
2054
2663
|
this.turnCorrelationId = meta.correlation_id || undefined;
|
|
2055
2664
|
// The last non-empty line of what we pasted: everything on screen after it
|
|
2056
2665
|
// is the agent's own output.
|
|
@@ -2066,13 +2675,38 @@ export class Daemon extends EventEmitter {
|
|
|
2066
2675
|
maybeProxyReplyOnTurnEnd(pane) {
|
|
2067
2676
|
const hadInbound = this.turnHadInbound;
|
|
2068
2677
|
const delivered = this.turnOutboundDelivered;
|
|
2678
|
+
const replyDelivered = this.turnReplyDelivered;
|
|
2069
2679
|
const correlationId = this.turnCorrelationId;
|
|
2070
2680
|
const inboundMarker = this.turnInboundMarker;
|
|
2071
2681
|
this.turnHadInbound = false;
|
|
2072
2682
|
this.turnOutboundDelivered = false;
|
|
2683
|
+
this.turnReplyDelivered = false;
|
|
2073
2684
|
this.turnCorrelationId = undefined;
|
|
2074
2685
|
this.turnInboundMarker = undefined;
|
|
2075
|
-
if (!hadInbound ||
|
|
2686
|
+
if (!hadInbound || this.isPaused)
|
|
2687
|
+
return;
|
|
2688
|
+
// Claude sometimes prints a broken XML tool call as plain text and returns
|
|
2689
|
+
// idle without ever invoking `reply`. This is independent of MCP liveness:
|
|
2690
|
+
// the model malformed the call before the server could receive it.
|
|
2691
|
+
if (!replyDelivered && this.isClaudeCodeBackend() && pane) {
|
|
2692
|
+
const malformed = detectMalformedClaudeToolCall(pane, { inboundMarker });
|
|
2693
|
+
if (malformed) {
|
|
2694
|
+
const stale = malformed.signature === this.lastMalformedToolCallSignature;
|
|
2695
|
+
this.lastMalformedToolCallSignature = malformed.signature;
|
|
2696
|
+
if (!stale) {
|
|
2697
|
+
const recovered = malformed.text != null;
|
|
2698
|
+
this.logger.warn({ correlationId, recovered }, recovered
|
|
2699
|
+
? "Malformed Claude tool call detected — attempting to recover reply text"
|
|
2700
|
+
: "Malformed Claude tool call detected — reply text could not be extracted");
|
|
2701
|
+
this.emit("malformed_tool_call", { name: this.name, correlationId, recovered });
|
|
2702
|
+
if (malformed.text)
|
|
2703
|
+
this.sendRecoveredMalformedReply(malformed.text, correlationId);
|
|
2704
|
+
}
|
|
2705
|
+
// Never let the broader dead-MCP fallback relay the XML/chrome too.
|
|
2706
|
+
return;
|
|
2707
|
+
}
|
|
2708
|
+
}
|
|
2709
|
+
if (delivered)
|
|
2076
2710
|
return;
|
|
2077
2711
|
// Opt-in: raw pane text can carry secrets past the regex redaction.
|
|
2078
2712
|
if (this.config.mcp_proxy_reply !== true)
|
|
@@ -2082,6 +2716,11 @@ export class Daemon extends EventEmitter {
|
|
|
2082
2716
|
return;
|
|
2083
2717
|
void this.sendProxyReply(pane, inboundMarker, correlationId);
|
|
2084
2718
|
}
|
|
2719
|
+
isClaudeCodeBackend() {
|
|
2720
|
+
return this.runtimeIdentity?.backend === "claude-code"
|
|
2721
|
+
|| this.config.backend === "claude-code"
|
|
2722
|
+
|| this.backend?.binaryName === "claude";
|
|
2723
|
+
}
|
|
2085
2724
|
/** Relay the pane's final text to the channel, marked as a daemon proxy reply. */
|
|
2086
2725
|
async sendProxyReply(pane, inboundMarker, correlationId) {
|
|
2087
2726
|
try {
|
|
@@ -2099,55 +2738,92 @@ export class Daemon extends EventEmitter {
|
|
|
2099
2738
|
let body = `⚠️ [MCP unavailable — proxy reply]\n\n${text}`;
|
|
2100
2739
|
if (correlationId)
|
|
2101
2740
|
body += `\n\n(correlation_id: ${correlationId})`;
|
|
2102
|
-
const args = { text: body };
|
|
2103
|
-
// Same context-bound routing as the reply tool in handleToolCall.
|
|
2104
|
-
if (this.lastChatId) {
|
|
2105
|
-
args.chat_id = this.lastChatId;
|
|
2106
|
-
if (this.lastThreadId)
|
|
2107
|
-
args.thread_id = this.lastThreadId;
|
|
2108
|
-
}
|
|
2109
2741
|
this.logger.warn({ correlationId }, "MCP server dead and the turn sent no reply — relaying the pane text as a proxy reply");
|
|
2110
2742
|
this.emit("mcp_proxy_reply", { name: this.name, correlationId });
|
|
2111
|
-
|
|
2112
|
-
if (adapters.length > 0) {
|
|
2113
|
-
routeToolCall(adapters[0], "reply", args, this.lastThreadId, (_result, error) => {
|
|
2114
|
-
if (error)
|
|
2115
|
-
this.logger.error({ error }, "Dead-MCP proxy reply failed");
|
|
2116
|
-
});
|
|
2117
|
-
return;
|
|
2118
|
-
}
|
|
2119
|
-
if (!this.ipcServer)
|
|
2120
|
-
return;
|
|
2121
|
-
const fleetReqId = `proxyreply_${++this.proxyReplySeq}`;
|
|
2122
|
-
this.ipcServer.broadcast({
|
|
2123
|
-
type: "fleet_outbound",
|
|
2124
|
-
tool: "reply",
|
|
2125
|
-
args,
|
|
2126
|
-
fleetRequestId: fleetReqId,
|
|
2127
|
-
adapterId: this.lastAdapterId,
|
|
2128
|
-
});
|
|
2129
|
-
const timeout = setTimeout(() => {
|
|
2130
|
-
this.pendingIpcRequests.delete(fleetReqId);
|
|
2131
|
-
this.logger.error("Dead-MCP proxy reply timed out waiting for the fleet manager");
|
|
2132
|
-
}, daemonBudgetMs("reply"));
|
|
2133
|
-
timeout.unref?.();
|
|
2134
|
-
this.pendingIpcRequests.set(fleetReqId, (respMsg) => {
|
|
2135
|
-
clearTimeout(timeout);
|
|
2136
|
-
if (respMsg.error)
|
|
2137
|
-
this.logger.error({ error: respMsg.error }, "Dead-MCP proxy reply failed");
|
|
2138
|
-
else
|
|
2139
|
-
this.logger.info("Dead-MCP proxy reply delivered");
|
|
2140
|
-
});
|
|
2743
|
+
this.deliverDaemonReply(body, "proxyreply", "Dead-MCP proxy reply");
|
|
2141
2744
|
}
|
|
2142
2745
|
catch (err) {
|
|
2143
2746
|
this.logger.error({ err: err.message }, "Dead-MCP proxy reply attempt failed");
|
|
2144
2747
|
}
|
|
2145
2748
|
}
|
|
2749
|
+
/** Send a narrowly extracted #648 reply; operator notification is a separate event. */
|
|
2750
|
+
sendRecoveredMalformedReply(text, correlationId) {
|
|
2751
|
+
try {
|
|
2752
|
+
this.deliverDaemonReply(text, "malformedreply", "Malformed tool-call recovery");
|
|
2753
|
+
}
|
|
2754
|
+
catch (err) {
|
|
2755
|
+
this.logger.error({ err: err.message, correlationId }, "Malformed tool-call recovery attempt failed");
|
|
2756
|
+
}
|
|
2757
|
+
}
|
|
2758
|
+
/** Context-bound reply path shared by dead-MCP and malformed-call recovery. */
|
|
2759
|
+
deliverDaemonReply(body, requestPrefix, logLabel) {
|
|
2760
|
+
const args = { text: body };
|
|
2761
|
+
if (this.lastChatId) {
|
|
2762
|
+
args.chat_id = this.lastChatId;
|
|
2763
|
+
if (this.lastThreadId)
|
|
2764
|
+
args.thread_id = this.lastThreadId;
|
|
2765
|
+
}
|
|
2766
|
+
const adapters = this.messageBus.getAllAdapters();
|
|
2767
|
+
if (adapters.length > 0) {
|
|
2768
|
+
routeToolCall(adapters[0], "reply", args, this.lastThreadId, (_result, error) => {
|
|
2769
|
+
if (error)
|
|
2770
|
+
this.logger.error({ error }, `${logLabel} failed`);
|
|
2771
|
+
else
|
|
2772
|
+
this.logger.info(`${logLabel} delivered`);
|
|
2773
|
+
});
|
|
2774
|
+
return;
|
|
2775
|
+
}
|
|
2776
|
+
if (!this.ipcServer) {
|
|
2777
|
+
this.logger.error(`${logLabel} failed — no adapter or fleet IPC route`);
|
|
2778
|
+
return;
|
|
2779
|
+
}
|
|
2780
|
+
const fleetReqId = `${requestPrefix}_${++this.proxyReplySeq}`;
|
|
2781
|
+
const timeout = setTimeout(() => {
|
|
2782
|
+
this.pendingIpcRequests.delete(fleetReqId);
|
|
2783
|
+
this.logger.error(`${logLabel} timed out waiting for the fleet manager`);
|
|
2784
|
+
}, daemonBudgetMs("reply"));
|
|
2785
|
+
timeout.unref?.();
|
|
2786
|
+
// Register before broadcast: an in-process/fast adapter can answer in the
|
|
2787
|
+
// same tick, and a waiter installed afterwards would miss that response.
|
|
2788
|
+
this.pendingIpcRequests.set(fleetReqId, (respMsg) => {
|
|
2789
|
+
clearTimeout(timeout);
|
|
2790
|
+
if (respMsg.error)
|
|
2791
|
+
this.logger.error({ error: respMsg.error }, `${logLabel} failed`);
|
|
2792
|
+
else
|
|
2793
|
+
this.logger.info(`${logLabel} delivered`);
|
|
2794
|
+
});
|
|
2795
|
+
this.ipcServer.broadcast({
|
|
2796
|
+
type: "fleet_outbound",
|
|
2797
|
+
tool: "reply",
|
|
2798
|
+
args,
|
|
2799
|
+
fleetRequestId: fleetReqId,
|
|
2800
|
+
adapterId: this.lastAdapterId,
|
|
2801
|
+
});
|
|
2802
|
+
}
|
|
2146
2803
|
clearInstanceStateIdleTimer() {
|
|
2147
2804
|
if (this.instanceStateIdleTimer)
|
|
2148
2805
|
clearTimeout(this.instanceStateIdleTimer);
|
|
2149
2806
|
this.instanceStateIdleTimer = null;
|
|
2150
2807
|
}
|
|
2808
|
+
clearInstanceStateOutputProbeTimer() {
|
|
2809
|
+
if (this.instanceStateOutputProbeTimer)
|
|
2810
|
+
clearTimeout(this.instanceStateOutputProbeTimer);
|
|
2811
|
+
this.instanceStateOutputProbeTimer = null;
|
|
2812
|
+
}
|
|
2813
|
+
/**
|
|
2814
|
+
* Periodic status-line hooks can generate `%output` without changing a single
|
|
2815
|
+
* visible cell. Probe the pane once per burst: a changed capture is real work;
|
|
2816
|
+
* an identical ready capture remains idle and emits no state edge.
|
|
2817
|
+
*/
|
|
2818
|
+
scheduleInstanceStateOutputProbe() {
|
|
2819
|
+
if (this.instanceStateOutputProbeTimer)
|
|
2820
|
+
return;
|
|
2821
|
+
this.instanceStateOutputProbeTimer = setTimeout(() => {
|
|
2822
|
+
this.instanceStateOutputProbeTimer = null;
|
|
2823
|
+
void this.captureAndEvaluateInstanceState("output_probe", this.instanceStateLastOutputAt);
|
|
2824
|
+
}, PERIODIC_REDRAW_PROBE_MS);
|
|
2825
|
+
this.instanceStateOutputProbeTimer.unref?.();
|
|
2826
|
+
}
|
|
2151
2827
|
clearInstanceStateStuckTimer() {
|
|
2152
2828
|
if (this.instanceStateStuckTimer)
|
|
2153
2829
|
clearTimeout(this.instanceStateStuckTimer);
|
|
@@ -2192,6 +2868,8 @@ export class Daemon extends EventEmitter {
|
|
|
2192
2868
|
if (this.statePollInFlight) {
|
|
2193
2869
|
if (reason === "idle_debounce")
|
|
2194
2870
|
this.scheduleInstanceStateIdleCapture();
|
|
2871
|
+
if (reason === "output_probe")
|
|
2872
|
+
this.scheduleInstanceStateOutputProbe();
|
|
2195
2873
|
return;
|
|
2196
2874
|
}
|
|
2197
2875
|
this.statePollInFlight = true;
|
|
@@ -2200,9 +2878,19 @@ export class Daemon extends EventEmitter {
|
|
|
2200
2878
|
const pane = await this.tmux.capturePane();
|
|
2201
2879
|
// Output received while capture-pane was in flight makes this snapshot
|
|
2202
2880
|
// stale. Its output handler has already armed a new debounce.
|
|
2203
|
-
|
|
2204
|
-
|| (this.instanceStateLastOutputAt > 0 && this.instanceStateLastOutputAt >= captureStartedAt)
|
|
2881
|
+
const outputMovedDuringCapture = (expectedOutputAt > 0 && this.instanceStateLastOutputAt > expectedOutputAt)
|
|
2882
|
+
|| (this.instanceStateLastOutputAt > 0 && this.instanceStateLastOutputAt >= captureStartedAt);
|
|
2883
|
+
if (outputMovedDuringCapture) {
|
|
2884
|
+
if (reason === "output_probe") {
|
|
2885
|
+
// A second burst overtook the probe. Do not bless its stale capture as
|
|
2886
|
+
// idle: continuous output is the safer working signal, while the next
|
|
2887
|
+
// quiet probe/debounce can still prove that it was only repainting.
|
|
2888
|
+
this.applyInstanceStateSnapshot(this.instanceStateMachine.recordOutput(this.instanceStateLastOutputAt));
|
|
2889
|
+
this.scheduleInstanceStateIdleCapture();
|
|
2890
|
+
this.scheduleInstanceStateStuckDeadline(this.instanceStateLastOutputAt);
|
|
2891
|
+
}
|
|
2205
2892
|
return;
|
|
2893
|
+
}
|
|
2206
2894
|
const observedChangeAt = expectedOutputAt || captureStartedAt;
|
|
2207
2895
|
// One observation per capture. This used to call observe() twice with the
|
|
2208
2896
|
// same pane: the second call always saw an unchanged pane, so the
|
|
@@ -2211,6 +2899,7 @@ export class Daemon extends EventEmitter {
|
|
|
2211
2899
|
// mismatched spinner frame was enough to report idle mid-turn.
|
|
2212
2900
|
const settled = this.instanceStateLastOutputAt === 0
|
|
2213
2901
|
|| captureStartedAt - this.instanceStateLastOutputAt >= this.instanceStateIdleDebounceMs;
|
|
2902
|
+
const paneActivity = this.backend?.getPaneActivity?.(pane) ?? null;
|
|
2214
2903
|
// The two times are genuinely different and both matter: the content
|
|
2215
2904
|
// changed when tmux reported output (observedChangeAt), but idle/stuck are
|
|
2216
2905
|
// decisions about *now*. The old double-observe expressed that by calling
|
|
@@ -2218,6 +2907,7 @@ export class Daemon extends EventEmitter {
|
|
|
2218
2907
|
const snapshot = this.instanceStateMachine.observe(pane, Date.now(), {
|
|
2219
2908
|
settled,
|
|
2220
2909
|
changeAt: observedChangeAt,
|
|
2910
|
+
forceBusy: paneActivity !== null,
|
|
2221
2911
|
});
|
|
2222
2912
|
this.applyInstanceStateSnapshot(snapshot, pane);
|
|
2223
2913
|
// Backends without a transcript feed can still say what they are doing, if
|
|
@@ -2225,7 +2915,7 @@ export class Daemon extends EventEmitter {
|
|
|
2225
2915
|
// extra tmux call. Cadence is the capture cadence (idle debounce + the 60s
|
|
2226
2916
|
// safety sweep), which matches the progress ticker's own 60s interval.
|
|
2227
2917
|
if (this.backend?.getPaneActivity) {
|
|
2228
|
-
this.publishActivity(snapshot.state === "idle" ? null :
|
|
2918
|
+
this.publishActivity(snapshot.state === "idle" ? null : paneActivity);
|
|
2229
2919
|
}
|
|
2230
2920
|
if (snapshot.state === "idle") {
|
|
2231
2921
|
this.clearInstanceStateStuckTimer();
|
|
@@ -2252,6 +2942,14 @@ export class Daemon extends EventEmitter {
|
|
|
2252
2942
|
if (!windowId || event.windowId !== windowId || !this.instanceStateMachine)
|
|
2253
2943
|
return;
|
|
2254
2944
|
this.instanceStateLastOutputAt = event.at;
|
|
2945
|
+
if (this.backend?.hasPeriodicPaneRedraw?.() === true && this.instanceState === "idle") {
|
|
2946
|
+
// Do not emit idle→working solely because agy repainted an identical
|
|
2947
|
+
// footer. The short capture probe below still observes genuine pane
|
|
2948
|
+
// changes promptly and moves the state to working.
|
|
2949
|
+
this.scheduleInstanceStateOutputProbe();
|
|
2950
|
+
this.scheduleInstanceStateIdleCapture();
|
|
2951
|
+
return;
|
|
2952
|
+
}
|
|
2255
2953
|
const snapshot = this.instanceStateMachine.recordOutput(event.at);
|
|
2256
2954
|
this.applyInstanceStateSnapshot(snapshot);
|
|
2257
2955
|
this.scheduleInstanceStateIdleCapture();
|
|
@@ -2302,6 +3000,7 @@ export class Daemon extends EventEmitter {
|
|
|
2302
3000
|
stopInstanceStateMonitor() {
|
|
2303
3001
|
this.instanceStateMonitorActive = false;
|
|
2304
3002
|
this.clearInstanceStateIdleTimer();
|
|
3003
|
+
this.clearInstanceStateOutputProbeTimer();
|
|
2305
3004
|
this.clearInstanceStateStuckTimer();
|
|
2306
3005
|
if (this.instanceStateMonitorTimer)
|
|
2307
3006
|
clearInterval(this.instanceStateMonitorTimer);
|
|
@@ -2336,6 +3035,15 @@ export class Daemon extends EventEmitter {
|
|
|
2336
3035
|
this.logger.debug(diagnostic, "Suppressing stuck notification without pending work");
|
|
2337
3036
|
return;
|
|
2338
3037
|
}
|
|
3038
|
+
// An auth-broken instance IS stuck, and stays stuck: it cannot finish the
|
|
3039
|
+
// turn it is holding, so pendingWork never clears and this fired on every
|
|
3040
|
+
// scan forever. The notification is also useless — it offers Restart/Wait
|
|
3041
|
+
// and asks General for help, but only the user re-logging in fixes this, and
|
|
3042
|
+
// they have already been told exactly that by the auth alert.
|
|
3043
|
+
if (this.authFailureUnresolved) {
|
|
3044
|
+
this.logger.debug(diagnostic, "Suppressing stuck notification — unresolved auth failure (re-login required)");
|
|
3045
|
+
return;
|
|
3046
|
+
}
|
|
2339
3047
|
this.logger.warn(diagnostic, "Instance pane stuck with pending work");
|
|
2340
3048
|
this.hangDetector?.emit("hang", { unchangedForMs: snapshot.unchangedForMs });
|
|
2341
3049
|
}
|
|
@@ -2355,6 +3063,7 @@ export class Daemon extends EventEmitter {
|
|
|
2355
3063
|
// fire into a stopped daemon).
|
|
2356
3064
|
this.clearMcpRestartRequest();
|
|
2357
3065
|
this.interactivePromptDetector.reset();
|
|
3066
|
+
this.blockingProcessDetector.reset();
|
|
2358
3067
|
this.stopInstanceStateMonitor();
|
|
2359
3068
|
this.transcriptMonitor?.stop();
|
|
2360
3069
|
this.guardian?.stop();
|
|
@@ -2380,9 +3089,9 @@ export class Daemon extends EventEmitter {
|
|
|
2380
3089
|
* Tell the fleet manager what this instance is doing right now, for the live
|
|
2381
3090
|
* progress line on the cancel button.
|
|
2382
3091
|
*
|
|
2383
|
-
*
|
|
2384
|
-
*
|
|
2385
|
-
*
|
|
3092
|
+
* Primarily cosmetic. The foreground-process detector also uses a current
|
|
3093
|
+
* shell activity as one of several positive signals, but never makes a state
|
|
3094
|
+
* decision from an arbitrary activity label alone.
|
|
2386
3095
|
*
|
|
2387
3096
|
* Repeats are dropped: the ticker only edits the channel message when the text
|
|
2388
3097
|
* changes, and a stream of identical broadcasts would defeat that.
|
|
@@ -2398,6 +3107,141 @@ export class Daemon extends EventEmitter {
|
|
|
2398
3107
|
activity: next,
|
|
2399
3108
|
});
|
|
2400
3109
|
}
|
|
3110
|
+
/** Effective tool_progress level, hardened against junk config values.
|
|
3111
|
+
* Off unless explicitly enabled: an update must never make bubbles start
|
|
3112
|
+
* listing tool activity the operator did not ask to broadcast. */
|
|
3113
|
+
toolProgressLevel() {
|
|
3114
|
+
const raw = this.config.tool_progress;
|
|
3115
|
+
return raw === "standard" || raw === "verbose" ? raw : "off";
|
|
3116
|
+
}
|
|
3117
|
+
/**
|
|
3118
|
+
* Apply a tool-progress setting without restarting the CLI session.
|
|
3119
|
+
*
|
|
3120
|
+
* Drop the current accumulator at the boundary: verbose entries may contain
|
|
3121
|
+
* command previews which must not survive a downgrade to standard/off, and a
|
|
3122
|
+
* newly enabled level should start with a clean, internally consistent list.
|
|
3123
|
+
*/
|
|
3124
|
+
updateToolProgress(level) {
|
|
3125
|
+
const previous = this.toolProgressLevel();
|
|
3126
|
+
this.config.tool_progress = level;
|
|
3127
|
+
if (this.toolProgressLevel() !== previous)
|
|
3128
|
+
this.resetToolProgress();
|
|
3129
|
+
}
|
|
3130
|
+
/** Snapshot the configuration actually owned by this live daemon. */
|
|
3131
|
+
getConfigSnapshot() {
|
|
3132
|
+
return structuredClone(this.config);
|
|
3133
|
+
}
|
|
3134
|
+
/**
|
|
3135
|
+
* Apply the whitelisted hot configuration carried by FleetManager IPC.
|
|
3136
|
+
* null removes an optional value; invalid/unlisted fields are ignored rather
|
|
3137
|
+
* than allowing another client on the per-instance socket to mutate cold
|
|
3138
|
+
* process/session settings.
|
|
3139
|
+
*/
|
|
3140
|
+
applyConfigUpdate(value) {
|
|
3141
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
3142
|
+
return;
|
|
3143
|
+
const update = value;
|
|
3144
|
+
if (update.tool_progress === null || ["off", "standard", "verbose"].includes(String(update.tool_progress))) {
|
|
3145
|
+
this.updateToolProgress(update.tool_progress === null ? undefined : update.tool_progress);
|
|
3146
|
+
}
|
|
3147
|
+
if (typeof update.mcp_proxy_reply === "boolean")
|
|
3148
|
+
this.config.mcp_proxy_reply = update.mcp_proxy_reply;
|
|
3149
|
+
else if (update.mcp_proxy_reply === null)
|
|
3150
|
+
delete this.config.mcp_proxy_reply;
|
|
3151
|
+
if (typeof update.warm_cap === "number" && Number.isInteger(update.warm_cap) && update.warm_cap >= 0)
|
|
3152
|
+
this.config.warm_cap = update.warm_cap;
|
|
3153
|
+
else if (update.warm_cap === null)
|
|
3154
|
+
delete this.config.warm_cap;
|
|
3155
|
+
const nextAutoPause = typeof update.auto_pause_after === "number"
|
|
3156
|
+
&& Number.isFinite(update.auto_pause_after)
|
|
3157
|
+
&& update.auto_pause_after >= 0
|
|
3158
|
+
? update.auto_pause_after
|
|
3159
|
+
: update.auto_pause_after === null ? 0 : undefined;
|
|
3160
|
+
if (nextAutoPause !== undefined) {
|
|
3161
|
+
const previousAutoPause = typeof this.config.auto_pause_after === "number" ? this.config.auto_pause_after : 0;
|
|
3162
|
+
if (update.auto_pause_after === null)
|
|
3163
|
+
delete this.config.auto_pause_after;
|
|
3164
|
+
else
|
|
3165
|
+
this.config.auto_pause_after = nextAutoPause;
|
|
3166
|
+
if (nextAutoPause !== previousAutoPause) {
|
|
3167
|
+
const isGeneral = this.config.general_topic === true || this.name === "general";
|
|
3168
|
+
this.autoPauseController.reconfigure((isGeneral ? 0 : nextAutoPause) * 60_000);
|
|
3169
|
+
}
|
|
3170
|
+
}
|
|
3171
|
+
for (const key of ["display_name", "description"]) {
|
|
3172
|
+
if (typeof update[key] === "string")
|
|
3173
|
+
this.config[key] = update[key];
|
|
3174
|
+
else if (update[key] === null)
|
|
3175
|
+
delete this.config[key];
|
|
3176
|
+
}
|
|
3177
|
+
if (Array.isArray(update.tags) && update.tags.every(tag => typeof tag === "string"))
|
|
3178
|
+
this.config.tags = [...update.tags];
|
|
3179
|
+
else if (update.tags === null)
|
|
3180
|
+
delete this.config.tags;
|
|
3181
|
+
if (["trace", "debug", "info", "warn", "error"].includes(String(update.log_level))) {
|
|
3182
|
+
const level = update.log_level;
|
|
3183
|
+
if (this.config.log_level !== level) {
|
|
3184
|
+
this.config.log_level = level;
|
|
3185
|
+
this.logger.level = level;
|
|
3186
|
+
}
|
|
3187
|
+
}
|
|
3188
|
+
}
|
|
3189
|
+
/**
|
|
3190
|
+
* Accumulate one semantic progress line for the channel bubble and schedule
|
|
3191
|
+
* a coalesced broadcast. Separate from publishActivity on purpose: that one
|
|
3192
|
+
* is the single-line statusline detail (terse, operator-facing), this is the
|
|
3193
|
+
* multi-line channel list (semantic, argument-free at `standard`) — see
|
|
3194
|
+
* tool-progress.ts for why the two labellers must not merge.
|
|
3195
|
+
*/
|
|
3196
|
+
recordToolProgress(name, input) {
|
|
3197
|
+
const level = this.toolProgressLevel();
|
|
3198
|
+
if (level === "off")
|
|
3199
|
+
return;
|
|
3200
|
+
if (this.turnProgress.add(summarizeProgress(name, input, level))) {
|
|
3201
|
+
this.scheduleProgressBroadcast();
|
|
3202
|
+
}
|
|
3203
|
+
}
|
|
3204
|
+
/** Coalesce progress broadcasts: at most one per 3s, trailing edge kept. */
|
|
3205
|
+
scheduleProgressBroadcast() {
|
|
3206
|
+
const MIN_INTERVAL_MS = 3_000;
|
|
3207
|
+
const since = Date.now() - this.lastProgressBroadcastAt;
|
|
3208
|
+
if (since >= MIN_INTERVAL_MS) {
|
|
3209
|
+
this.broadcastToolProgress();
|
|
3210
|
+
return;
|
|
3211
|
+
}
|
|
3212
|
+
if (this.progressBroadcastTimer)
|
|
3213
|
+
return;
|
|
3214
|
+
this.progressBroadcastTimer = setTimeout(() => {
|
|
3215
|
+
this.progressBroadcastTimer = null;
|
|
3216
|
+
this.broadcastToolProgress();
|
|
3217
|
+
}, MIN_INTERVAL_MS - since);
|
|
3218
|
+
this.progressBroadcastTimer.unref?.();
|
|
3219
|
+
}
|
|
3220
|
+
broadcastToolProgress() {
|
|
3221
|
+
const rendered = this.turnProgress.render();
|
|
3222
|
+
if (rendered === this.lastProgressBroadcast)
|
|
3223
|
+
return;
|
|
3224
|
+
this.lastProgressBroadcast = rendered;
|
|
3225
|
+
this.lastProgressBroadcastAt = Date.now();
|
|
3226
|
+
this.ipcServer?.broadcast({
|
|
3227
|
+
type: "instance_progress",
|
|
3228
|
+
instanceName: this.name,
|
|
3229
|
+
progress: rendered,
|
|
3230
|
+
});
|
|
3231
|
+
}
|
|
3232
|
+
/** New turn (or turn over): drop the list and tell the fleet to clear it. */
|
|
3233
|
+
resetToolProgress() {
|
|
3234
|
+
if (this.progressBroadcastTimer) {
|
|
3235
|
+
clearTimeout(this.progressBroadcastTimer);
|
|
3236
|
+
this.progressBroadcastTimer = null;
|
|
3237
|
+
}
|
|
3238
|
+
this.turnProgress.reset();
|
|
3239
|
+
if (this.lastProgressBroadcast !== "") {
|
|
3240
|
+
this.lastProgressBroadcast = "";
|
|
3241
|
+
this.lastProgressBroadcastAt = Date.now();
|
|
3242
|
+
this.ipcServer?.broadcast({ type: "instance_progress", instanceName: this.name, progress: "" });
|
|
3243
|
+
}
|
|
3244
|
+
}
|
|
2401
3245
|
/**
|
|
2402
3246
|
* Options for every system-initiated paste (startup notice, session snapshot,
|
|
2403
3247
|
* runtime-dialog keys).
|
|
@@ -2433,12 +3277,121 @@ export class Daemon extends EventEmitter {
|
|
|
2433
3277
|
return ""; // skip channel tools
|
|
2434
3278
|
return name;
|
|
2435
3279
|
}
|
|
3280
|
+
/**
|
|
3281
|
+
* The one place an inbound message grows its metadata wrapper ([user:]/
|
|
3282
|
+
* [from:] prefix, pending reactions, handoff metadata, reply instructions).
|
|
3283
|
+
* Both the normal queued path (pushChannelMessage) and /steer go through
|
|
3284
|
+
* here — a steered message must read EXACTLY like a queued one to the
|
|
3285
|
+
* agent, or the two drift apart in what the agent is told about replying.
|
|
3286
|
+
*/
|
|
3287
|
+
formatInboundMessage(content, meta) {
|
|
3288
|
+
const user = meta.user || "unknown";
|
|
3289
|
+
const fromInstance = meta.from_instance;
|
|
3290
|
+
let formatted;
|
|
3291
|
+
if (fromInstance) {
|
|
3292
|
+
formatted = formatCrossInstanceInboundMessage(content, meta);
|
|
3293
|
+
}
|
|
3294
|
+
else {
|
|
3295
|
+
const via = meta.source ? ` via ${meta.source}` : "";
|
|
3296
|
+
const idTag = meta.user_id ? `, id:${meta.user_id}` : "";
|
|
3297
|
+
formatted = `[user:${user}${via}${idTag}] ${content}`;
|
|
3298
|
+
// Reactions queued since the last real message (#432). One leading line of
|
|
3299
|
+
// context, present only when something is pending — a reaction no longer
|
|
3300
|
+
// costs a turn of its own.
|
|
3301
|
+
if (meta.pending_reactions) {
|
|
3302
|
+
formatted = `[Recent reactions: ${meta.pending_reactions}]\n${formatted}`;
|
|
3303
|
+
}
|
|
3304
|
+
formatted += renderHandoffMetadata(meta);
|
|
3305
|
+
formatted += "\n(Reply using the reply tool — do NOT respond with direct text)";
|
|
3306
|
+
}
|
|
3307
|
+
if (meta.reply_to_text) {
|
|
3308
|
+
formatted += `\n(reply_to: "${meta.reply_to_text}")`;
|
|
3309
|
+
}
|
|
3310
|
+
return formatted;
|
|
3311
|
+
}
|
|
3312
|
+
/**
|
|
3313
|
+
* /steer: interject into the CURRENT turn instead of queueing for idle.
|
|
3314
|
+
*
|
|
3315
|
+
* Differences from pushChannelMessage, and nothing else:
|
|
3316
|
+
* - serialized on steerLock, not pasteLock — it must overtake, not queue
|
|
3317
|
+
* - deliverMessage runs with { steer: true }, which takes the busy branch
|
|
3318
|
+
* that pastes immediately (the codex native-queue transaction) instead of
|
|
3319
|
+
* waiting for idle. Verification and silent-loss fallback are the ones
|
|
3320
|
+
* that path already has: pane-capture visibility check, then one
|
|
3321
|
+
* idle-gated redelivery — so on a TUI that swallows busy input the steer
|
|
3322
|
+
* degrades to "next message after this turn", never silently vanishes.
|
|
3323
|
+
*
|
|
3324
|
+
* The steered text goes through formatInboundMessage so the agent sees a
|
|
3325
|
+
* normal inbound message, with a steering notice prepended for context.
|
|
3326
|
+
*/
|
|
3327
|
+
steerMessage(content, meta, deliveryEpoch = this.deliveryEpoch) {
|
|
3328
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
3329
|
+
return;
|
|
3330
|
+
this.updateLastChat(meta.chat_id, meta.thread_id, meta.adapter_id);
|
|
3331
|
+
this.pendingWork.recordInbound();
|
|
3332
|
+
this.recordRecentUserMessage(content, meta);
|
|
3333
|
+
const formatted = "[STEERING — mid-task course correction from the user. Fold this into the CURRENT work.]\n"
|
|
3334
|
+
+ this.formatInboundMessage(content, meta);
|
|
3335
|
+
const chatId = meta.chat_id;
|
|
3336
|
+
const messageId = meta.message_id;
|
|
3337
|
+
const status = (chatId && messageId)
|
|
3338
|
+
? { chatId: meta.thread_id || chatId, messageId }
|
|
3339
|
+
: undefined;
|
|
3340
|
+
this.steerLock = this.steerLock.then(async () => {
|
|
3341
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
3342
|
+
return;
|
|
3343
|
+
await this.wake();
|
|
3344
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
3345
|
+
return;
|
|
3346
|
+
if (await this.deliverMessage(formatted, status, { steer: true, deliveryEpoch })) {
|
|
3347
|
+
this.markTurnStarted(meta, formatted);
|
|
3348
|
+
}
|
|
3349
|
+
}).catch(err => {
|
|
3350
|
+
this.logger.warn({ err: err.message }, "steer delivery error");
|
|
3351
|
+
});
|
|
3352
|
+
}
|
|
3353
|
+
/**
|
|
3354
|
+
* /btw: submit a separately-labelled side question immediately while asking
|
|
3355
|
+
* the agent not to change the active task. Claude's native `/btw` opens a
|
|
3356
|
+
* branchAndResume panel whose answer never reaches the channel or transcript,
|
|
3357
|
+
* so channel delivery intentionally uses a normal wrapped inbound instead.
|
|
3358
|
+
*/
|
|
3359
|
+
btwMessage(content, meta, deliveryEpoch = this.deliveryEpoch) {
|
|
3360
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
3361
|
+
return;
|
|
3362
|
+
this.updateLastChat(meta.chat_id, meta.thread_id, meta.adapter_id);
|
|
3363
|
+
this.pendingWork.recordInbound();
|
|
3364
|
+
this.recordRecentUserMessage(content, meta);
|
|
3365
|
+
const formatted = "[BTW — side question from the user. Answer it separately without changing or interrupting the CURRENT work.]\n"
|
|
3366
|
+
+ this.formatInboundMessage(content, meta);
|
|
3367
|
+
const chatId = meta.chat_id;
|
|
3368
|
+
const messageId = meta.message_id;
|
|
3369
|
+
const status = (chatId && messageId)
|
|
3370
|
+
? { chatId: meta.thread_id || chatId, messageId }
|
|
3371
|
+
: undefined;
|
|
3372
|
+
this.steerLock = this.steerLock.then(async () => {
|
|
3373
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
3374
|
+
return;
|
|
3375
|
+
await this.wake();
|
|
3376
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
3377
|
+
return;
|
|
3378
|
+
if (await this.deliverMessage(formatted, status, { steer: true, deliveryEpoch })) {
|
|
3379
|
+
this.markTurnStarted(meta, formatted);
|
|
3380
|
+
}
|
|
3381
|
+
}).catch(err => {
|
|
3382
|
+
this.logger.warn({ err: err.message }, "btw delivery error");
|
|
3383
|
+
});
|
|
3384
|
+
}
|
|
2436
3385
|
/**
|
|
2437
3386
|
* Push an inbound channel message to a specific MCP session.
|
|
2438
3387
|
* If targetSession is provided, only send to the matching socket.
|
|
2439
3388
|
* Otherwise send to the instance's own session (this.name).
|
|
2440
3389
|
*/
|
|
2441
|
-
pushChannelMessage(content, meta, _targetSession) {
|
|
3390
|
+
pushChannelMessage(content, meta, _targetSession, deliveryEpoch = this.deliveryEpoch) {
|
|
3391
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch)) {
|
|
3392
|
+
this.logger.info("Pending channel delivery dropped by user cancel");
|
|
3393
|
+
return;
|
|
3394
|
+
}
|
|
2442
3395
|
if (!this.tmux) {
|
|
2443
3396
|
this.logger.warn("Cannot push channel message: tmux not running");
|
|
2444
3397
|
return;
|
|
@@ -2472,46 +3425,19 @@ export class Daemon extends EventEmitter {
|
|
|
2472
3425
|
const rawText = content.slice(5);
|
|
2473
3426
|
this.logger.info({ user }, "Raw paste from topic mode user");
|
|
2474
3427
|
this.pasteLock = this.pasteLock.then(async () => {
|
|
2475
|
-
if (
|
|
3428
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
3429
|
+
return;
|
|
3430
|
+
if (await this.deliverMessage(rawText, undefined, { deliveryEpoch }))
|
|
2476
3431
|
this.markTurnStarted(meta, rawText);
|
|
2477
3432
|
}).catch(err => {
|
|
2478
3433
|
this.logger.warn({ err: err.message }, "pasteLock raw delivery error");
|
|
2479
3434
|
});
|
|
2480
3435
|
return;
|
|
2481
3436
|
}
|
|
2482
|
-
|
|
2483
|
-
if (fromInstance) {
|
|
2484
|
-
// #77: show the sender's display name for readability, keeping the machine
|
|
2485
|
-
// instance name in parens so the recipient's send_to_instance target is valid.
|
|
2486
|
-
const fromLabel = meta.from_display ? `${meta.from_display} (${fromInstance})` : fromInstance;
|
|
2487
|
-
formatted = `[from:${fromLabel}] ${content}`;
|
|
2488
|
-
formatted += renderHandoffMetadata(meta);
|
|
2489
|
-
// A delegated task that requires a reply must not read like a chatty FYI —
|
|
2490
|
-
// the "you may stay silent" line is for the latter only.
|
|
2491
|
-
formatted += meta.requires_reply === "true"
|
|
2492
|
-
? "\n(A reply IS required: use report_result with the correlation_id above — or send_to_instance. Not direct text.)"
|
|
2493
|
-
: "\n(If you need to reply, use send_to_instance tool, NOT direct text. If there is nothing to add, you may stay silent.)";
|
|
2494
|
-
}
|
|
2495
|
-
else {
|
|
2496
|
-
const via = meta.source ? ` via ${meta.source}` : "";
|
|
2497
|
-
const idTag = meta.user_id ? `, id:${meta.user_id}` : "";
|
|
2498
|
-
formatted = `[user:${user}${via}${idTag}] ${content}`;
|
|
2499
|
-
// Reactions queued since the last real message (#432). One leading line of
|
|
2500
|
-
// context, present only when something is pending — a reaction no longer
|
|
2501
|
-
// costs a turn of its own.
|
|
2502
|
-
if (meta.pending_reactions) {
|
|
2503
|
-
formatted = `[Recent reactions: ${meta.pending_reactions}]\n${formatted}`;
|
|
2504
|
-
}
|
|
2505
|
-
formatted += renderHandoffMetadata(meta);
|
|
2506
|
-
formatted += "\n(Reply using the reply tool — do NOT respond with direct text)";
|
|
2507
|
-
}
|
|
2508
|
-
if (meta.reply_to_text) {
|
|
2509
|
-
formatted += `\n(reply_to: "${meta.reply_to_text}")`;
|
|
2510
|
-
}
|
|
3437
|
+
const formatted = this.formatInboundMessage(content, meta);
|
|
2511
3438
|
// Serialize deliveries: each message waits for the previous to complete,
|
|
2512
3439
|
// and each waits for the CLI to be idle before pasting. Messages are never
|
|
2513
|
-
// dropped for age —
|
|
2514
|
-
// (the user can press Cancel to interrupt and let the queue drain sooner).
|
|
3440
|
+
// dropped for age — only an explicit user Cancel invalidates queued work.
|
|
2515
3441
|
const chatId = meta.chat_id;
|
|
2516
3442
|
const messageId = meta.message_id;
|
|
2517
3443
|
const wasQueued = this.pasteQueueDepth > 0;
|
|
@@ -2524,18 +3450,40 @@ export class Daemon extends EventEmitter {
|
|
|
2524
3450
|
}
|
|
2525
3451
|
this.pasteLock = this.pasteLock.then(async () => {
|
|
2526
3452
|
try {
|
|
3453
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch)) {
|
|
3454
|
+
this.logger.info("Pending channel delivery dropped by user cancel");
|
|
3455
|
+
return;
|
|
3456
|
+
}
|
|
2527
3457
|
if (this.config.pre_task_command) {
|
|
2528
|
-
await this.deliverMessage(this.config.pre_task_command);
|
|
3458
|
+
await this.deliverMessage(this.config.pre_task_command, undefined, { deliveryEpoch });
|
|
3459
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
3460
|
+
return;
|
|
2529
3461
|
}
|
|
2530
3462
|
if (this.pendingInstructionsNotice) {
|
|
2531
3463
|
this.pendingInstructionsNotice = false;
|
|
2532
|
-
await this.deliverMessage(buildInstructionReloadNotice(this.backend?.binaryName ?? "unknown", this.name, this.instanceDir));
|
|
3464
|
+
await this.deliverMessage(buildInstructionReloadNotice(this.backend?.binaryName ?? "unknown", this.name, this.instanceDir), undefined, { deliveryEpoch });
|
|
3465
|
+
if (!this.isDeliveryEpochCurrent(deliveryEpoch))
|
|
3466
|
+
return;
|
|
2533
3467
|
}
|
|
2534
3468
|
const status = (chatId && messageId)
|
|
2535
3469
|
? { chatId: meta.thread_id || chatId, messageId }
|
|
2536
3470
|
: undefined;
|
|
2537
|
-
|
|
3471
|
+
// A fresh delivery begins a fresh turn — its bubble must not inherit
|
|
3472
|
+
// the previous turn's tool list.
|
|
3473
|
+
this.resetToolProgress();
|
|
3474
|
+
if (await this.deliverMessage(formatted, status, { deliveryEpoch })) {
|
|
2538
3475
|
this.markTurnStarted(meta, formatted);
|
|
3476
|
+
}
|
|
3477
|
+
else if (meta.from_instance && this.isDeliveryEpochCurrent(deliveryEpoch)) {
|
|
3478
|
+
const error = this.tmux?.getLastPasteError?.() ?? "target pane rejected the delivery";
|
|
3479
|
+
this.ipcServer?.broadcast({
|
|
3480
|
+
type: "cross_instance_delivery_failed",
|
|
3481
|
+
senderSession: meta.from_instance,
|
|
3482
|
+
targetInstance: this.name,
|
|
3483
|
+
correlationId: meta.correlation_id ?? "unknown",
|
|
3484
|
+
error,
|
|
3485
|
+
});
|
|
3486
|
+
}
|
|
2539
3487
|
}
|
|
2540
3488
|
finally {
|
|
2541
3489
|
this.pasteQueueDepth--;
|
|
@@ -2561,31 +3509,55 @@ export class Daemon extends EventEmitter {
|
|
|
2561
3509
|
* immediately and own the application-level queue themselves. The pasteLock remains
|
|
2562
3510
|
* serial in both cases so separate PTY writes can never overlap.
|
|
2563
3511
|
*/
|
|
2564
|
-
async deliverMessage(formatted, status) {
|
|
3512
|
+
async deliverMessage(formatted, status, opts) {
|
|
3513
|
+
const cancelled = () => opts?.deliveryEpoch !== undefined
|
|
3514
|
+
&& !this.isDeliveryEpochCurrent(opts.deliveryEpoch);
|
|
3515
|
+
if (cancelled() || this.stormWindow?.isStopped())
|
|
3516
|
+
return false;
|
|
2565
3517
|
// Sanitize unclosed code fences — they cause CLI to wait for closure on Enter
|
|
2566
3518
|
const fenceCount = (formatted.match(/```/g) || []).length;
|
|
2567
3519
|
if (fenceCount % 2 !== 0) {
|
|
2568
3520
|
// Odd number of fences = unclosed. Remove all code fences from the message.
|
|
2569
3521
|
formatted = formatted.replace(/```/g, "");
|
|
2570
3522
|
}
|
|
3523
|
+
// Direct paths such as /steer, /btw, /raw and schedules do not pass the
|
|
3524
|
+
// FleetManager idle gate. Hold them at the final common pane-write path so
|
|
3525
|
+
// a tmux storm cannot force any text into a half-booted CLI.
|
|
3526
|
+
if (this.stormWindow?.isDeliveryHeld(this.name)) {
|
|
3527
|
+
this.logger.warn("Pane delivery held during tmux server recovery");
|
|
3528
|
+
await this.stormWindow.waitForDeliveryAllowed(this.name);
|
|
3529
|
+
if (cancelled() || this.stormWindow.isStopped())
|
|
3530
|
+
return false;
|
|
3531
|
+
}
|
|
2571
3532
|
// Before anything reads the window id: a spawn in progress is about to change it.
|
|
2572
3533
|
await this.waitForSpawnToSettle();
|
|
3534
|
+
if (cancelled())
|
|
3535
|
+
return false;
|
|
3536
|
+
if (this.refuseFatalStartupDelivery(status))
|
|
3537
|
+
return false;
|
|
2573
3538
|
let windowId = this.getWindowId();
|
|
2574
3539
|
const supportsQueuedInput = this.backend?.supportsQueuedInput?.() === true;
|
|
2575
3540
|
let handingOffToNativeQueue = false;
|
|
2576
3541
|
// If the CLI is busy, either hand the complete submission to its native input
|
|
2577
3542
|
// queue or wait for idle. Native queue support is an explicit backend capability:
|
|
2578
3543
|
// normal Enter input has steering/interrupt semantics in several other CLIs.
|
|
2579
|
-
if (windowId && this.controlClient && !this.
|
|
3544
|
+
if (windowId && this.controlClient && !this.isPaneIdleForDelivery(windowId)) {
|
|
2580
3545
|
if (status)
|
|
2581
3546
|
this.emit("message_queued", status);
|
|
2582
|
-
if (supportsQueuedInput) {
|
|
3547
|
+
if (supportsQueuedInput || opts?.steer) {
|
|
3548
|
+
// Native queue (codex), or an explicit /steer: hand the complete
|
|
3549
|
+
// paste+Enter transaction to the busy CLI now. For steer this is the
|
|
3550
|
+
// point — the user asked to interject, and the transaction's
|
|
3551
|
+
// visibility check + idle-gated fallback catch TUIs that swallow
|
|
3552
|
+
// busy input (see steerMessage).
|
|
2583
3553
|
handingOffToNativeQueue = true;
|
|
2584
|
-
this.logger.debug("CLI busy — handing message to backend-native input queue");
|
|
3554
|
+
this.logger.debug(opts?.steer ? "CLI busy — steering into the running turn" : "CLI busy — handing message to backend-native input queue");
|
|
2585
3555
|
}
|
|
2586
3556
|
else {
|
|
2587
3557
|
this.logger.debug("CLI busy — queuing message until idle");
|
|
2588
|
-
const becameIdle = await this.
|
|
3558
|
+
const becameIdle = await this.waitForPaneIdleForDelivery(windowId);
|
|
3559
|
+
if (cancelled())
|
|
3560
|
+
return false;
|
|
2589
3561
|
if (!becameIdle) {
|
|
2590
3562
|
// The pane never freed up. Report the failure instead of pasting into a
|
|
2591
3563
|
// wedged CLI (where the text would sit unsubmitted and the next message
|
|
@@ -2601,7 +3573,74 @@ export class Daemon extends EventEmitter {
|
|
|
2601
3573
|
// held under the pane lock — holding it across the idle wait (up to 30 min)
|
|
2602
3574
|
// would starve the runtime-dialog dismisser, which is often the very thing
|
|
2603
3575
|
// that would let the pane go idle again.
|
|
2604
|
-
return this.paneWriteLock.run(() =>
|
|
3576
|
+
return this.paneWriteLock.run(async () => {
|
|
3577
|
+
if (cancelled())
|
|
3578
|
+
return false;
|
|
3579
|
+
// Re-checked here because the flag can be set during the waits above
|
|
3580
|
+
// (a spawn's dialog scan runs concurrently with a queued delivery).
|
|
3581
|
+
if (this.refuseFatalStartupDelivery(status))
|
|
3582
|
+
return false;
|
|
3583
|
+
return this.writeMessageToPane(formatted, windowId, handingOffToNativeQueue, status);
|
|
3584
|
+
});
|
|
3585
|
+
}
|
|
3586
|
+
/**
|
|
3587
|
+
* While the CLI is parked on a fatal startup screen (see fatalStartupBlocked),
|
|
3588
|
+
* no pane write may happen: paste+Enter into the corrupt-config modal would
|
|
3589
|
+
* confirm "Exit and fix manually" and exit the CLI. The message is failed (❌)
|
|
3590
|
+
* rather than held — the accompanying config_error notification tells the user
|
|
3591
|
+
* what to fix, and holding would only build a queue that floods the pane the
|
|
3592
|
+
* moment the flag clears.
|
|
3593
|
+
*/
|
|
3594
|
+
refuseFatalStartupDelivery(status) {
|
|
3595
|
+
if (!this.fatalStartupBlocked)
|
|
3596
|
+
return false;
|
|
3597
|
+
this.logger.error("Delivery refused — CLI is parked on a fatal startup screen");
|
|
3598
|
+
if (status)
|
|
3599
|
+
this.emit("message_failed", status); // ❌
|
|
3600
|
+
return true;
|
|
3601
|
+
}
|
|
3602
|
+
/**
|
|
3603
|
+
* A raw `%output` silence check is authoritative for ordinary TUIs. It is not
|
|
3604
|
+
* for a backend such as agy whose status-line hook periodically redraws an
|
|
3605
|
+
* identical screen. In that case trust an idle pane-state snapshot only when
|
|
3606
|
+
* it was captured after both the latest output and the latest control-client
|
|
3607
|
+
* observation reset.
|
|
3608
|
+
*/
|
|
3609
|
+
isPaneIdleForDelivery(windowId) {
|
|
3610
|
+
if (!this.controlClient)
|
|
3611
|
+
return true;
|
|
3612
|
+
if (this.controlClient.isIdle(windowId))
|
|
3613
|
+
return true;
|
|
3614
|
+
if (this.backend?.hasPeriodicPaneRedraw?.() !== true || this.instanceState !== "idle")
|
|
3615
|
+
return false;
|
|
3616
|
+
const lastOutputAt = this.controlClient.getLastOutputAt(windowId);
|
|
3617
|
+
if (lastOutputAt === undefined || !this.instanceStateMachine)
|
|
3618
|
+
return false;
|
|
3619
|
+
const observedAt = this.instanceStateMachine.snapshot().observedAt;
|
|
3620
|
+
return observedAt >= lastOutputAt
|
|
3621
|
+
&& observedAt >= this.controlClient.getObservationResetAt();
|
|
3622
|
+
}
|
|
3623
|
+
waitForPaneIdleForDelivery(windowId, timeoutMs = 30 * 60_000) {
|
|
3624
|
+
if (!this.controlClient)
|
|
3625
|
+
return Promise.resolve(true);
|
|
3626
|
+
if (this.backend?.hasPeriodicPaneRedraw?.() !== true) {
|
|
3627
|
+
return this.controlClient.waitUntilIdle(windowId, timeoutMs);
|
|
3628
|
+
}
|
|
3629
|
+
if (this.isPaneIdleForDelivery(windowId))
|
|
3630
|
+
return Promise.resolve(true);
|
|
3631
|
+
return new Promise(resolve => {
|
|
3632
|
+
const check = setInterval(() => {
|
|
3633
|
+
if (!this.isPaneIdleForDelivery(windowId))
|
|
3634
|
+
return;
|
|
3635
|
+
clearInterval(check);
|
|
3636
|
+
clearTimeout(timer);
|
|
3637
|
+
resolve(true);
|
|
3638
|
+
}, 50);
|
|
3639
|
+
const timer = setTimeout(() => {
|
|
3640
|
+
clearInterval(check);
|
|
3641
|
+
resolve(false);
|
|
3642
|
+
}, timeoutMs);
|
|
3643
|
+
});
|
|
2605
3644
|
}
|
|
2606
3645
|
/**
|
|
2607
3646
|
* The write half of a delivery: paste → Enter → confirm, with retries.
|
|
@@ -2629,7 +3668,16 @@ export class Daemon extends EventEmitter {
|
|
|
2629
3668
|
const pasteStartedAt = Date.now();
|
|
2630
3669
|
const pasted = await this.tmux.pasteBuffer(formatted);
|
|
2631
3670
|
if (!pasted) {
|
|
2632
|
-
this.
|
|
3671
|
+
const tmuxError = this.tmux.getLastPasteError?.() ?? "unknown tmux paste failure";
|
|
3672
|
+
const recoverable = this.tmux.isLastPasteFailureRecoverable?.() ?? true;
|
|
3673
|
+
this.logger[recoverable ? "warn" : "error"]({ attempt, tmuxError, recoverable }, recoverable
|
|
3674
|
+
? "pasteBuffer failed — recovering window and backing off"
|
|
3675
|
+
: "pasteBuffer failed — non-retryable tmux error");
|
|
3676
|
+
if (!recoverable) {
|
|
3677
|
+
if (status)
|
|
3678
|
+
this.emit("message_failed", status);
|
|
3679
|
+
return false;
|
|
3680
|
+
}
|
|
2633
3681
|
windowId = (await this.recoverWindow()) ?? windowId;
|
|
2634
3682
|
if (attempt < maxAttempts)
|
|
2635
3683
|
await new Promise(r => setTimeout(r, 2000));
|
|
@@ -2711,7 +3759,9 @@ export class Daemon extends EventEmitter {
|
|
|
2711
3759
|
}
|
|
2712
3760
|
const repasted = await this.tmux.pasteBuffer(formatted);
|
|
2713
3761
|
if (!repasted) {
|
|
2714
|
-
this.logger.error(
|
|
3762
|
+
this.logger.error({
|
|
3763
|
+
tmuxError: this.tmux.getLastPasteError?.() ?? "unknown tmux paste failure",
|
|
3764
|
+
}, "Idle-gated redelivery paste failed after native-queue silent loss");
|
|
2715
3765
|
if (status)
|
|
2716
3766
|
this.emit("message_failed", status); // ❌
|
|
2717
3767
|
return false;
|
|
@@ -2911,8 +3961,13 @@ export class Daemon extends EventEmitter {
|
|
|
2911
3961
|
// for this turn: the agent proved it can still speak for itself.
|
|
2912
3962
|
if (!error && result != null && TURN_REPLY_TOOLS.has(tool)) {
|
|
2913
3963
|
this.turnOutboundDelivered = true;
|
|
3964
|
+
if (tool === "reply")
|
|
3965
|
+
this.turnReplyDelivered = true;
|
|
3966
|
+
}
|
|
3967
|
+
const sent = this.ipcServer?.send(socket, { requestId, result, error }) ?? false;
|
|
3968
|
+
if (!sent) {
|
|
3969
|
+
this.logger.warn({ tool, requestId }, "Tool result could not be returned — originating MCP socket is closed");
|
|
2914
3970
|
}
|
|
2915
|
-
this.ipcServer?.send(socket, { requestId, result, error });
|
|
2916
3971
|
};
|
|
2917
3972
|
// Repo checkout — handled locally in daemon (no fleet-manager)
|
|
2918
3973
|
if (tool === "checkout_repo") {
|
|
@@ -2925,13 +3980,7 @@ export class Daemon extends EventEmitter {
|
|
|
2925
3980
|
}
|
|
2926
3981
|
if (tool === "set_display_name" || tool === "set_description") {
|
|
2927
3982
|
const type = tool === "set_display_name" ? "fleet_set_display_name" : "fleet_set_description";
|
|
2928
|
-
const fleetReqId = `${tool === "set_display_name" ? "dn" : "desc"}_${requestId}`;
|
|
2929
|
-
this.ipcServer?.broadcast({
|
|
2930
|
-
type,
|
|
2931
|
-
payload: args,
|
|
2932
|
-
meta: { instance_name: this.name },
|
|
2933
|
-
fleetRequestId: fleetReqId,
|
|
2934
|
-
});
|
|
3983
|
+
const fleetReqId = `${tool === "set_display_name" ? "dn" : "desc"}_${++this.fleetRequestSeq}_${requestId}`;
|
|
2935
3984
|
const timeout = setTimeout(() => {
|
|
2936
3985
|
this.pendingIpcRequests.delete(fleetReqId);
|
|
2937
3986
|
respond(null, `${tool} timed out`);
|
|
@@ -2940,16 +3989,16 @@ export class Daemon extends EventEmitter {
|
|
|
2940
3989
|
clearTimeout(timeout);
|
|
2941
3990
|
respond(respMsg.result, respMsg.error);
|
|
2942
3991
|
});
|
|
2943
|
-
return;
|
|
2944
|
-
}
|
|
2945
|
-
if (tool === TASK_TOOL) {
|
|
2946
|
-
const fleetReqId = `task_${requestId}`;
|
|
2947
3992
|
this.ipcServer?.broadcast({
|
|
2948
|
-
type
|
|
3993
|
+
type,
|
|
2949
3994
|
payload: args,
|
|
2950
3995
|
meta: { instance_name: this.name },
|
|
2951
3996
|
fleetRequestId: fleetReqId,
|
|
2952
3997
|
});
|
|
3998
|
+
return;
|
|
3999
|
+
}
|
|
4000
|
+
if (tool === TASK_TOOL) {
|
|
4001
|
+
const fleetReqId = `task_${++this.fleetRequestSeq}_${requestId}`;
|
|
2953
4002
|
const timeout = setTimeout(() => {
|
|
2954
4003
|
this.pendingIpcRequests.delete(fleetReqId);
|
|
2955
4004
|
respond(null, `Task operation timed out after ${daemonBudgetMs(tool) / 1000}s`);
|
|
@@ -2958,6 +4007,12 @@ export class Daemon extends EventEmitter {
|
|
|
2958
4007
|
clearTimeout(timeout);
|
|
2959
4008
|
respond(respMsg.result, respMsg.error);
|
|
2960
4009
|
});
|
|
4010
|
+
this.ipcServer?.broadcast({
|
|
4011
|
+
type: "fleet_task",
|
|
4012
|
+
payload: args,
|
|
4013
|
+
meta: { instance_name: this.name },
|
|
4014
|
+
fleetRequestId: fleetReqId,
|
|
4015
|
+
});
|
|
2961
4016
|
return;
|
|
2962
4017
|
}
|
|
2963
4018
|
if (DECISION_TOOLS.has(tool)) {
|
|
@@ -2966,13 +4021,7 @@ export class Daemon extends EventEmitter {
|
|
|
2966
4021
|
list_decisions: "fleet_decision_list",
|
|
2967
4022
|
update_decision: "fleet_decision_update",
|
|
2968
4023
|
};
|
|
2969
|
-
const fleetReqId = `dec_${requestId}`;
|
|
2970
|
-
this.ipcServer?.broadcast({
|
|
2971
|
-
type: typeMap[tool],
|
|
2972
|
-
payload: args,
|
|
2973
|
-
meta: { instance_name: this.name, working_directory: this.config.working_directory },
|
|
2974
|
-
fleetRequestId: fleetReqId,
|
|
2975
|
-
});
|
|
4024
|
+
const fleetReqId = `dec_${++this.fleetRequestSeq}_${requestId}`;
|
|
2976
4025
|
const timeout = setTimeout(() => {
|
|
2977
4026
|
this.pendingIpcRequests.delete(fleetReqId);
|
|
2978
4027
|
respond(null, `Decision operation timed out after ${daemonBudgetMs(tool) / 1000}s`);
|
|
@@ -2981,6 +4030,12 @@ export class Daemon extends EventEmitter {
|
|
|
2981
4030
|
clearTimeout(timeout);
|
|
2982
4031
|
respond(respMsg.result, respMsg.error);
|
|
2983
4032
|
});
|
|
4033
|
+
this.ipcServer?.broadcast({
|
|
4034
|
+
type: typeMap[tool],
|
|
4035
|
+
payload: args,
|
|
4036
|
+
meta: { instance_name: this.name, working_directory: this.config.working_directory },
|
|
4037
|
+
fleetRequestId: fleetReqId,
|
|
4038
|
+
});
|
|
2984
4039
|
return;
|
|
2985
4040
|
}
|
|
2986
4041
|
if (SCHEDULE_TOOLS.has(tool)) {
|
|
@@ -2992,13 +4047,7 @@ export class Daemon extends EventEmitter {
|
|
|
2992
4047
|
};
|
|
2993
4048
|
// Use fleetRequestId (not requestId) to avoid MCP server resolving the
|
|
2994
4049
|
// pending tool call prematurely when it receives the broadcast.
|
|
2995
|
-
const fleetReqId = `sched_${requestId}`;
|
|
2996
|
-
this.ipcServer?.broadcast({
|
|
2997
|
-
type: typeMap[tool],
|
|
2998
|
-
payload: args,
|
|
2999
|
-
meta: { chat_id: this.lastChatId, thread_id: this.lastThreadId, instance_name: this.name },
|
|
3000
|
-
fleetRequestId: fleetReqId,
|
|
3001
|
-
});
|
|
4050
|
+
const fleetReqId = `sched_${++this.fleetRequestSeq}_${requestId}`;
|
|
3002
4051
|
// Wait for fleet_schedule_response via pending request map
|
|
3003
4052
|
const timeout = setTimeout(() => {
|
|
3004
4053
|
this.pendingIpcRequests.delete(fleetReqId);
|
|
@@ -3008,6 +4057,12 @@ export class Daemon extends EventEmitter {
|
|
|
3008
4057
|
clearTimeout(timeout);
|
|
3009
4058
|
respond(respMsg.result, respMsg.error);
|
|
3010
4059
|
});
|
|
4060
|
+
this.ipcServer?.broadcast({
|
|
4061
|
+
type: typeMap[tool],
|
|
4062
|
+
payload: args,
|
|
4063
|
+
meta: { chat_id: this.lastChatId, thread_id: this.lastThreadId, instance_name: this.name },
|
|
4064
|
+
fleetRequestId: fleetReqId,
|
|
4065
|
+
});
|
|
3011
4066
|
return;
|
|
3012
4067
|
}
|
|
3013
4068
|
if (CROSS_INSTANCE_TOOLS.has(tool)) {
|
|
@@ -3015,15 +4070,8 @@ export class Daemon extends EventEmitter {
|
|
|
3015
4070
|
if (this.topicMode && this.ipcServer) {
|
|
3016
4071
|
// Use fleetRequestId (not requestId) to avoid MCP server resolving the
|
|
3017
4072
|
// pending tool call prematurely when it receives the broadcast.
|
|
3018
|
-
const fleetReqId = `xmsg_${requestId}`;
|
|
4073
|
+
const fleetReqId = `xmsg_${++this.fleetRequestSeq}_${requestId}`;
|
|
3019
4074
|
const senderSessionName = this.socketSessionNames.get(socket);
|
|
3020
|
-
this.ipcServer.broadcast({
|
|
3021
|
-
type: "fleet_outbound",
|
|
3022
|
-
tool,
|
|
3023
|
-
args,
|
|
3024
|
-
fleetRequestId: fleetReqId,
|
|
3025
|
-
senderSessionName,
|
|
3026
|
-
});
|
|
3027
4075
|
// Shared table, so delete_instance and the deployment tools get the same
|
|
3028
4076
|
// budget here as the MCP client expects (the ad-hoc list omitted them).
|
|
3029
4077
|
const crossTimeoutMs = daemonBudgetMs(tool);
|
|
@@ -3035,6 +4083,13 @@ export class Daemon extends EventEmitter {
|
|
|
3035
4083
|
clearTimeout(timeout);
|
|
3036
4084
|
respond(respMsg.result, respMsg.error);
|
|
3037
4085
|
});
|
|
4086
|
+
this.ipcServer.broadcast({
|
|
4087
|
+
type: "fleet_outbound",
|
|
4088
|
+
tool,
|
|
4089
|
+
args,
|
|
4090
|
+
fleetRequestId: fleetReqId,
|
|
4091
|
+
senderSessionName,
|
|
4092
|
+
});
|
|
3038
4093
|
}
|
|
3039
4094
|
else {
|
|
3040
4095
|
respond(null, "Cross-instance messaging requires topic mode");
|
|
@@ -3068,8 +4123,24 @@ export class Daemon extends EventEmitter {
|
|
|
3068
4123
|
// The fleet manager's IPC client receives this and routes to shared adapter.
|
|
3069
4124
|
// Use fleetRequestId (not requestId) to avoid other MCP sessions on this daemon
|
|
3070
4125
|
// from prematurely resolving their pending requests when they receive the broadcast.
|
|
3071
|
-
|
|
4126
|
+
// requestId is local to one MCP server process. Claude may keep multiple
|
|
4127
|
+
// MCP children attached (main session, resumed session, subagent); those
|
|
4128
|
+
// processes all emit requestId=1,2,... and previously overwrote this
|
|
4129
|
+
// shared pending map. The wrong caller could then receive another POST's
|
|
4130
|
+
// success + messageId while its own reply had never been confirmed.
|
|
4131
|
+
const fleetReqId = `tool_${++this.fleetRequestSeq}_${requestId}`;
|
|
3072
4132
|
const outboundKey = fleetReqId;
|
|
4133
|
+
const timeout = setTimeout(() => {
|
|
4134
|
+
this.pendingIpcRequests.delete(outboundKey);
|
|
4135
|
+
respond(null, `Fleet outbound timed out after ${daemonBudgetMs(tool) / 1000}s`);
|
|
4136
|
+
}, daemonBudgetMs(tool));
|
|
4137
|
+
this.pendingIpcRequests.set(outboundKey, (respMsg) => {
|
|
4138
|
+
clearTimeout(timeout);
|
|
4139
|
+
respond(respMsg.result, respMsg.error);
|
|
4140
|
+
});
|
|
4141
|
+
// Register the waiter before broadcasting. A fast/mock adapter can settle
|
|
4142
|
+
// immediately; broadcasting first left a response-before-registration
|
|
4143
|
+
// window that degraded into a timeout despite a completed POST.
|
|
3073
4144
|
this.ipcServer?.broadcast({
|
|
3074
4145
|
type: "fleet_outbound",
|
|
3075
4146
|
tool,
|
|
@@ -3080,14 +4151,6 @@ export class Daemon extends EventEmitter {
|
|
|
3080
4151
|
// from a secondary world, and message ids are not portable across bots.
|
|
3081
4152
|
adapterId: this.lastAdapterId,
|
|
3082
4153
|
});
|
|
3083
|
-
const timeout = setTimeout(() => {
|
|
3084
|
-
this.pendingIpcRequests.delete(outboundKey);
|
|
3085
|
-
respond(null, `Fleet outbound timed out after ${daemonBudgetMs(tool) / 1000}s`);
|
|
3086
|
-
}, daemonBudgetMs(tool));
|
|
3087
|
-
this.pendingIpcRequests.set(outboundKey, (respMsg) => {
|
|
3088
|
-
clearTimeout(timeout);
|
|
3089
|
-
respond(respMsg.result, respMsg.error);
|
|
3090
|
-
});
|
|
3091
4154
|
return;
|
|
3092
4155
|
}
|
|
3093
4156
|
const adapter = adapters[0];
|
|
@@ -3097,7 +4160,10 @@ export class Daemon extends EventEmitter {
|
|
|
3097
4160
|
}
|
|
3098
4161
|
/** Build config object for the CLI backend */
|
|
3099
4162
|
buildBackendConfig() {
|
|
3100
|
-
|
|
4163
|
+
// Antigravity 1.1.17 has no per-workspace MCP config, but its global MCP
|
|
4164
|
+
// children inherit the CLI process environment. The backend installs one
|
|
4165
|
+
// environment-selecting launcher, so MCP is now the safe default here too.
|
|
4166
|
+
const isCliMode = this.config.agent_mode === "cli";
|
|
3101
4167
|
const sockPath = join(this.instanceDir, "channel.sock");
|
|
3102
4168
|
let serverJs = join(__dirname, "channel", "mcp-server.js");
|
|
3103
4169
|
if (!existsSync(serverJs)) {
|
|
@@ -3234,6 +4300,8 @@ export class Daemon extends EventEmitter {
|
|
|
3234
4300
|
});
|
|
3235
4301
|
}
|
|
3236
4302
|
const agentPort = parseInt(process.env.AGEND_PORT ?? "19280", 10);
|
|
4303
|
+
const backendName = this.config.backend ?? "claude-code";
|
|
4304
|
+
const backendOptions = this.config.backend_options?.[backendName];
|
|
3237
4305
|
return {
|
|
3238
4306
|
workingDirectory: this.config.working_directory,
|
|
3239
4307
|
instanceDir: this.instanceDir,
|
|
@@ -3253,6 +4321,7 @@ export class Daemon extends EventEmitter {
|
|
|
3253
4321
|
instructions,
|
|
3254
4322
|
agentMode: isCliMode ? "cli" : "mcp",
|
|
3255
4323
|
agentPort: isCliMode ? agentPort : undefined,
|
|
4324
|
+
backendOptions,
|
|
3256
4325
|
};
|
|
3257
4326
|
}
|
|
3258
4327
|
/**
|
|
@@ -3405,6 +4474,15 @@ export class Daemon extends EventEmitter {
|
|
|
3405
4474
|
* Returns true if CLI is ready, false if it failed or got stuck.
|
|
3406
4475
|
*/
|
|
3407
4476
|
async trySpawn(reuseWindow = false, startupTimeoutMs) {
|
|
4477
|
+
if (!this.spawnGate)
|
|
4478
|
+
return this.trySpawnInsideGate(reuseWindow, startupTimeoutMs);
|
|
4479
|
+
return this.spawnGate.run({
|
|
4480
|
+
instanceName: this.name,
|
|
4481
|
+
workingDirectory: this.config.working_directory,
|
|
4482
|
+
reason: reuseWindow ? "wake" : this.lastSpawnAt > 0 ? "recovery" : "startup",
|
|
4483
|
+
}, () => this.trySpawnInsideGate(reuseWindow, startupTimeoutMs));
|
|
4484
|
+
}
|
|
4485
|
+
async trySpawnInsideGate(reuseWindow = false, startupTimeoutMs) {
|
|
3408
4486
|
const backendConfig = this.buildBackendConfig();
|
|
3409
4487
|
// Compare freshly-built instructions against the last value the agent was
|
|
3410
4488
|
// told about. Computed for ALL backends (not gated by
|
|
@@ -3469,6 +4547,9 @@ export class Daemon extends EventEmitter {
|
|
|
3469
4547
|
const cmd = `${envPrefix} ` + this.backend.buildCommand(backendConfig);
|
|
3470
4548
|
// Ensure tmux session exists (may have been destroyed if all windows died)
|
|
3471
4549
|
await TmuxManager.ensureSession(this.tmuxSessionName);
|
|
4550
|
+
if (this.stormWindow?.observeServerAlive(await TmuxManager.getServerPid(this.tmuxSessionName))) {
|
|
4551
|
+
this.emit("tmux_server_crash", this.name);
|
|
4552
|
+
}
|
|
3472
4553
|
let windowId;
|
|
3473
4554
|
if (reuseWindow) {
|
|
3474
4555
|
this.controlClient?.unregisterWindow(this.tmux.getWindowId());
|
|
@@ -3523,7 +4604,10 @@ export class Daemon extends EventEmitter {
|
|
|
3523
4604
|
if (!await this.tmux.isWindowAlive())
|
|
3524
4605
|
return false;
|
|
3525
4606
|
const ready = await this.dismissDialogsUntilReady(3);
|
|
3526
|
-
|
|
4607
|
+
// A fatal startup screen returns true (the spawn is over — respawning
|
|
4608
|
+
// would only re-show the same screen) but is NOT ready-for-delivery:
|
|
4609
|
+
// fatalStartupBlocked gates every pane write in deliverMessage.
|
|
4610
|
+
if (ready && !this.fatalStartupBlocked)
|
|
3527
4611
|
this.firstDeliveryDelay.recordReady();
|
|
3528
4612
|
return ready;
|
|
3529
4613
|
}
|
|
@@ -3546,6 +4630,26 @@ export class Daemon extends EventEmitter {
|
|
|
3546
4630
|
let matched = false;
|
|
3547
4631
|
for (const dialog of startupDialogs) {
|
|
3548
4632
|
if (dialog.pattern.test(pane)) {
|
|
4633
|
+
// A fatal startup screen cannot be dismissed by keypresses (e.g. a
|
|
4634
|
+
// corrupt claude.json modal offering only "exit" / "reset config").
|
|
4635
|
+
// It must be caught HERE and not fall through to the ready check:
|
|
4636
|
+
// the modal contains the `❯` selector the ready pattern matches, so
|
|
4637
|
+
// it would otherwise pass as ready and queued messages would be
|
|
4638
|
+
// typed into the modal. Report the error (pause stops delivery) and
|
|
4639
|
+
// stop the retry loop — like the sign-in screen case below, only a
|
|
4640
|
+
// human fixing the config ends this state.
|
|
4641
|
+
if (dialog.fatal) {
|
|
4642
|
+
// Block deliveries BEFORE the emit: the pty_error pause handler
|
|
4643
|
+
// is async, and a queued message would otherwise win the race
|
|
4644
|
+
// and be pasted into the modal.
|
|
4645
|
+
this.fatalStartupBlocked = true;
|
|
4646
|
+
if (!this.fatalStartupReported) {
|
|
4647
|
+
this.fatalStartupReported = true;
|
|
4648
|
+
this.logger.warn({ description: dialog.description }, "Fatal startup screen detected — reporting instead of dismissing");
|
|
4649
|
+
this.emit("pty_error", { name: this.name, ...dialog.fatal });
|
|
4650
|
+
}
|
|
4651
|
+
return true;
|
|
4652
|
+
}
|
|
3549
4653
|
this.logger.debug(`Dismissing startup dialog: ${dialog.description}`);
|
|
3550
4654
|
// Restart is exactly when inbound messages pile up, and nothing gates
|
|
3551
4655
|
// delivery on `spawning`. Take the pane lock for the key sequence so a
|
|
@@ -3578,8 +4682,31 @@ export class Daemon extends EventEmitter {
|
|
|
3578
4682
|
if (matched)
|
|
3579
4683
|
continue;
|
|
3580
4684
|
// CLI is ready (pattern defined by each backend)
|
|
3581
|
-
if (this.backend.getReadyPattern().test(pane))
|
|
4685
|
+
if (this.backend.getReadyPattern().test(pane)) {
|
|
4686
|
+
// A real ready prompt (with no fatal dialog on screen — those are
|
|
4687
|
+
// matched above, before this check) means the fatal screen is gone,
|
|
4688
|
+
// e.g. the user fixed the config and the instance respawned.
|
|
4689
|
+
this.fatalStartupBlocked = false;
|
|
3582
4690
|
return true;
|
|
4691
|
+
}
|
|
4692
|
+
// A CLI parked at its own sign-in screen is an AUTH incident: no dialog
|
|
4693
|
+
// key can dismiss it and no ready pattern will ever appear. Report it as
|
|
4694
|
+
// an auth error (the lifecycle double-checks with the token-free probe)
|
|
4695
|
+
// instead of letting it decay into crash/hang/MCP-died noise, and stop
|
|
4696
|
+
// the retry loop — only re-login (see /login) ends this state.
|
|
4697
|
+
const loginScreen = LOGIN_FLOWS[this.config.backend ?? "claude-code"]?.loginScreenPattern;
|
|
4698
|
+
if (loginScreen?.test(pane) && !this.loginScreenReported) {
|
|
4699
|
+
this.loginScreenReported = true;
|
|
4700
|
+
this.authFailureUnresolved = true;
|
|
4701
|
+
this.logger.warn({ backend: this.config.backend }, "CLI is waiting at its sign-in screen — reporting auth error");
|
|
4702
|
+
this.emit("pty_error", {
|
|
4703
|
+
name: this.name,
|
|
4704
|
+
type: "auth_error",
|
|
4705
|
+
action: "pause",
|
|
4706
|
+
message: "CLI is waiting at its sign-in screen — credentials are missing or expired",
|
|
4707
|
+
});
|
|
4708
|
+
return true;
|
|
4709
|
+
}
|
|
3583
4710
|
// Fatal: command not found (must match full phrase to avoid false positives
|
|
3584
4711
|
// like Kiro's "agent X not found, using default")
|
|
3585
4712
|
if (/command not found|: not found$/m.test(pane))
|
|
@@ -3616,9 +4743,29 @@ export class Daemon extends EventEmitter {
|
|
|
3616
4743
|
// back from statusline.json — that would re-arm --continue and re-loop.
|
|
3617
4744
|
if (this.skipResume)
|
|
3618
4745
|
return;
|
|
3619
|
-
|
|
3620
|
-
|
|
3621
|
-
|
|
4746
|
+
try {
|
|
4747
|
+
const sid = this.backend?.getSessionId();
|
|
4748
|
+
if (!sid)
|
|
4749
|
+
return;
|
|
4750
|
+
const path = join(this.instanceDir, "session-id");
|
|
4751
|
+
// Idle observations happen after turns. Avoid needless writes (and mtime
|
|
4752
|
+
// churn) while still updating the marker when /new changes the session.
|
|
4753
|
+
try {
|
|
4754
|
+
if (readFileSync(path, "utf-8").trim() === sid)
|
|
4755
|
+
return;
|
|
4756
|
+
}
|
|
4757
|
+
catch { /* first checkpoint */ }
|
|
4758
|
+
writeFileSync(path, sid);
|
|
4759
|
+
this.sessionCheckpointWarningEmitted = false;
|
|
4760
|
+
this.logger.debug("Session id checkpointed");
|
|
4761
|
+
}
|
|
4762
|
+
catch (err) {
|
|
4763
|
+
// Session discovery is best-effort and must never break the pane state
|
|
4764
|
+
// monitor. Existing stop/pause checkpoints get another chance later.
|
|
4765
|
+
if (!this.sessionCheckpointWarningEmitted) {
|
|
4766
|
+
this.sessionCheckpointWarningEmitted = true;
|
|
4767
|
+
this.logger.warn({ err: err.message }, "Session id checkpoint failed");
|
|
4768
|
+
}
|
|
3622
4769
|
}
|
|
3623
4770
|
}
|
|
3624
4771
|
readContextPercentage() {
|