@songsid/agend 2.1.6-beta.8 → 2.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backend/codex.d.ts +52 -0
- package/dist/backend/codex.js +618 -35
- package/dist/backend/codex.js.map +1 -1
- package/dist/backend/kiro.d.ts +9 -0
- package/dist/backend/kiro.js +129 -0
- package/dist/backend/kiro.js.map +1 -1
- package/dist/backend/muse.d.ts +37 -4
- package/dist/backend/muse.js +150 -61
- package/dist/backend/muse.js.map +1 -1
- package/dist/backend/types.d.ts +25 -1
- package/dist/backend/types.js.map +1 -1
- package/dist/daemon.d.ts +98 -10
- package/dist/daemon.js +836 -94
- package/dist/daemon.js.map +1 -1
- package/dist/fleet-manager.d.ts +68 -2
- package/dist/fleet-manager.js +290 -81
- package/dist/fleet-manager.js.map +1 -1
- package/dist/instance-lifecycle.d.ts +17 -0
- package/dist/instance-lifecycle.js +118 -5
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/locale.js +22 -0
- package/dist/locale.js.map +1 -1
- package/dist/muse-usage-relay.d.ts +72 -0
- package/dist/muse-usage-relay.js +431 -0
- package/dist/muse-usage-relay.js.map +1 -0
- package/dist/tmux-manager.d.ts +2 -0
- package/dist/tmux-manager.js +20 -0
- package/dist/tmux-manager.js.map +1 -1
- package/dist/tool-permissions.d.ts +27 -0
- package/dist/tool-permissions.js +64 -3
- package/dist/tool-permissions.js.map +1 -1
- package/dist/topic-commands.d.ts +9 -0
- package/dist/topic-commands.js +36 -2
- package/dist/topic-commands.js.map +1 -1
- package/dist/ui/view.html +2 -2
- package/dist/usage/i18n-keys.d.ts +1 -1
- package/dist/usage/i18n-keys.js +1 -1
- package/dist/usage/i18n-keys.js.map +1 -1
- package/dist/usage/providers.d.ts +4 -0
- package/dist/usage/providers.js +86 -9
- package/dist/usage/providers.js.map +1 -1
- package/dist/usage/usage-api.d.ts +0 -11
- package/dist/usage/usage-api.js +50 -8
- package/dist/usage/usage-api.js.map +1 -1
- package/package.json +2 -1
package/dist/daemon.js
CHANGED
|
@@ -29,6 +29,7 @@ import { formatCrossInstanceInboundMessage, renderCrossInstanceHandoffMetadata,
|
|
|
29
29
|
import { bottomRowIsReady, inputAreaText, inputShowsPastedText, pastedTextSignature, pasteLeftInInput, strandedAgendMessageInInput } from "./pane-input-residue.js";
|
|
30
30
|
import { TurnReplyGuard } from "./turn-reply-guard.js";
|
|
31
31
|
import { t } from "./locale.js";
|
|
32
|
+
import { MuseUsageRelay, clearMuseUsageSnapshot } from "./muse-usage-relay.js";
|
|
32
33
|
const __filename = fileURLToPath(import.meta.url);
|
|
33
34
|
const __dirname = dirname(__filename);
|
|
34
35
|
// Tool routing sets — module-level to avoid re-creation on every handleToolCall
|
|
@@ -398,6 +399,8 @@ export class PendingWorkTracker {
|
|
|
398
399
|
const NORMAL_ENTER_SETTLE_MS = 500;
|
|
399
400
|
/** How long to keep looking for proof after a retry Enter before failing it. */
|
|
400
401
|
const POST_ENTER_PROOF_WINDOW_MS = 3_000;
|
|
402
|
+
/** A final bounded observation before an ambiguous Codex submit can be classified. */
|
|
403
|
+
const CODEX_LATE_PROOF_MS = 5_000;
|
|
401
404
|
const POST_ENTER_PROOF_POLL_MS = 250;
|
|
402
405
|
/** Attempts to read the pre-paste pane before a delivery gives up on a baseline. */
|
|
403
406
|
const BASELINE_CAPTURE_ATTEMPTS = 3;
|
|
@@ -422,6 +425,37 @@ const STARTUP_DIALOG_BUDGET_MS = 30_000;
|
|
|
422
425
|
const DIALOG_PARKED_NOTIFY_MS = 60_000;
|
|
423
426
|
/** How many "dialog painted just before the write → wait → retry" rounds a delivery tolerates. */
|
|
424
427
|
const LATE_DIALOG_WRITE_ROUNDS = 3;
|
|
428
|
+
/** How many times a delivery redoes itself when a spawn starts between its settle wait and its pane write. */
|
|
429
|
+
const DELIVERY_SPAWN_RACE_MAX_ROUNDS = 3;
|
|
430
|
+
/**
|
|
431
|
+
* After relay exhaustion, how long to wait for an authoritatively idle pane
|
|
432
|
+
* before giving up the immediate resume-direct (the fallback flag and the
|
|
433
|
+
* notification already cover the next natural restart). Muse turns can run
|
|
434
|
+
* for many minutes; polling keeps each check a fresh capture.
|
|
435
|
+
*/
|
|
436
|
+
const MUSE_DIRECT_RESUME_IDLE_BUDGET_MS = 10 * 60_000;
|
|
437
|
+
const MUSE_DIRECT_RESUME_IDLE_POLL_MS = 5_000;
|
|
438
|
+
/** Kiro can drop /quit while a turn is running; cancel it and wait for its
|
|
439
|
+
* verified prompt before asking the native process to exit. Each instance is
|
|
440
|
+
* bounded so a fleet update can stop all daemons in parallel. */
|
|
441
|
+
const KIRO_STOP_IDLE_BUDGET_MS = 15_000;
|
|
442
|
+
const KIRO_STOP_QUIT_GRACE_MS = 5_000;
|
|
443
|
+
const KIRO_STOP_SIGTERM_GRACE_MS = 2_000;
|
|
444
|
+
/**
|
|
445
|
+
* How recently AgEnD must have asked the CLI to stop for a pane death to be
|
|
446
|
+
* attributed to AgEnD rather than to the CLI itself (#927). Covers the longest
|
|
447
|
+
* quit grace plus the SIGTERM/SIGKILL fallback, and one health tick more.
|
|
448
|
+
*/
|
|
449
|
+
const STOP_ATTRIBUTION_WINDOW_MS = 90_000;
|
|
450
|
+
/** Lines of pane output kept with a death record: enough for a vendor error. */
|
|
451
|
+
const DEATH_OUTPUT_LINES = 20;
|
|
452
|
+
/** The last `lines` non-empty lines of a captured pane, escape codes already stripped. */
|
|
453
|
+
function paneTail(output, lines = DEATH_OUTPUT_LINES) {
|
|
454
|
+
if (!output)
|
|
455
|
+
return undefined;
|
|
456
|
+
const tail = output.split("\n").filter(line => line.trim()).slice(-lines).join("\n");
|
|
457
|
+
return tail ? tail.slice(-2_000) : undefined;
|
|
458
|
+
}
|
|
425
459
|
/**
|
|
426
460
|
* Startup failed because the CLI's backend is unreachable (see backend-outage.ts).
|
|
427
461
|
* The session-id is deliberately KEPT; the fleet schedules a delayed retry.
|
|
@@ -436,6 +470,10 @@ export class BackendUnreachableStartupError extends Error {
|
|
|
436
470
|
}
|
|
437
471
|
/** Bounded wait (under the pane lock) for the prompt to return before retrying a dropped Enter. */
|
|
438
472
|
const STRANDED_RETRY_READY_WAIT_MS = 30_000;
|
|
473
|
+
/** A pasted Kiro message may be processing for longer than the retry window. */
|
|
474
|
+
const KIRO_SUBMISSION_OBSERVE_MS = 10 * 60_000;
|
|
475
|
+
const KIRO_SUBMISSION_POLL_MS = 1_000;
|
|
476
|
+
const KIRO_SUBMISSION_UNREADABLE_POLLS = 40;
|
|
439
477
|
/** A passive startup phase must clear within this bound before an Enter is sent. */
|
|
440
478
|
/**
|
|
441
479
|
* How long a passive startup/resume transient may sit WITHOUT REPAINTING before
|
|
@@ -481,9 +519,13 @@ const SPAWN_SETTLE_MAX_WAIT_MS = 60_000;
|
|
|
481
519
|
* prompt. A TUI that is still completing its first redraw can swallow Enter.
|
|
482
520
|
*
|
|
483
521
|
* Since the adaptive settle (see {@link waitForPasteSettle}) this value is the
|
|
484
|
-
* *
|
|
485
|
-
*
|
|
486
|
-
*
|
|
522
|
+
* *minimum* settle window for the first delivery — it prevents Enter from firing
|
|
523
|
+
* before the minimum elapses even when the paste's own render goes quiet early.
|
|
524
|
+
* For observed deliveries the quiet-exit path in waitForPasteSettle now checks
|
|
525
|
+
* `now >= fallbackDeadline`, so 1750ms governs every first delivery regardless
|
|
526
|
+
* of whether output is observed. For unobserved deliveries (no control mode,
|
|
527
|
+
* native-queue handoff, mid-wait reconnect, or a paste that never visibly
|
|
528
|
+
* renders) it is the flat fallback delay, same as before.
|
|
487
529
|
*/
|
|
488
530
|
export class FirstDeliveryDelay {
|
|
489
531
|
readyAt = 0;
|
|
@@ -520,9 +562,15 @@ const PASTE_SETTLE_POLL_MS = 100;
|
|
|
520
562
|
* So: watch `lastOutputAt` and send Enter only once the paste's own render has
|
|
521
563
|
* been quiet for {@link PASTE_QUIET_MS}. Bounded both ways —
|
|
522
564
|
*
|
|
523
|
-
* - never earlier than
|
|
524
|
-
*
|
|
525
|
-
*
|
|
565
|
+
* - never earlier than `fallbackMs` from settle start: `fallbackMs` is a
|
|
566
|
+
* **minimum** settle window, not merely a fallback. For first deliveries
|
|
567
|
+
* (fallbackMs = 1750ms) the compositor may render quickly then go quiet at
|
|
568
|
+
* ~550ms while still initialising — the minimum ensures Enter is held until
|
|
569
|
+
* 1750ms regardless of observed output. This applies to every backend's first
|
|
570
|
+
* delivery after ready, not only codex/Luna Reserve. For normal deliveries
|
|
571
|
+
* (fallbackMs = 500ms ≈ PASTE_QUIET_MS) the minimum adds at most a few ms
|
|
572
|
+
* and is effectively unchanged. When the paste produces no observable output
|
|
573
|
+
* at all (`usedFallback`), the same `fallbackMs` deadline governs;
|
|
526
574
|
* - never later than {@link PASTE_SETTLE_CAP_MS} after the paste (`capHit`),
|
|
527
575
|
* so a chatty pane cannot stall delivery.
|
|
528
576
|
*
|
|
@@ -546,7 +594,13 @@ export async function waitForPasteSettle(client, windowId, pasteStartedAt, fallb
|
|
|
546
594
|
const last = client.getLastOutputAt(windowId);
|
|
547
595
|
if (last != null && last > pasteStartedAt) {
|
|
548
596
|
observed = true;
|
|
549
|
-
|
|
597
|
+
// Quiet for PASTE_QUIET_MS AND the minimum settle window has elapsed.
|
|
598
|
+
// The minimum (fallbackDeadline) is the key fix for Luna Reserve: the
|
|
599
|
+
// TUI's first compositor render produces output quickly then goes quiet,
|
|
600
|
+
// but Enter sent at the 500ms quiet point lands while the compositor is
|
|
601
|
+
// still initialising and is swallowed. Holding until fallbackDeadline
|
|
602
|
+
// (1750ms for first deliveries, 500ms for normal ones) eliminates the race.
|
|
603
|
+
if (now - last >= PASTE_QUIET_MS && now >= fallbackDeadline) {
|
|
550
604
|
return { settleMs: now - settleStart, observedPostPasteOutput: true, capHit: false, usedFallback: false };
|
|
551
605
|
}
|
|
552
606
|
}
|
|
@@ -558,16 +612,19 @@ export async function waitForPasteSettle(client, windowId, pasteStartedAt, fallb
|
|
|
558
612
|
}
|
|
559
613
|
// Sleep to the next decision point, at most one poll tick — so the normal
|
|
560
614
|
// paths return at their exact deadlines instead of a poll-width late.
|
|
615
|
+
// When output has been observed but we are still inside the minimum window,
|
|
616
|
+
// sleep to the later of (last quiet deadline) and (fallback deadline) so
|
|
617
|
+
// the loop does not busy-spin between quiet and minimum.
|
|
561
618
|
let wake = capDeadline;
|
|
562
619
|
if (last != null && last > pasteStartedAt)
|
|
563
|
-
wake = Math.min(wake, last + PASTE_QUIET_MS);
|
|
620
|
+
wake = Math.min(wake, Math.max(last + PASTE_QUIET_MS, fallbackDeadline));
|
|
564
621
|
else
|
|
565
622
|
wake = Math.min(wake, fallbackDeadline);
|
|
566
623
|
await new Promise(r => setTimeout(r, Math.min(PASTE_SETTLE_POLL_MS, Math.max(1, wake - now))));
|
|
567
624
|
}
|
|
568
625
|
}
|
|
569
626
|
/** Redact likely credentials and control sequences before pane text reaches logs. */
|
|
570
|
-
export function sanitizePaneTail(pane, lineCount = 5) {
|
|
627
|
+
export function sanitizePaneTail(pane, lineCount = 5, excludeLine) {
|
|
571
628
|
const secretAssignment = /\b(token|secret|password|passwd|api[_-]?key|authorization)\b\s*[:=]\s*\S+/gi;
|
|
572
629
|
const bearer = /\bBearer\s+\S+/gi;
|
|
573
630
|
const knownToken = /\b(?:sk-[A-Za-z0-9_-]+|ghp_[A-Za-z0-9]+|github_pat_[A-Za-z0-9_]+|AKIA[A-Z0-9]{16})\b/g;
|
|
@@ -580,6 +637,10 @@ export function sanitizePaneTail(pane, lineCount = 5) {
|
|
|
580
637
|
lines.pop();
|
|
581
638
|
return lines
|
|
582
639
|
.slice(-lineCount)
|
|
640
|
+
// The proxy-reply chrome predicate sees only control-normalized lines,
|
|
641
|
+
// but must run BEFORE redaction. Redaction can replace a status-footer
|
|
642
|
+
// session UUID with [REDACTED], destroying its structural signature.
|
|
643
|
+
.filter(line => !excludeLine?.(line))
|
|
583
644
|
.map(line => line
|
|
584
645
|
.replace(bearer, "Bearer [REDACTED]")
|
|
585
646
|
.replace(secretAssignment, "$1=[REDACTED]")
|
|
@@ -596,12 +657,14 @@ export function sanitizePaneTail(pane, lineCount = 5) {
|
|
|
596
657
|
* final answer exists only on screen. Everything up to and including the last
|
|
597
658
|
* line of the inbound message we pasted is cut (the reply starts after it),
|
|
598
659
|
* lines with no letters or digits are dropped (borders, separators, spinners,
|
|
599
|
-
* bare prompts), and
|
|
600
|
-
* when
|
|
601
|
-
*
|
|
660
|
+
* bare prompts), and UI chrome is dropped by a backend-specific per-line
|
|
661
|
+
* filter when available (or a legacy single-line ready pattern). Whole-pane
|
|
662
|
+
* readiness regexes cannot identify individual prompt/footer lines. Returns
|
|
663
|
+
* null when what remains is trivial — a proxy message must carry an answer,
|
|
664
|
+
* not chrome. Secrets are redacted by sanitizePaneTail, same as stuck diagnostics.
|
|
602
665
|
*/
|
|
603
666
|
export function extractProxyReplyText(pane, opts = {}) {
|
|
604
|
-
const lines = sanitizePaneTail(pane, opts.maxLines ?? 40);
|
|
667
|
+
const lines = sanitizePaneTail(pane, opts.maxLines ?? 40, opts.isChromeLine);
|
|
605
668
|
const marker = opts.inboundMarker?.trim();
|
|
606
669
|
// Require a distinctive marker: a short one ("ok") would match agent text.
|
|
607
670
|
if (marker && marker.length >= 8) {
|
|
@@ -970,6 +1033,11 @@ export class Daemon extends EventEmitter {
|
|
|
970
1033
|
fatalStartupBlocked = false;
|
|
971
1034
|
/** model_error seen mid-turn: notify only if it survives to the idle screen. */
|
|
972
1035
|
pendingModelErrorKey = null;
|
|
1036
|
+
/** Optional per-instance localhost relay used to observe Muse usage events. */
|
|
1037
|
+
museUsageRelay = null;
|
|
1038
|
+
/** Set after an in-run relay failure so subsequent Muse respawns go direct. */
|
|
1039
|
+
museRelayFallback = false;
|
|
1040
|
+
museRelayFallbackInFlight = null;
|
|
973
1041
|
/**
|
|
974
1042
|
* Let the next pause() proceed from a stuck pane. Set only for the auth-deferred
|
|
975
1043
|
* pause — see the pausePending consumption site for why waiting for idle there
|
|
@@ -1089,6 +1157,42 @@ export class Daemon extends EventEmitter {
|
|
|
1089
1157
|
}
|
|
1090
1158
|
get lastPausedAt() { return this.autoPauseController.lastPausedAt; }
|
|
1091
1159
|
getPauseWakeState() { return this.pauseWakeState; }
|
|
1160
|
+
/**
|
|
1161
|
+
* Codex can show an exhausted-account notice while the user is switching to
|
|
1162
|
+
* a reserve account. Once the live composer and Context footer are back, the
|
|
1163
|
+
* notice is scrollback, not a reason to pause and send /quit.
|
|
1164
|
+
*/
|
|
1165
|
+
isCodexLivePaneSnapshot(pane) {
|
|
1166
|
+
if (this.backend?.binaryName !== "codex")
|
|
1167
|
+
return false;
|
|
1168
|
+
try {
|
|
1169
|
+
if (this.backend.isDeliveryInputReadyPane?.(pane) !== true)
|
|
1170
|
+
return false;
|
|
1171
|
+
const busy = this.backend.getBusyPattern?.();
|
|
1172
|
+
if (busy) {
|
|
1173
|
+
busy.lastIndex = 0;
|
|
1174
|
+
if (busy.test(pane))
|
|
1175
|
+
return false;
|
|
1176
|
+
}
|
|
1177
|
+
const ready = this.backend.getReadyPattern();
|
|
1178
|
+
ready.lastIndex = 0;
|
|
1179
|
+
return ready.test(pane);
|
|
1180
|
+
}
|
|
1181
|
+
catch {
|
|
1182
|
+
return false;
|
|
1183
|
+
}
|
|
1184
|
+
}
|
|
1185
|
+
/** Re-check the current pane after an async quota probe before pausing it. */
|
|
1186
|
+
async isCodexLivePane() {
|
|
1187
|
+
if (this.backend?.binaryName !== "codex" || !this.tmux)
|
|
1188
|
+
return false;
|
|
1189
|
+
try {
|
|
1190
|
+
return this.isCodexLivePaneSnapshot(await this.tmux.capturePane());
|
|
1191
|
+
}
|
|
1192
|
+
catch {
|
|
1193
|
+
return false;
|
|
1194
|
+
}
|
|
1195
|
+
}
|
|
1092
1196
|
/** Whether this instance is in a crash loop (3+ consecutive crashes). */
|
|
1093
1197
|
get isCrashLoop() {
|
|
1094
1198
|
return this.crashCount >= 3;
|
|
@@ -1860,6 +1964,9 @@ export class Daemon extends EventEmitter {
|
|
|
1860
1964
|
// Normal exit (e.g. user Ctrl+C or /exit) — no crash, no respawn
|
|
1861
1965
|
if (paneStatus && exitCode === 0) {
|
|
1862
1966
|
this.setProcessStatus("stopped");
|
|
1967
|
+
// Status 0 is not proof of a clean exit: a codex that hits a quota
|
|
1968
|
+
// wall exits 0 too. Capture what it printed before the window goes.
|
|
1969
|
+
this.logPaneDeath(cliLabel, exitCode, await this.capturePaneOutput());
|
|
1863
1970
|
this.logger.info("CLI exited normally (code 0) — pausing health check");
|
|
1864
1971
|
await this.tmux.killWindow();
|
|
1865
1972
|
this.healthCheckPaused = true;
|
|
@@ -1936,6 +2043,7 @@ export class Daemon extends EventEmitter {
|
|
|
1936
2043
|
lastOutput = cleaned.trimEnd() || undefined;
|
|
1937
2044
|
}
|
|
1938
2045
|
catch { /* best effort — pane may already be gone */ }
|
|
2046
|
+
this.logPaneDeath(cliLabel, exitCode, lastOutput);
|
|
1939
2047
|
// Kill the dead window (remain-on-exit keeps it around) before respawn
|
|
1940
2048
|
if (paneStatus) {
|
|
1941
2049
|
await this.tmux.killWindow();
|
|
@@ -2007,7 +2115,7 @@ export class Daemon extends EventEmitter {
|
|
|
2007
2115
|
this.crashTimestamps = this.crashTimestamps.filter(t => t > Date.now() - crashWindowMs);
|
|
2008
2116
|
if (this.crashTimestamps.length >= 3) {
|
|
2009
2117
|
this.healthCheckPaused = true;
|
|
2010
|
-
this.logger.error({ crashesInWindow: this.crashTimestamps.length }, "3+ crashes in 5 minutes — pausing respawn");
|
|
2118
|
+
this.logger.error({ crashesInWindow: this.crashTimestamps.length, lastExitCode: exitCode ?? null, lastOutput: paneTail(lastOutput) ?? null }, "3+ crashes in 5 minutes — pausing respawn");
|
|
2011
2119
|
// P1: Persist crash state so next process restart skips resume
|
|
2012
2120
|
try {
|
|
2013
2121
|
writeFileSync(join(this.instanceDir, "crash-state.json"), JSON.stringify({
|
|
@@ -2027,7 +2135,7 @@ export class Daemon extends EventEmitter {
|
|
|
2027
2135
|
this.crashCount++;
|
|
2028
2136
|
this.lastCrashAt = Date.now();
|
|
2029
2137
|
if (this.crashCount > max_retries) {
|
|
2030
|
-
this.logger.error({ crashCount: this.crashCount, maxRetries: max_retries }, "Max crash retries exceeded — not respawning");
|
|
2138
|
+
this.logger.error({ crashCount: this.crashCount, maxRetries: max_retries, lastExitCode: exitCode ?? null, lastOutput: paneTail(lastOutput) ?? null }, "Max crash retries exceeded — not respawning");
|
|
2031
2139
|
this.healthCheckPaused = true;
|
|
2032
2140
|
this.emitSupervisionEnded(`it crashed ${this.crashCount} times, exceeding restart_policy.max_retries (${max_retries})`, "Check the logs for the cause, then restart it.");
|
|
2033
2141
|
return; // don't schedule next — given up
|
|
@@ -2331,6 +2439,7 @@ export class Daemon extends EventEmitter {
|
|
|
2331
2439
|
// rebaselining the occurrence count while the error is still displayed — one
|
|
2332
2440
|
// notification, a false recovery log, then silence.
|
|
2333
2441
|
const looksReady = () => !busyPattern?.test(pane) && readyPattern.test(pane);
|
|
2442
|
+
const codexLivePane = this.isCodexLivePaneSnapshot(pane);
|
|
2334
2443
|
// State: waiting for recovery. A missing/outdated ready pattern must not
|
|
2335
2444
|
// suppress every future error forever, so the gate has a hard deadline.
|
|
2336
2445
|
if (this.errorWaitingForRecovery) {
|
|
@@ -2394,6 +2503,17 @@ export class Daemon extends EventEmitter {
|
|
|
2394
2503
|
const key = Daemon.errorPatternKey(ep);
|
|
2395
2504
|
const count = countMatches(ep.pattern);
|
|
2396
2505
|
const seen = this.lastErrorCount.get(key) ?? 0;
|
|
2506
|
+
// The generic Codex usage-limit line remains in scrollback after the
|
|
2507
|
+
// user selects Luna Reserve. A live composer + Context footer is positive
|
|
2508
|
+
// evidence that the pane recovered; baseline the stale occurrence so it
|
|
2509
|
+
// cannot reach the destructive quota pause path.
|
|
2510
|
+
if (codexLivePane && ep.type === "quota" && ep.action === "pause"
|
|
2511
|
+
&& ep.message.startsWith("Codex usage limit reached")) {
|
|
2512
|
+
if (count > 0)
|
|
2513
|
+
this.lastErrorCount.set(key, count);
|
|
2514
|
+
this.logger.debug("Codex usage-limit text is stale — live reserve pane is running");
|
|
2515
|
+
continue;
|
|
2516
|
+
}
|
|
2397
2517
|
if (count <= seen) {
|
|
2398
2518
|
// Occurrences scrolled out of the capture buffer → lower the baseline
|
|
2399
2519
|
// so a future re-occurrence still counts as new (no permanent suppress).
|
|
@@ -2588,12 +2708,61 @@ export class Daemon extends EventEmitter {
|
|
|
2588
2708
|
// is a successful no-op here, not an error.
|
|
2589
2709
|
this.logger.debug({ dialog: dialog.description }, "Backend clear dialog did not appear");
|
|
2590
2710
|
}
|
|
2711
|
+
/**
|
|
2712
|
+
* The last time AgEnD itself asked the CLI to go away, and why (#927). Read
|
|
2713
|
+
* when a pane dies, to tell "AgEnD stopped it" from "the CLI exited on its
|
|
2714
|
+
* own" — a codex that exits 0 on its own and one AgEnD sent /quit look the
|
|
2715
|
+
* same from the pane.
|
|
2716
|
+
*/
|
|
2717
|
+
lastStopRequest = null;
|
|
2718
|
+
/** Record, and log, that AgEnD is about to stop the CLI. */
|
|
2719
|
+
noteStopRequest(via, reason) {
|
|
2720
|
+
this.lastStopRequest = { via, reason, at: Date.now() };
|
|
2721
|
+
this.logger.info({ via, reason }, `AgEnD is stopping the CLI: ${via} (${reason})`);
|
|
2722
|
+
}
|
|
2723
|
+
/** Capture the pane's recent output with escape codes stripped. Never throws. */
|
|
2724
|
+
async capturePaneOutput() {
|
|
2725
|
+
try {
|
|
2726
|
+
const raw = await this.tmux?.capturePaneWithHistory(50);
|
|
2727
|
+
const cleaned = raw?.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
|
|
2728
|
+
return cleaned?.trimEnd() || undefined;
|
|
2729
|
+
}
|
|
2730
|
+
catch {
|
|
2731
|
+
return undefined;
|
|
2732
|
+
}
|
|
2733
|
+
}
|
|
2734
|
+
/**
|
|
2735
|
+
* One record per pane death: exit status, when, who, and what the CLI printed
|
|
2736
|
+
* last — the line that was missing when a codex on Luna Reserve died with
|
|
2737
|
+
* status 0 and daemon.log said only "exited normally" (#927). Logging only;
|
|
2738
|
+
* the caller decides what happens next.
|
|
2739
|
+
*/
|
|
2740
|
+
logPaneDeath(cliLabel, exitCode, lastOutput) {
|
|
2741
|
+
const request = this.lastStopRequest;
|
|
2742
|
+
const sinceStopMs = request ? Date.now() - request.at : null;
|
|
2743
|
+
const byAgend = request !== null && sinceStopMs !== null && sinceStopMs <= STOP_ATTRIBUTION_WINDOW_MS;
|
|
2744
|
+
const record = {
|
|
2745
|
+
exitCode: exitCode ?? null,
|
|
2746
|
+
diedAt: new Date().toISOString(),
|
|
2747
|
+
initiatedBy: byAgend ? "agend" : "cli",
|
|
2748
|
+
...(byAgend ? { stopVia: request.via, stopReason: request.reason, sinceStopMs } : {}),
|
|
2749
|
+
lastOutput: paneTail(lastOutput) ?? null,
|
|
2750
|
+
};
|
|
2751
|
+
if (byAgend) {
|
|
2752
|
+
this.logger.info(record, `${cliLabel} exited after AgEnD stopped it (${request.via}: ${request.reason})`);
|
|
2753
|
+
}
|
|
2754
|
+
else {
|
|
2755
|
+
this.logger.warn(record, `${cliLabel} exited on its own (exit code ${exitCode ?? "unknown"}) — last output recorded`);
|
|
2756
|
+
}
|
|
2757
|
+
}
|
|
2591
2758
|
/** Send the backend-specific graceful quit command/key sequence. */
|
|
2592
|
-
async sendQuitSequence() {
|
|
2759
|
+
async sendQuitSequence(reason = "unspecified") {
|
|
2593
2760
|
if (!this.tmux || !this.backend)
|
|
2594
2761
|
return false;
|
|
2595
2762
|
const quitCmd = this.backend.getQuitCommand();
|
|
2596
2763
|
const quitKey = this.backend.getQuitKey?.();
|
|
2764
|
+
if (quitCmd || quitKey)
|
|
2765
|
+
this.noteStopRequest(quitCmd ? `quit command ${quitCmd}` : `quit key ${quitKey}`, reason);
|
|
2597
2766
|
if (quitCmd) {
|
|
2598
2767
|
if (!await this.tmux.sendKeys(quitCmd))
|
|
2599
2768
|
return false;
|
|
@@ -2614,6 +2783,26 @@ export class Daemon extends EventEmitter {
|
|
|
2614
2783
|
}
|
|
2615
2784
|
return true;
|
|
2616
2785
|
}
|
|
2786
|
+
/**
|
|
2787
|
+
* Kiro drops `/quit` while a turn is running. Stop that turn first, then use
|
|
2788
|
+
* the same delivery readiness gate that protects Enter from the busy pane.
|
|
2789
|
+
* Returning false is fail-closed: the caller skips `/quit` and enters the
|
|
2790
|
+
* bounded SIGTERM -> SIGKILL fallback instead of typing into an unknown pane.
|
|
2791
|
+
*/
|
|
2792
|
+
async drainBusyKiroForStop(windowId) {
|
|
2793
|
+
if (this.backend?.binaryName !== "kiro-cli")
|
|
2794
|
+
return true;
|
|
2795
|
+
const readiness = await this.paneReadinessForDelivery(windowId);
|
|
2796
|
+
if (readiness === "ready")
|
|
2797
|
+
return true;
|
|
2798
|
+
if (readiness !== "busy")
|
|
2799
|
+
return false;
|
|
2800
|
+
const cancelKey = this.backend.getCancelKey?.() ?? "Escape";
|
|
2801
|
+
const sent = await this.tmux?.sendSpecialKey(cancelKey);
|
|
2802
|
+
if (!sent)
|
|
2803
|
+
return false;
|
|
2804
|
+
return this.waitForPaneReadyForDelivery(windowId, KIRO_STOP_IDLE_BUDGET_MS);
|
|
2805
|
+
}
|
|
2617
2806
|
async stop() {
|
|
2618
2807
|
this.logger.info("Stopping daemon instance");
|
|
2619
2808
|
this.turnReplyGuard.reset();
|
|
@@ -2622,6 +2811,13 @@ export class Daemon extends EventEmitter {
|
|
|
2622
2811
|
this.inputTransientGuardGeneration = null;
|
|
2623
2812
|
this.freezeRuntimeMonitors();
|
|
2624
2813
|
this.pendingIpcRequests.clear();
|
|
2814
|
+
if (this.museUsageRelay) {
|
|
2815
|
+
await this.museUsageRelay.stop().catch(() => { });
|
|
2816
|
+
this.museUsageRelay = null;
|
|
2817
|
+
}
|
|
2818
|
+
else if (this.backend?.binaryName === "muse") {
|
|
2819
|
+
clearMuseUsageSnapshot(this.instanceDir);
|
|
2820
|
+
}
|
|
2625
2821
|
if (this.adapter)
|
|
2626
2822
|
await this.adapter.stop();
|
|
2627
2823
|
// Notify MCP servers of graceful shutdown (prevents reconnect attempts)
|
|
@@ -2632,11 +2828,18 @@ export class Daemon extends EventEmitter {
|
|
|
2632
2828
|
this.saveSessionId();
|
|
2633
2829
|
this.healthCheckPaused = true;
|
|
2634
2830
|
let killed = false;
|
|
2635
|
-
const
|
|
2831
|
+
const windowId = this.tmux.getWindowId();
|
|
2832
|
+
const kiroReady = windowId ? await this.drainBusyKiroForStop(windowId) : true;
|
|
2833
|
+
const quitSent = kiroReady && await this.sendQuitSequence("graceful stop");
|
|
2834
|
+
const quitGraceMs = this.backend?.binaryName === "kiro-cli"
|
|
2835
|
+
? KIRO_STOP_QUIT_GRACE_MS : 3_000;
|
|
2836
|
+
const sigtermGraceMs = this.backend?.binaryName === "kiro-cli"
|
|
2837
|
+
? KIRO_STOP_SIGTERM_GRACE_MS : 1_000;
|
|
2636
2838
|
if (quitSent) {
|
|
2637
|
-
// Wait up to
|
|
2839
|
+
// Wait up to the backend's bounded graceful-exit window, polling every
|
|
2840
|
+
// 200ms. A healthy CLI
|
|
2638
2841
|
// exits within ~1s; a longer wait just delays the force-kill fallback.
|
|
2639
|
-
for (let
|
|
2842
|
+
for (let elapsed = 0; elapsed < quitGraceMs; elapsed += 200) {
|
|
2640
2843
|
await new Promise(r => setTimeout(r, 200));
|
|
2641
2844
|
const status = await this.tmux.getPaneStatus();
|
|
2642
2845
|
if (!status || !status.alive) {
|
|
@@ -2646,9 +2849,9 @@ export class Daemon extends EventEmitter {
|
|
|
2646
2849
|
}
|
|
2647
2850
|
}
|
|
2648
2851
|
if (!killed) {
|
|
2649
|
-
this.logger.warn({ quitSent }, "CLI did not exit gracefully within
|
|
2650
|
-
await this.killProcessTree("SIGTERM");
|
|
2651
|
-
for (let
|
|
2852
|
+
this.logger.warn({ quitSent, quitGraceMs }, "CLI did not exit gracefully within its bounded quit grace — falling back to SIGTERM");
|
|
2853
|
+
await this.killProcessTree("SIGTERM", "graceful stop: the CLI outlived its quit grace");
|
|
2854
|
+
for (let elapsed = 0; elapsed < sigtermGraceMs; elapsed += 200) {
|
|
2652
2855
|
await new Promise(r => setTimeout(r, 200));
|
|
2653
2856
|
const status = await this.tmux.getPaneStatus();
|
|
2654
2857
|
if (!status || !status.alive) {
|
|
@@ -2659,7 +2862,7 @@ export class Daemon extends EventEmitter {
|
|
|
2659
2862
|
}
|
|
2660
2863
|
if (!killed) {
|
|
2661
2864
|
this.logger.warn("CLI process tree survived SIGTERM — falling back to SIGKILL");
|
|
2662
|
-
await this.killProcessTree("SIGKILL");
|
|
2865
|
+
await this.killProcessTree("SIGKILL", "graceful stop: the CLI survived SIGTERM");
|
|
2663
2866
|
await new Promise(r => setTimeout(r, 200));
|
|
2664
2867
|
}
|
|
2665
2868
|
// Always kill window — remain-on-exit keeps dead panes around after CLI exits
|
|
@@ -2761,6 +2964,9 @@ export class Daemon extends EventEmitter {
|
|
|
2761
2964
|
// into an ordinary idle-timeout pause of a busy instance.
|
|
2762
2965
|
const allowStuck = this.pauseAllowStuck;
|
|
2763
2966
|
this.pauseAllowStuck = false;
|
|
2967
|
+
// Why this pause, for the stop record (#927): an idle pause and an
|
|
2968
|
+
// auth-deferred one look the same from the pane.
|
|
2969
|
+
const pauseReason = allowStuck ? "pause (auth-deferred, stuck pane)" : "pause (idle)";
|
|
2764
2970
|
const pausableState = this.instanceState === "idle"
|
|
2765
2971
|
|| (allowStuck && this.instanceState === "stuck");
|
|
2766
2972
|
if (!pausableState || this.pasteQueueDepth > 0) {
|
|
@@ -2773,7 +2979,7 @@ export class Daemon extends EventEmitter {
|
|
|
2773
2979
|
const transition = (async () => {
|
|
2774
2980
|
try {
|
|
2775
2981
|
this.saveSessionId();
|
|
2776
|
-
await this.sendQuitSequence();
|
|
2982
|
+
await this.sendQuitSequence(pauseReason);
|
|
2777
2983
|
let exited = false;
|
|
2778
2984
|
for (let i = 0; i < 15; i++) {
|
|
2779
2985
|
await new Promise(r => setTimeout(r, 200));
|
|
@@ -2784,11 +2990,11 @@ export class Daemon extends EventEmitter {
|
|
|
2784
2990
|
}
|
|
2785
2991
|
}
|
|
2786
2992
|
if (!exited) {
|
|
2787
|
-
await this.killProcessTree("SIGTERM");
|
|
2993
|
+
await this.killProcessTree("SIGTERM", `${pauseReason}: the CLI outlived its quit grace`);
|
|
2788
2994
|
await new Promise(r => setTimeout(r, 1_000));
|
|
2789
2995
|
const status = await this.tmux?.getPaneStatus();
|
|
2790
2996
|
if (status?.alive) {
|
|
2791
|
-
await this.killProcessTree("SIGKILL");
|
|
2997
|
+
await this.killProcessTree("SIGKILL", `${pauseReason}: the CLI survived SIGTERM`);
|
|
2792
2998
|
await new Promise(r => setTimeout(r, 200));
|
|
2793
2999
|
}
|
|
2794
3000
|
}
|
|
@@ -3201,7 +3407,11 @@ export class Daemon extends EventEmitter {
|
|
|
3201
3407
|
pane = await this.tmux?.capturePane();
|
|
3202
3408
|
if (!pane)
|
|
3203
3409
|
return;
|
|
3204
|
-
const text = extractProxyReplyText(pane, {
|
|
3410
|
+
const text = extractProxyReplyText(pane, {
|
|
3411
|
+
inboundMarker: target.inboundMarker,
|
|
3412
|
+
readyPattern: this.instanceStateReadyPattern,
|
|
3413
|
+
isChromeLine: line => this.backend?.isProxyReplyChromeLine?.(line) ?? false,
|
|
3414
|
+
});
|
|
3205
3415
|
if (!text) {
|
|
3206
3416
|
this.logger.debug("Dead-MCP proxy reply skipped — pane tail is trivial");
|
|
3207
3417
|
return;
|
|
@@ -3939,14 +4149,16 @@ export class Daemon extends EventEmitter {
|
|
|
3939
4149
|
* steer's verdict decide whether a queued message's sender is told its
|
|
3940
4150
|
* delivery failed — a false ❌ carrying the wrong correlation id.
|
|
3941
4151
|
*/
|
|
3942
|
-
failDelivery(verdict, status) {
|
|
4152
|
+
failDelivery(verdict, status, phase = "unknown", proof = "undelivered") {
|
|
3943
4153
|
verdict.reached = true;
|
|
4154
|
+
verdict.phase = phase;
|
|
4155
|
+
verdict.proof = proof;
|
|
3944
4156
|
if (status)
|
|
3945
4157
|
this.emit("message_failed", status); // ❌
|
|
3946
4158
|
return false;
|
|
3947
4159
|
}
|
|
3948
4160
|
/** Tell the fleet when an already-accepted cross-instance pane write failed. */
|
|
3949
|
-
reportCrossInstanceDeliveryFailure(meta, error) {
|
|
4161
|
+
reportCrossInstanceDeliveryFailure(meta, verdict, error) {
|
|
3950
4162
|
if (!meta.from_instance)
|
|
3951
4163
|
return;
|
|
3952
4164
|
this.ipcServer?.broadcast({
|
|
@@ -3954,7 +4166,10 @@ export class Daemon extends EventEmitter {
|
|
|
3954
4166
|
senderSession: meta.from_instance,
|
|
3955
4167
|
targetInstance: this.name,
|
|
3956
4168
|
correlationId: meta.correlation_id ?? "unknown",
|
|
3957
|
-
|
|
4169
|
+
// Never use getLastPasteError as a generic fallback: it may describe an
|
|
4170
|
+
// earlier paste, and #910 was a post-submit proof failure, not a paste
|
|
4171
|
+
// rejection. Keep the sender diagnostic phase-specific and non-secret.
|
|
4172
|
+
error: `delivery failed: phase=${verdict?.phase ?? (error ? "exception" : "unknown")}; proof=${verdict?.proof ?? (error ? "threw" : "none")}`,
|
|
3958
4173
|
});
|
|
3959
4174
|
}
|
|
3960
4175
|
/**
|
|
@@ -4008,12 +4223,12 @@ export class Daemon extends EventEmitter {
|
|
|
4008
4223
|
// Same rule as the queued path: a steer that never got to try is not a
|
|
4009
4224
|
// delivery failure. This holder is the steer's own, which is the point
|
|
4010
4225
|
// — it runs on steerLock while a queued delivery runs on pasteLock.
|
|
4011
|
-
this.reportCrossInstanceDeliveryFailure(meta);
|
|
4226
|
+
this.reportCrossInstanceDeliveryFailure(meta, verdict);
|
|
4012
4227
|
}
|
|
4013
4228
|
}).catch(err => {
|
|
4014
4229
|
this.logger.warn({ err: err.message }, "steer delivery error");
|
|
4015
4230
|
if (this.isDeliveryEpochCurrent(deliveryEpoch)) {
|
|
4016
|
-
this.reportCrossInstanceDeliveryFailure(meta, err.message);
|
|
4231
|
+
this.reportCrossInstanceDeliveryFailure(meta, undefined, err.message);
|
|
4017
4232
|
}
|
|
4018
4233
|
});
|
|
4019
4234
|
}
|
|
@@ -4147,7 +4362,7 @@ export class Daemon extends EventEmitter {
|
|
|
4147
4362
|
// not ready yet, a cancel, a storm hold or a shutdown all return
|
|
4148
4363
|
// false without one, and telling the sender its message was lost
|
|
4149
4364
|
// there would be the false ❌ of #826 in its other form.
|
|
4150
|
-
this.reportCrossInstanceDeliveryFailure(meta);
|
|
4365
|
+
this.reportCrossInstanceDeliveryFailure(meta, verdict);
|
|
4151
4366
|
}
|
|
4152
4367
|
}
|
|
4153
4368
|
finally {
|
|
@@ -4156,7 +4371,7 @@ export class Daemon extends EventEmitter {
|
|
|
4156
4371
|
}).catch(err => {
|
|
4157
4372
|
this.logger.warn({ err: err.message }, "pasteLock delivery error — chain continues");
|
|
4158
4373
|
if (this.isDeliveryEpochCurrent(deliveryEpoch)) {
|
|
4159
|
-
this.reportCrossInstanceDeliveryFailure(meta, err.message);
|
|
4374
|
+
this.reportCrossInstanceDeliveryFailure(meta, undefined, err.message);
|
|
4160
4375
|
}
|
|
4161
4376
|
});
|
|
4162
4377
|
this.logger.debug({ user: meta.user, text: content.slice(0, 100) }, "Queued channel message for delivery");
|
|
@@ -4201,7 +4416,16 @@ export class Daemon extends EventEmitter {
|
|
|
4201
4416
|
return false;
|
|
4202
4417
|
}
|
|
4203
4418
|
// Before anything reads the window id: a spawn in progress is about to change it.
|
|
4204
|
-
|
|
4419
|
+
// A false return means the cap expired with a spawn STILL running — the
|
|
4420
|
+
// generation captured below is in-progress evidence, not settled evidence,
|
|
4421
|
+
// so the critical section must still back out (see settledClean).
|
|
4422
|
+
if (this.spawnSettled && status)
|
|
4423
|
+
this.emit("message_queued", status);
|
|
4424
|
+
const settledClean = await this.waitForSpawnToSettle();
|
|
4425
|
+
// Everything captured below (window id, readiness verdicts) belongs to
|
|
4426
|
+
// this spawn generation. A spawn that starts afterwards is detected inside
|
|
4427
|
+
// the critical section, where this delivery backs out and redoes itself.
|
|
4428
|
+
const settleGeneration = this.spawnGeneration;
|
|
4205
4429
|
if (cancelled())
|
|
4206
4430
|
return false;
|
|
4207
4431
|
if (this.refuseFatalStartupDelivery(verdict, status))
|
|
@@ -4226,7 +4450,8 @@ export class Daemon extends EventEmitter {
|
|
|
4226
4450
|
// fall through to the wait.
|
|
4227
4451
|
const canHandOff = (supportsQueuedInput || opts?.steer)
|
|
4228
4452
|
&& readiness === "busy"
|
|
4229
|
-
&& (await this.probeBlockingDialog()).state === "clear"
|
|
4453
|
+
&& (await this.probeBlockingDialog()).state === "clear"
|
|
4454
|
+
&& await this.hasPositiveDeliveryInput();
|
|
4230
4455
|
if (canHandOff) {
|
|
4231
4456
|
// Native queue (codex), or an explicit /steer: hand the complete
|
|
4232
4457
|
// paste+Enter transaction to the busy CLI now. For steer this is the
|
|
@@ -4246,7 +4471,7 @@ export class Daemon extends EventEmitter {
|
|
|
4246
4471
|
// wedged CLI (where the text would sit unsubmitted and the next message
|
|
4247
4472
|
// would land on top of it) — and instead of holding the queue silently.
|
|
4248
4473
|
this.logger.error("Pane still busy after the idle wait — reporting delivery failure");
|
|
4249
|
-
return this.failDelivery(verdict, status);
|
|
4474
|
+
return this.failDelivery(verdict, status, "readiness", "timeout-before-write");
|
|
4250
4475
|
}
|
|
4251
4476
|
}
|
|
4252
4477
|
}
|
|
@@ -4267,7 +4492,7 @@ export class Daemon extends EventEmitter {
|
|
|
4267
4492
|
// recovery Enter, so exactly STRANDED_INPUT_MAX_ROUNDS of them go out.
|
|
4268
4493
|
if (round >= STRANDED_INPUT_MAX_ROUNDS) {
|
|
4269
4494
|
this.logger.error({ round }, "Input row still not clear after the stranded-text recovery budget — reporting delivery failure");
|
|
4270
|
-
return this.failDelivery(verdict, status);
|
|
4495
|
+
return this.failDelivery(verdict, status, "stranded-input", "retry-budget-exhausted");
|
|
4271
4496
|
}
|
|
4272
4497
|
const stranded = await this.submitStrandedInputIfAny(windowId);
|
|
4273
4498
|
if (cancelled())
|
|
@@ -4276,14 +4501,14 @@ export class Daemon extends EventEmitter {
|
|
|
4276
4501
|
break;
|
|
4277
4502
|
if (stranded === "failed") {
|
|
4278
4503
|
this.logger.error({ round }, "Could not submit the stranded text — reporting delivery failure");
|
|
4279
|
-
return this.failDelivery(verdict, status);
|
|
4504
|
+
return this.failDelivery(verdict, status, "stranded-input", "enter-failed");
|
|
4280
4505
|
}
|
|
4281
4506
|
const readyAgain = await this.waitForPaneReadyForDelivery(windowId);
|
|
4282
4507
|
if (cancelled())
|
|
4283
4508
|
return false;
|
|
4284
4509
|
if (!readyAgain) {
|
|
4285
4510
|
this.logger.error("Pane never returned to its prompt after submitting stranded input — reporting delivery failure");
|
|
4286
|
-
return this.failDelivery(verdict, status);
|
|
4511
|
+
return this.failDelivery(verdict, status, "stranded-input", "prompt-timeout");
|
|
4287
4512
|
}
|
|
4288
4513
|
}
|
|
4289
4514
|
}
|
|
@@ -4295,13 +4520,22 @@ export class Daemon extends EventEmitter {
|
|
|
4295
4520
|
const outcome = await this.paneWriteLock.run(async () => {
|
|
4296
4521
|
if (cancelled())
|
|
4297
4522
|
return false;
|
|
4523
|
+
// A spawn that started after the settle wait above invalidates the
|
|
4524
|
+
// window id and every readiness verdict since: the pane about to be
|
|
4525
|
+
// written may already belong to a replacement process. The same holds
|
|
4526
|
+
// when the settle wait itself timed out (!settledClean) or a spawn is
|
|
4527
|
+
// running right now — none of those is settled evidence. Never wait
|
|
4528
|
+
// here (startup dismissal needs this lock) — back out and redo the
|
|
4529
|
+
// delivery from the top, where the settle wait runs again outside it.
|
|
4530
|
+
if (!settledClean || settleGeneration !== this.spawnGeneration || this.spawning)
|
|
4531
|
+
return "spawn-started";
|
|
4298
4532
|
if (this.refuseFatalStartupDelivery(verdict, status))
|
|
4299
4533
|
return false;
|
|
4300
4534
|
// A passive startup phase may paint after the outer readiness probe.
|
|
4301
4535
|
// It clears without input, so waiting under the pane lock cannot starve
|
|
4302
4536
|
// a dialog dismisser and closes the final clear→paste TOCTOU window.
|
|
4303
4537
|
if (!(await this.waitForInputTransientToClear("pre-write"))) {
|
|
4304
|
-
return this.failDelivery(verdict, status);
|
|
4538
|
+
return this.failDelivery(verdict, status, "pre-write", "input-transient-timeout");
|
|
4305
4539
|
}
|
|
4306
4540
|
// TOCTOU: the probes above ran outside this lock, and the CLI repaints
|
|
4307
4541
|
// whenever it likes — a resume prompt can be painted between "clear" and
|
|
@@ -4312,8 +4546,25 @@ export class Daemon extends EventEmitter {
|
|
|
4312
4546
|
if (probe.state !== "clear")
|
|
4313
4547
|
return "dialog";
|
|
4314
4548
|
}
|
|
4549
|
+
if (!(await this.hasPositiveDeliveryInput()))
|
|
4550
|
+
return "dialog";
|
|
4315
4551
|
return this.writeMessageToPane(formatted, windowId, handingOffToNativeQueue, status, opts?.submissionId, verdict);
|
|
4316
4552
|
});
|
|
4553
|
+
if (outcome === "spawn-started") {
|
|
4554
|
+
// The pane changed under this delivery: its queued paste must not land
|
|
4555
|
+
// in the replacement process's first screen. Redo the whole delivery
|
|
4556
|
+
// (fresh window id, fresh probes) — bounded, so spawn churn ends in an
|
|
4557
|
+
// honest failure rather than a message that never lands.
|
|
4558
|
+
const retries = (opts?.spawnRetry ?? 0) + 1;
|
|
4559
|
+
if (retries > DELIVERY_SPAWN_RACE_MAX_ROUNDS) {
|
|
4560
|
+
this.logger.error({ rounds: retries }, "A spawn kept starting before the pane write — reporting delivery failure");
|
|
4561
|
+
return this.failDelivery(verdict, status);
|
|
4562
|
+
}
|
|
4563
|
+
return this.deliverMessage(formatted, status, { ...opts, spawnRetry: retries });
|
|
4564
|
+
}
|
|
4565
|
+
if (typeof outcome === "object") {
|
|
4566
|
+
return this.observeKiroSubmission(formatted, outcome, status, verdict, cancelled);
|
|
4567
|
+
}
|
|
4317
4568
|
if (outcome !== "dialog")
|
|
4318
4569
|
return outcome;
|
|
4319
4570
|
// Wait OUTSIDE the lock (holding it would starve the runtime dismisser),
|
|
@@ -4321,14 +4572,14 @@ export class Daemon extends EventEmitter {
|
|
|
4321
4572
|
// an honest failure rather than a message that never lands.
|
|
4322
4573
|
if (round + 1 >= LATE_DIALOG_WRITE_ROUNDS) {
|
|
4323
4574
|
this.logger.error({ rounds: round + 1 }, "A dialog kept appearing before the pane write — reporting delivery failure");
|
|
4324
|
-
return this.failDelivery(verdict, status);
|
|
4575
|
+
return this.failDelivery(verdict, status, "pre-write", "dialog-or-input-not-ready");
|
|
4325
4576
|
}
|
|
4326
|
-
this.logger.info("Dialog appeared before the pane write — waiting for
|
|
4577
|
+
this.logger.info("Dialog or input transition appeared before the pane write — waiting for readiness");
|
|
4327
4578
|
const clear = gateWindowId ? await this.waitForPaneReadyForDelivery(gateWindowId) : false;
|
|
4328
4579
|
if (cancelled())
|
|
4329
4580
|
return false;
|
|
4330
4581
|
if (!clear) {
|
|
4331
|
-
return this.failDelivery(verdict, status);
|
|
4582
|
+
return this.failDelivery(verdict, status, "pre-write", "readiness-timeout");
|
|
4332
4583
|
}
|
|
4333
4584
|
}
|
|
4334
4585
|
}
|
|
@@ -4436,8 +4687,18 @@ export class Daemon extends EventEmitter {
|
|
|
4436
4687
|
this.logger.warn({ phase, generation }, "Spawn changed during the input-availability probe — refusing to send Enter into the replacement pane");
|
|
4437
4688
|
return false;
|
|
4438
4689
|
}
|
|
4439
|
-
if (probe.state === "clear")
|
|
4690
|
+
if (probe.state === "clear") {
|
|
4691
|
+
// Re-arm the first-delivery delay when a transient was actually observed
|
|
4692
|
+
// and then cleared, but ONLY from the pre-write phase. That phase runs
|
|
4693
|
+
// before consume() in writeMessageToPane, so the re-arm benefits the
|
|
4694
|
+
// current delivery's paste settle. Enter-path callers (initial-submit,
|
|
4695
|
+
// retries) run after consume() — re-arming there would leave the flag
|
|
4696
|
+
// for the NEXT unrelated delivery, which must not be slowed.
|
|
4697
|
+
if (observedDescription !== null && phase === "pre-write") {
|
|
4698
|
+
this.firstDeliveryDelay.recordReady();
|
|
4699
|
+
}
|
|
4440
4700
|
return true;
|
|
4701
|
+
}
|
|
4441
4702
|
if (probe.state === "active" && observedDescription !== probe.transient.description) {
|
|
4442
4703
|
observedDescription = probe.transient.description;
|
|
4443
4704
|
this.logger.info({ phase, transient: probe.transient.description, generation }, "CLI is still completing startup — waiting before sending Enter");
|
|
@@ -4589,6 +4850,27 @@ export class Daemon extends EventEmitter {
|
|
|
4589
4850
|
async isPaneReadyForDelivery(windowId) {
|
|
4590
4851
|
return (await this.paneReadinessForDelivery(windowId)) === "ready";
|
|
4591
4852
|
}
|
|
4853
|
+
needsStartupInputProof() {
|
|
4854
|
+
return !!this.backend?.isDeliveryInputReadyPane
|
|
4855
|
+
&& this.inputTransientGuardGeneration === this.spawnGeneration;
|
|
4856
|
+
}
|
|
4857
|
+
/** Codex can paint a prompt before the TTY enters raw mode. Both are required. */
|
|
4858
|
+
async hasPositiveDeliveryInput() {
|
|
4859
|
+
if (!this.needsStartupInputProof())
|
|
4860
|
+
return true;
|
|
4861
|
+
const check = this.backend?.isDeliveryInputReadyPane;
|
|
4862
|
+
if (!check || !this.tmux)
|
|
4863
|
+
return !check;
|
|
4864
|
+
try {
|
|
4865
|
+
const mode = await this.tmux.getPaneInputMode?.();
|
|
4866
|
+
if (mode !== "raw")
|
|
4867
|
+
return false;
|
|
4868
|
+
return check.call(this.backend, await this.tmux.capturePane());
|
|
4869
|
+
}
|
|
4870
|
+
catch {
|
|
4871
|
+
return false;
|
|
4872
|
+
}
|
|
4873
|
+
}
|
|
4592
4874
|
/**
|
|
4593
4875
|
* Why a pane is not deliverable matters. "busy" may be handed to a native
|
|
4594
4876
|
* input queue or steered (after its own dialog probe); "dialog", "transient"
|
|
@@ -4607,6 +4889,9 @@ export class Daemon extends EventEmitter {
|
|
|
4607
4889
|
return "transient";
|
|
4608
4890
|
if (transient.state === "unknown")
|
|
4609
4891
|
return "unknown";
|
|
4892
|
+
if (this.needsStartupInputProof()) {
|
|
4893
|
+
return await this.hasPositiveDeliveryInput() ? "ready" : "transient";
|
|
4894
|
+
}
|
|
4610
4895
|
if (this.backend?.dropsEnterWhileBusy?.() !== true)
|
|
4611
4896
|
return "ready";
|
|
4612
4897
|
const prompt = this.backend.getBottomReadyPattern?.();
|
|
@@ -4651,7 +4936,8 @@ export class Daemon extends EventEmitter {
|
|
|
4651
4936
|
// screen", and a dialog disappears without any output edge the silence
|
|
4652
4937
|
// gate could see.
|
|
4653
4938
|
const deadline = Date.now() + timeoutMs;
|
|
4654
|
-
const bottomGated = this.backend?.dropsEnterWhileBusy?.() === true
|
|
4939
|
+
const bottomGated = this.backend?.dropsEnterWhileBusy?.() === true
|
|
4940
|
+
|| this.needsStartupInputProof();
|
|
4655
4941
|
let unknownStreak = 0;
|
|
4656
4942
|
let transientBudget = null;
|
|
4657
4943
|
for (;;) {
|
|
@@ -4774,11 +5060,12 @@ export class Daemon extends EventEmitter {
|
|
|
4774
5060
|
return "busy";
|
|
4775
5061
|
if (this.backend?.getBusyPattern?.()?.test(pane))
|
|
4776
5062
|
return "busy";
|
|
5063
|
+
if (this.backend?.isDeliveryInputReadyPane && !this.backend.isDeliveryInputReadyPane(pane))
|
|
5064
|
+
return "busy";
|
|
4777
5065
|
if (strandedAgendMessageInInput(pane, prompt))
|
|
4778
5066
|
return "stranded";
|
|
4779
|
-
// Kiro
|
|
4780
|
-
//
|
|
4781
|
-
// enabled solely to recover a positively identified old AgEnD strand.
|
|
5067
|
+
// Kiro uses the bottom row; Codex's positive prompt/footer check above
|
|
5068
|
+
// rules out a historical transcript echo or a transition/modal screen.
|
|
4782
5069
|
return !requireBottomReady || bottomRowIsReady(pane, prompt) ? "clear" : "busy";
|
|
4783
5070
|
}
|
|
4784
5071
|
/**
|
|
@@ -4816,6 +5103,126 @@ export class Daemon extends EventEmitter {
|
|
|
4816
5103
|
this.logger.warn("Could not read the pane after Enter — not confirming the submission");
|
|
4817
5104
|
return sawOutput && residue === "absent";
|
|
4818
5105
|
}
|
|
5106
|
+
/** Once Kiro returns to its prompt, its unique envelope id in history proves submission. */
|
|
5107
|
+
async kiroSubmissionEvidence(windowId, formatted, signature) {
|
|
5108
|
+
if (!this.tmux)
|
|
5109
|
+
return "unreadable";
|
|
5110
|
+
let pane;
|
|
5111
|
+
try {
|
|
5112
|
+
pane = await this.tmux.capturePane();
|
|
5113
|
+
}
|
|
5114
|
+
catch {
|
|
5115
|
+
return "unreadable";
|
|
5116
|
+
}
|
|
5117
|
+
const prompt = this.backend?.getBottomReadyPattern?.();
|
|
5118
|
+
if (prompt && bottomRowIsReady(pane, prompt)
|
|
5119
|
+
&& inputShowsPastedText(inputAreaText(pane, prompt) ?? "", signature.value))
|
|
5120
|
+
return "stranded";
|
|
5121
|
+
if (prompt && pasteLeftInInput(pane, prompt, formatted))
|
|
5122
|
+
return "stranded";
|
|
5123
|
+
// During generation the same text can still be typeahead for an older
|
|
5124
|
+
// turn. A history hit is safe only after this pane returns to its prompt.
|
|
5125
|
+
if (!signature.unique || !this.tmux.capturePaneWithHistory
|
|
5126
|
+
|| !(await this.isPaneReadyForDelivery(windowId)))
|
|
5127
|
+
return "unknown";
|
|
5128
|
+
try {
|
|
5129
|
+
const history = await this.tmux.capturePaneWithHistory(300);
|
|
5130
|
+
// History includes the CURRENT input row. The earlier viewport capture
|
|
5131
|
+
// and readiness probe may belong to an older frame: Kiro can finish a
|
|
5132
|
+
// turn and expose our unsent typeahead between those reads. Judge both
|
|
5133
|
+
// readiness and input residue on this same, final snapshot before its
|
|
5134
|
+
// message id can count as a submitted transcript echo.
|
|
5135
|
+
if (!prompt || !bottomRowIsReady(history, prompt))
|
|
5136
|
+
return "unknown";
|
|
5137
|
+
if (inputShowsPastedText(inputAreaText(history, prompt) ?? "", signature.value)
|
|
5138
|
+
|| pasteLeftInInput(history, prompt, formatted))
|
|
5139
|
+
return "stranded";
|
|
5140
|
+
if (countOccurrences(history.replace(/\s+/g, ""), signature.value) > 0)
|
|
5141
|
+
return "submitted";
|
|
5142
|
+
}
|
|
5143
|
+
catch { /* missing history cannot establish a verdict */ }
|
|
5144
|
+
return "unknown";
|
|
5145
|
+
}
|
|
5146
|
+
/** Observe the Kiro paste outside paneWriteLock; a long agent turn must not block dialog handling. */
|
|
5147
|
+
async observeKiroSubmission(formatted, pending, status, verdict, cancelled) {
|
|
5148
|
+
const deadline = Date.now() + KIRO_SUBMISSION_OBSERVE_MS;
|
|
5149
|
+
let retriedStrand = false;
|
|
5150
|
+
let unreadable = 0;
|
|
5151
|
+
const current = () => !cancelled()
|
|
5152
|
+
&& !this.spawning
|
|
5153
|
+
&& this.spawnGeneration === pending.spawnGeneration
|
|
5154
|
+
&& this.getWindowId() === pending.windowId
|
|
5155
|
+
&& this.tmux?.getWindowId() === pending.windowId;
|
|
5156
|
+
for (;;) {
|
|
5157
|
+
if (!current())
|
|
5158
|
+
return false;
|
|
5159
|
+
const evidence = await this.kiroSubmissionEvidence(pending.windowId, formatted, pending.signature);
|
|
5160
|
+
if (!current())
|
|
5161
|
+
return false;
|
|
5162
|
+
if (evidence === "submitted") {
|
|
5163
|
+
if (status)
|
|
5164
|
+
this.emit("message_confirmed", status);
|
|
5165
|
+
return true;
|
|
5166
|
+
}
|
|
5167
|
+
unreadable = evidence === "unreadable" ? unreadable + 1 : 0;
|
|
5168
|
+
if (unreadable >= KIRO_SUBMISSION_UNREADABLE_POLLS) {
|
|
5169
|
+
return this.failDelivery(verdict, status, "post-submit-proof", "pane-unreadable");
|
|
5170
|
+
}
|
|
5171
|
+
if (evidence === "stranded" && await this.isPaneReadyForDelivery(pending.windowId)) {
|
|
5172
|
+
if (!current())
|
|
5173
|
+
return false;
|
|
5174
|
+
if (retriedStrand)
|
|
5175
|
+
return this.failDelivery(verdict, status, "stranded-text-retry", "stranded");
|
|
5176
|
+
const retry = await this.paneWriteLock.run(async () => {
|
|
5177
|
+
if (!current())
|
|
5178
|
+
return "cancelled";
|
|
5179
|
+
const fresh = await this.kiroSubmissionEvidence(pending.windowId, formatted, pending.signature);
|
|
5180
|
+
if (!current())
|
|
5181
|
+
return "cancelled";
|
|
5182
|
+
if (fresh === "submitted")
|
|
5183
|
+
return "submitted";
|
|
5184
|
+
if (fresh !== "stranded" || !(await this.isPaneReadyForDelivery(pending.windowId)))
|
|
5185
|
+
return "changed";
|
|
5186
|
+
if (!current())
|
|
5187
|
+
return "cancelled";
|
|
5188
|
+
const sent = await this.sendDeliveryEnter("stranded-text-retry", current);
|
|
5189
|
+
if (!current())
|
|
5190
|
+
return "cancelled";
|
|
5191
|
+
return sent ? "sent" : "failed";
|
|
5192
|
+
});
|
|
5193
|
+
if (!current())
|
|
5194
|
+
return false;
|
|
5195
|
+
if (retry === "cancelled")
|
|
5196
|
+
return false;
|
|
5197
|
+
if (retry === "submitted") {
|
|
5198
|
+
if (status)
|
|
5199
|
+
this.emit("message_confirmed", status);
|
|
5200
|
+
return true;
|
|
5201
|
+
}
|
|
5202
|
+
if (retry === "failed")
|
|
5203
|
+
return this.failDelivery(verdict, status, "stranded-text-retry", "tmux-send-keys-failed");
|
|
5204
|
+
if (retry === "sent") {
|
|
5205
|
+
retriedStrand = true;
|
|
5206
|
+
await new Promise(r => setTimeout(r, POST_ENTER_PROOF_WINDOW_MS));
|
|
5207
|
+
continue;
|
|
5208
|
+
}
|
|
5209
|
+
}
|
|
5210
|
+
const alive = await this.tmux?.isWindowAlive().catch(() => false);
|
|
5211
|
+
if (!current())
|
|
5212
|
+
return false;
|
|
5213
|
+
if (!alive)
|
|
5214
|
+
return this.failDelivery(verdict, status, "post-submit-proof", "window-gone");
|
|
5215
|
+
if (Date.now() >= deadline) {
|
|
5216
|
+
// A live, busy pane and no positive strand are still ambiguous. The
|
|
5217
|
+
// hang detector owns hung CLI recovery; elapsed time alone is not ❌.
|
|
5218
|
+
this.logger.warn("Kiro submission remains unproven; keeping the delivery queued without a false failure");
|
|
5219
|
+
verdict.phase = "post-submit-proof";
|
|
5220
|
+
verdict.proof = "unproven";
|
|
5221
|
+
return false;
|
|
5222
|
+
}
|
|
5223
|
+
await new Promise(r => setTimeout(r, KIRO_SUBMISSION_POLL_MS));
|
|
5224
|
+
}
|
|
5225
|
+
}
|
|
4819
5226
|
/**
|
|
4820
5227
|
* While the CLI is parked on a fatal startup screen (see fatalStartupBlocked),
|
|
4821
5228
|
* no pane write may happen: paste+Enter into the corrupt-config modal would
|
|
@@ -4828,7 +5235,7 @@ export class Daemon extends EventEmitter {
|
|
|
4828
5235
|
if (!this.fatalStartupBlocked)
|
|
4829
5236
|
return false;
|
|
4830
5237
|
this.logger.error("Delivery refused — CLI is parked on a fatal startup screen");
|
|
4831
|
-
this.failDelivery(verdict, status);
|
|
5238
|
+
this.failDelivery(verdict, status, "fatal-startup", "blocked-before-write");
|
|
4832
5239
|
return true;
|
|
4833
5240
|
}
|
|
4834
5241
|
/**
|
|
@@ -4895,9 +5302,13 @@ export class Daemon extends EventEmitter {
|
|
|
4895
5302
|
* precisely so the lock's scope is visible at the call site rather than being
|
|
4896
5303
|
* an invariant maintained by comments.
|
|
4897
5304
|
*/
|
|
4898
|
-
async sendDeliveryEnter(phase) {
|
|
5305
|
+
async sendDeliveryEnter(phase, stillCurrent) {
|
|
4899
5306
|
if (!(await this.waitForInputTransientToClear(phase)))
|
|
4900
5307
|
return false;
|
|
5308
|
+
// A recovery Enter may have waited for a transient while cancel or spawn
|
|
5309
|
+
// replaced the delivery. Check at the last point before the tmux write.
|
|
5310
|
+
if (stillCurrent && !stillCurrent())
|
|
5311
|
+
return false;
|
|
4901
5312
|
const sent = await this.tmux.sendSpecialKey("Enter");
|
|
4902
5313
|
if (!sent) {
|
|
4903
5314
|
this.logger.error({
|
|
@@ -4931,7 +5342,7 @@ export class Daemon extends EventEmitter {
|
|
|
4931
5342
|
if (!recoverable) {
|
|
4932
5343
|
// A tmux error the window cannot be recovered from: the text will
|
|
4933
5344
|
// never reach this pane, so it is a verdict like the others.
|
|
4934
|
-
return this.failDelivery(verdict, status);
|
|
5345
|
+
return this.failDelivery(verdict, status, "paste", "non-retryable-tmux-error");
|
|
4935
5346
|
}
|
|
4936
5347
|
windowId = (await this.recoverWindow()) ?? windowId;
|
|
4937
5348
|
if (attempt < maxAttempts)
|
|
@@ -4961,7 +5372,7 @@ export class Daemon extends EventEmitter {
|
|
|
4961
5372
|
}
|
|
4962
5373
|
let enterAt = Date.now();
|
|
4963
5374
|
if (!(await this.sendDeliveryEnter("initial-submit"))) {
|
|
4964
|
-
return this.failDelivery(verdict, status);
|
|
5375
|
+
return this.failDelivery(verdict, status, "submit-enter", "tmux-send-keys-failed");
|
|
4965
5376
|
}
|
|
4966
5377
|
// Kiro's legacy TUI can swallow Enter while it is still processing a large
|
|
4967
5378
|
// paste — not only during the post-ready redraw (#479): on slower hosts it
|
|
@@ -5010,8 +5421,16 @@ export class Daemon extends EventEmitter {
|
|
|
5010
5421
|
// Our text is on the pane but cannot be shown to have left the input
|
|
5011
5422
|
// row — either the backend exposes no input row (only /steer reaches
|
|
5012
5423
|
// this path on such a backend) or the pre-paste pane was unreadable.
|
|
5013
|
-
//
|
|
5014
|
-
//
|
|
5424
|
+
// For Codex, this can also mean its current layout is not a trusted
|
|
5425
|
+
// input/queue frame. Do not turn that uncertainty into a false ✅.
|
|
5426
|
+
if (this.backend?.isDeliveryInputReadyPane) {
|
|
5427
|
+
this.logger.warn({ phase: "native-queue-proof", proof }, "Codex native-queue outcome uncertain — not re-pasting or confirming");
|
|
5428
|
+
verdict.phase = "native-queue-proof";
|
|
5429
|
+
verdict.proof = proof;
|
|
5430
|
+
return false;
|
|
5431
|
+
}
|
|
5432
|
+
// Re-pasting on a backend without structured Codex pane evidence
|
|
5433
|
+
// would risk duplication; retain its legacy best-effort behavior.
|
|
5015
5434
|
this.logger.warn("Paste reached the pane but could not be verified as submitted — accepting without proof");
|
|
5016
5435
|
if (status)
|
|
5017
5436
|
this.emit("message_confirmed", status); // ✅ (best-effort)
|
|
@@ -5044,9 +5463,14 @@ export class Daemon extends EventEmitter {
|
|
|
5044
5463
|
this.logger.warn("Message still in the input row after idle — submitting the existing text instead of pasting it again");
|
|
5045
5464
|
const strandedAt = Date.now();
|
|
5046
5465
|
if (!(await this.sendDeliveryEnter("native-queue-stranded-submit"))) {
|
|
5047
|
-
return this.failDelivery(verdict, status);
|
|
5466
|
+
return this.failDelivery(verdict, status, "native-queue-submit", "tmux-send-keys-failed");
|
|
5048
5467
|
}
|
|
5049
|
-
|
|
5468
|
+
// The recovery Enter can be accepted before Codex paints its new
|
|
5469
|
+
// transcript. One immediate capture is not a failure verdict: give
|
|
5470
|
+
// that echo/queue a bounded chance to appear, without re-pasting.
|
|
5471
|
+
const afterEnter = this.backend?.isDeliveryInputReadyPane
|
|
5472
|
+
? await this.lateCodexSubmissionProof(signature, pasteBaseline, true)
|
|
5473
|
+
: await this.confirmSubmitted(signature, pasteBaseline);
|
|
5050
5474
|
if (afterEnter === "submitted") {
|
|
5051
5475
|
if (status)
|
|
5052
5476
|
this.emit("message_confirmed", status); // ✅
|
|
@@ -5058,8 +5482,23 @@ export class Daemon extends EventEmitter {
|
|
|
5058
5482
|
// another turn's output — and treating that as proof re-confirms a
|
|
5059
5483
|
// message nobody submitted. Output is corroboration; text sitting in
|
|
5060
5484
|
// the input row is disqualifying, and disqualifying evidence wins.
|
|
5485
|
+
if (this.backend?.isDeliveryInputReadyPane && afterEnter !== "stranded") {
|
|
5486
|
+
this.logger.warn({ phase: "native-queue-submit", proof: afterEnter, strandedAt }, "Codex recovery Enter outcome uncertain — no hard failure or duplicate paste");
|
|
5487
|
+
verdict.phase = "native-queue-submit";
|
|
5488
|
+
verdict.proof = afterEnter;
|
|
5489
|
+
return false;
|
|
5490
|
+
}
|
|
5061
5491
|
this.logger.error({ afterEnter, strandedAt }, "Stranded message could not be submitted by Enter");
|
|
5062
|
-
return this.failDelivery(verdict, status);
|
|
5492
|
+
return this.failDelivery(verdict, status, "native-queue-submit", afterEnter);
|
|
5493
|
+
}
|
|
5494
|
+
if (this.backend?.isDeliveryInputReadyPane) {
|
|
5495
|
+
// In Codex a missing viewport echo is inconclusive, not a proof of
|
|
5496
|
+
// loss. Another paste could run the same request twice. Leave the
|
|
5497
|
+
// already-pasted delivery at 👀 and let the next observation decide.
|
|
5498
|
+
this.logger.warn({ phase: "native-queue-proof", proof: settled }, "Codex native-queue outcome uncertain — not re-pasting");
|
|
5499
|
+
verdict.phase = "native-queue-proof";
|
|
5500
|
+
verdict.proof = settled;
|
|
5501
|
+
return false;
|
|
5063
5502
|
}
|
|
5064
5503
|
// "unproven": nothing of ours is on screen — the paste itself was lost,
|
|
5065
5504
|
// so pasting it again cannot duplicate anything.
|
|
@@ -5068,12 +5507,12 @@ export class Daemon extends EventEmitter {
|
|
|
5068
5507
|
this.logger.error({
|
|
5069
5508
|
tmuxError: this.tmux.getLastPasteError?.() ?? "unknown tmux paste failure",
|
|
5070
5509
|
}, "Idle-gated redelivery paste failed after native-queue silent loss");
|
|
5071
|
-
return this.failDelivery(verdict, status);
|
|
5510
|
+
return this.failDelivery(verdict, status, "native-queue-redelivery", "tmux-paste-failed");
|
|
5072
5511
|
}
|
|
5073
5512
|
await new Promise(r => setTimeout(r, NORMAL_ENTER_SETTLE_MS));
|
|
5074
5513
|
const retryAt = Date.now();
|
|
5075
5514
|
if (!(await this.sendDeliveryEnter("native-queue-idle-redelivery"))) {
|
|
5076
|
-
return this.failDelivery(verdict, status);
|
|
5515
|
+
return this.failDelivery(verdict, status, "native-queue-redelivery", "tmux-send-keys-failed");
|
|
5077
5516
|
}
|
|
5078
5517
|
if (windowId && this.controlClient) {
|
|
5079
5518
|
if (await this.confirmAfterEnter(windowId, retryAt, signature, pasteBaseline, "native-queue-idle-redelivery-retry")) {
|
|
@@ -5088,21 +5527,31 @@ export class Daemon extends EventEmitter {
|
|
|
5088
5527
|
return true;
|
|
5089
5528
|
}
|
|
5090
5529
|
this.logger.error("Idle-gated redelivery also failed after native-queue silent loss");
|
|
5091
|
-
return this.failDelivery(verdict, status);
|
|
5530
|
+
return this.failDelivery(verdict, status, "native-queue-redelivery", "not-submitted");
|
|
5092
5531
|
}
|
|
5093
5532
|
if (windowId && this.controlClient && this.backend?.dropsEnterWhileBusy?.() === true) {
|
|
5094
5533
|
// F2: output after Enter is necessary but not sufficient — the paste
|
|
5095
5534
|
// must also have LEFT the input row.
|
|
5096
5535
|
let submitted = await this.confirmSubmittedAfterEnter(windowId, enterAt, formatted);
|
|
5536
|
+
if (!submitted && signature.unique) {
|
|
5537
|
+
submitted = await this.kiroSubmissionEvidence(windowId, formatted, signature) === "submitted";
|
|
5538
|
+
if (!submitted) {
|
|
5539
|
+
// No output edge is not proof of a lost message. Leave the pane
|
|
5540
|
+
// lock before waiting through a long Kiro turn, so runtime dialog
|
|
5541
|
+
// handling can still make progress.
|
|
5542
|
+
if (status)
|
|
5543
|
+
this.emit("message_queued", status);
|
|
5544
|
+
return { kind: "kiro-pending", windowId, spawnGeneration: this.spawnGeneration, signature };
|
|
5545
|
+
}
|
|
5546
|
+
}
|
|
5097
5547
|
if (!submitted) {
|
|
5098
|
-
//
|
|
5099
|
-
//
|
|
5100
|
-
// bounded, since we hold the pane lock here.
|
|
5548
|
+
// Legacy system pastes have no unique envelope id. Preserve their
|
|
5549
|
+
// existing bounded retry until they can be tied to a specific turn.
|
|
5101
5550
|
this.logger.warn("Message not submitted after Enter — waiting for the prompt, then re-sending Enter once");
|
|
5102
5551
|
const promptBack = await this.waitForPaneReadyForDelivery(windowId, STRANDED_RETRY_READY_WAIT_MS);
|
|
5103
5552
|
const retryAt = Date.now();
|
|
5104
5553
|
if (promptBack && !(await this.sendDeliveryEnter("stranded-text-retry"))) {
|
|
5105
|
-
return this.failDelivery(verdict, status);
|
|
5554
|
+
return this.failDelivery(verdict, status, "stranded-text-retry", "tmux-send-keys-failed");
|
|
5106
5555
|
}
|
|
5107
5556
|
submitted = promptBack && await this.confirmSubmittedAfterEnter(windowId, retryAt, formatted);
|
|
5108
5557
|
}
|
|
@@ -5112,7 +5561,7 @@ export class Daemon extends EventEmitter {
|
|
|
5112
5561
|
}
|
|
5113
5562
|
else {
|
|
5114
5563
|
this.logger.error("Message pasted but never submitted (text still in the input row after Enter retry)");
|
|
5115
|
-
return this.failDelivery(verdict, status);
|
|
5564
|
+
return this.failDelivery(verdict, status, "stranded-text-retry", "stranded");
|
|
5116
5565
|
}
|
|
5117
5566
|
}
|
|
5118
5567
|
else if (windowId && this.controlClient) {
|
|
@@ -5132,12 +5581,26 @@ export class Daemon extends EventEmitter {
|
|
|
5132
5581
|
this.emit("message_confirmed", status); // ✅
|
|
5133
5582
|
}
|
|
5134
5583
|
else {
|
|
5135
|
-
|
|
5136
|
-
|
|
5137
|
-
|
|
5138
|
-
|
|
5139
|
-
|
|
5140
|
-
|
|
5584
|
+
const proof = this.backend?.isDeliveryInputReadyPane
|
|
5585
|
+
? await this.lateCodexSubmissionProof(signature, pasteBaseline)
|
|
5586
|
+
: await this.confirmSubmitted(signature, pasteBaseline);
|
|
5587
|
+
if (proof === "submitted") {
|
|
5588
|
+
if (status)
|
|
5589
|
+
this.emit("message_confirmed", status);
|
|
5590
|
+
return true;
|
|
5591
|
+
}
|
|
5592
|
+
if (this.backend?.isDeliveryInputReadyPane && proof !== "stranded") {
|
|
5593
|
+
// This is the observed #910 race: the CLI processed the message
|
|
5594
|
+
// although its echo had not appeared within the proof window. A
|
|
5595
|
+
// missing viewport signature cannot establish non-delivery; keep
|
|
5596
|
+
// 👀 and never emit the sender's hard ❌ or re-paste blindly.
|
|
5597
|
+
this.logger.warn({ phase: "post-submit-proof", proof }, "Codex delivery outcome uncertain — no hard failure or duplicate paste");
|
|
5598
|
+
verdict.phase = "post-submit-proof";
|
|
5599
|
+
verdict.proof = proof;
|
|
5600
|
+
return false;
|
|
5601
|
+
}
|
|
5602
|
+
this.logger.error({ phase: "post-submit-proof", proof }, "Message remains unsubmitted after Enter retry");
|
|
5603
|
+
return this.failDelivery(verdict, status, "post-submit-proof", proof);
|
|
5141
5604
|
}
|
|
5142
5605
|
}
|
|
5143
5606
|
else {
|
|
@@ -5150,7 +5613,7 @@ export class Daemon extends EventEmitter {
|
|
|
5150
5613
|
return true;
|
|
5151
5614
|
}
|
|
5152
5615
|
this.logger.error("Message delivery failed after retries — window not ready");
|
|
5153
|
-
return this.failDelivery(verdict, status);
|
|
5616
|
+
return this.failDelivery(verdict, status, "paste", "retry-budget-exhausted");
|
|
5154
5617
|
}
|
|
5155
5618
|
/**
|
|
5156
5619
|
* The single place a pasted message is judged submitted, shared by the
|
|
@@ -5208,6 +5671,42 @@ export class Daemon extends EventEmitter {
|
|
|
5208
5671
|
}
|
|
5209
5672
|
return busy;
|
|
5210
5673
|
}
|
|
5674
|
+
if (this.backend?.isDeliveryInputReadyPane) {
|
|
5675
|
+
// Codex's first post-wake redraw may hide the echo for a few seconds.
|
|
5676
|
+
// Absence from a viewport is NOT proof the paste was lost, so only a
|
|
5677
|
+
// positively identified strand authorizes another Enter. Never re-paste
|
|
5678
|
+
// here: the CLI may already be processing the unique message_id.
|
|
5679
|
+
let proof = "unproven";
|
|
5680
|
+
const firstDeadline = Date.now() + POST_ENTER_PROOF_WINDOW_MS;
|
|
5681
|
+
for (;;) {
|
|
5682
|
+
proof = await this.confirmSubmitted(signature, baseline);
|
|
5683
|
+
if (proof === "submitted")
|
|
5684
|
+
return true;
|
|
5685
|
+
if (proof === "stranded" || Date.now() >= firstDeadline)
|
|
5686
|
+
break;
|
|
5687
|
+
await new Promise(r => setTimeout(r, POST_ENTER_PROOF_POLL_MS));
|
|
5688
|
+
}
|
|
5689
|
+
if (proof !== "stranded")
|
|
5690
|
+
return false;
|
|
5691
|
+
if (!(await this.waitForPaneReadyForDelivery(windowId, STRANDED_RETRY_READY_WAIT_MS)))
|
|
5692
|
+
return false;
|
|
5693
|
+
proof = await this.confirmSubmitted(signature, baseline);
|
|
5694
|
+
if (proof === "submitted")
|
|
5695
|
+
return true;
|
|
5696
|
+
if (proof !== "stranded")
|
|
5697
|
+
return false;
|
|
5698
|
+
if (!(await this.sendDeliveryEnter(retryPhase)))
|
|
5699
|
+
return false;
|
|
5700
|
+
const retryDeadline = Date.now() + POST_ENTER_PROOF_WINDOW_MS;
|
|
5701
|
+
for (;;) {
|
|
5702
|
+
proof = await this.confirmSubmitted(signature, baseline);
|
|
5703
|
+
if (proof === "submitted")
|
|
5704
|
+
return true;
|
|
5705
|
+
if (Date.now() >= retryDeadline)
|
|
5706
|
+
return false;
|
|
5707
|
+
await new Promise(r => setTimeout(r, POST_ENTER_PROOF_POLL_MS));
|
|
5708
|
+
}
|
|
5709
|
+
}
|
|
5211
5710
|
if (await this.confirmSubmitted(signature, baseline) === "submitted")
|
|
5212
5711
|
return true;
|
|
5213
5712
|
// Retry once the prompt is back — an Enter sent into a mid-redraw TUI is
|
|
@@ -5257,6 +5756,14 @@ export class Daemon extends EventEmitter {
|
|
|
5257
5756
|
: "unproven";
|
|
5258
5757
|
}
|
|
5259
5758
|
const after = this.paneEvidence(pane, signature);
|
|
5759
|
+
// A Codex layout without its current input/footer pair may show a quoted
|
|
5760
|
+
// or historical `›` row. Unless its active Working banner corroborates a
|
|
5761
|
+
// real turn, text on that unknown screen cannot prove it left stdin.
|
|
5762
|
+
if (this.backend?.isDeliveryInputReadyPane
|
|
5763
|
+
&& !this.backend.isDeliveryInputReadyPane(pane)
|
|
5764
|
+
&& !this.backend.getBusyPattern?.()?.test(pane)) {
|
|
5765
|
+
return after.payload > 0 ? "unverifiable" : "unproven";
|
|
5766
|
+
}
|
|
5260
5767
|
// 1. Disqualifying evidence, checked FIRST and never overridden by the
|
|
5261
5768
|
// corroborating evidence below: our text is sitting in the input row, so
|
|
5262
5769
|
// it was not submitted — whatever else is on screen.
|
|
@@ -5295,6 +5802,35 @@ export class Daemon extends EventEmitter {
|
|
|
5295
5802
|
this.logger.warn("Could not read the pane before pasting — this delivery cannot be verified either way");
|
|
5296
5803
|
return "unverifiable";
|
|
5297
5804
|
}
|
|
5805
|
+
/** Only positive echo/queue evidence may turn an ambiguous Codex write into ✅. */
|
|
5806
|
+
async lateCodexSubmissionProof(signature, baseline, waitThroughStranded = false) {
|
|
5807
|
+
const deadline = Date.now() + CODEX_LATE_PROOF_MS;
|
|
5808
|
+
let proof = "unproven";
|
|
5809
|
+
for (;;) {
|
|
5810
|
+
proof = await this.confirmSubmitted(signature, baseline);
|
|
5811
|
+
// Ordinary idle-path proof may return a strand immediately. After a
|
|
5812
|
+
// native-queue recovery Enter, though, it can be the previous frame
|
|
5813
|
+
// still on screen; only a strand that survives the bounded repaint
|
|
5814
|
+
// window proves that Enter was swallowed.
|
|
5815
|
+
if (proof === "submitted" || (proof === "stranded" && !waitThroughStranded))
|
|
5816
|
+
return proof;
|
|
5817
|
+
if (signature.unique && this.tmux?.capturePaneWithHistory) {
|
|
5818
|
+
try {
|
|
5819
|
+
const history = await this.tmux.capturePaneWithHistory(300);
|
|
5820
|
+
const seen = this.paneEvidence(history, signature);
|
|
5821
|
+
const known = !this.backend?.isDeliveryInputReadyPane
|
|
5822
|
+
|| this.backend.isDeliveryInputReadyPane(history)
|
|
5823
|
+
|| this.backend.getBusyPattern?.()?.test(history);
|
|
5824
|
+
if (known && seen.payload > 0 && !seen.strandedInput)
|
|
5825
|
+
return this.submittedProof();
|
|
5826
|
+
}
|
|
5827
|
+
catch { /* a failed history read proves neither delivery nor loss */ }
|
|
5828
|
+
}
|
|
5829
|
+
if (Date.now() >= deadline)
|
|
5830
|
+
return proof;
|
|
5831
|
+
await new Promise(r => setTimeout(r, POST_ENTER_PROOF_POLL_MS));
|
|
5832
|
+
}
|
|
5833
|
+
}
|
|
5298
5834
|
/**
|
|
5299
5835
|
* What the pane currently shows of a given message. Taken once before the
|
|
5300
5836
|
* paste and once after, so confirmSubmitted can require a NEW marker or a NEW
|
|
@@ -5310,7 +5846,8 @@ export class Daemon extends EventEmitter {
|
|
|
5310
5846
|
const pane = await this.tmux.capturePane();
|
|
5311
5847
|
const evidence = this.paneEvidence(pane, signature);
|
|
5312
5848
|
const prompt = this.backend?.getBottomReadyPattern?.();
|
|
5313
|
-
if (prompt &&
|
|
5849
|
+
if (prompt && (!this.backend?.isDeliveryInputReadyPane || this.backend.isDeliveryInputReadyPane(pane))
|
|
5850
|
+
&& strandedAgendMessageInInput(pane, prompt)) {
|
|
5314
5851
|
// Whatever we paste now lands after it, and one Enter submits both as
|
|
5315
5852
|
// a single message. Nothing here can undo that; saying so beats
|
|
5316
5853
|
// letting two messages silently merge.
|
|
@@ -5331,7 +5868,8 @@ export class Daemon extends EventEmitter {
|
|
|
5331
5868
|
paneEvidence(pane, signature) {
|
|
5332
5869
|
const marker = this.backend?.getQueuedInputMarker?.();
|
|
5333
5870
|
const prompt = this.backend?.getBottomReadyPattern?.();
|
|
5334
|
-
const input = prompt
|
|
5871
|
+
const input = prompt && (!this.backend?.isDeliveryInputReadyPane || this.backend.isDeliveryInputReadyPane(pane))
|
|
5872
|
+
? inputAreaText(pane, prompt) : null;
|
|
5335
5873
|
return {
|
|
5336
5874
|
queued: marker ? pane.split(/\r?\n/).filter(row => marker.test(row)).length : 0,
|
|
5337
5875
|
payload: countOccurrences(pane.replace(/\s+/g, ""), signature.value),
|
|
@@ -5382,6 +5920,13 @@ export class Daemon extends EventEmitter {
|
|
|
5382
5920
|
return false;
|
|
5383
5921
|
let proof = await this.confirmSubmitted(signature, baseline);
|
|
5384
5922
|
if (proof === "unverifiable") {
|
|
5923
|
+
if (this.backend?.isDeliveryInputReadyPane) {
|
|
5924
|
+
// A Codex screen without a structurally current input/footer pair is
|
|
5925
|
+
// not a successful snapshot restore. The old best-effort rule below
|
|
5926
|
+
// is only for backends that expose no readable input row at all.
|
|
5927
|
+
this.logger.warn({ label }, "Codex system paste could not be verified");
|
|
5928
|
+
return false;
|
|
5929
|
+
}
|
|
5385
5930
|
// No input row to read: this backend gets exactly what the old pasteText
|
|
5386
5931
|
// path gave it, including the unconditional second Enter for queue-less
|
|
5387
5932
|
// TUIs that swallow the first. Narrowing that to one Enter on the grounds
|
|
@@ -5983,26 +6528,33 @@ export class Daemon extends EventEmitter {
|
|
|
5983
6528
|
* Bounded, and called BEFORE the pane lock is taken — waiting on the spawn while
|
|
5984
6529
|
* holding the lock the spawn itself needs would deadlock.
|
|
5985
6530
|
*/
|
|
5986
|
-
|
|
6531
|
+
/**
|
|
6532
|
+
* Wait for an in-flight spawn to finish, up to the cap. Returns true when no
|
|
6533
|
+
* spawn is in flight afterwards; false when the cap expired with one still
|
|
6534
|
+
* running. A false return is NOT settled evidence — the caller must not
|
|
6535
|
+
* treat the current generation as a completed spawn.
|
|
6536
|
+
*/
|
|
6537
|
+
async waitForSpawnToSettle(capMs = SPAWN_SETTLE_MAX_WAIT_MS) {
|
|
5987
6538
|
const settled = this.spawnSettled;
|
|
5988
6539
|
if (!settled)
|
|
5989
|
-
return;
|
|
6540
|
+
return true;
|
|
5990
6541
|
this.logger.debug("Holding delivery until the CLI has finished starting up");
|
|
5991
6542
|
let timer;
|
|
5992
6543
|
const cap = new Promise(resolve => {
|
|
5993
|
-
timer = setTimeout(resolve,
|
|
6544
|
+
timer = setTimeout(() => resolve(false), capMs);
|
|
5994
6545
|
timer.unref?.();
|
|
5995
6546
|
});
|
|
5996
6547
|
try {
|
|
5997
|
-
await Promise.race([settled, cap]);
|
|
6548
|
+
const finished = await Promise.race([settled.then(() => true), cap]);
|
|
6549
|
+
if (!finished) {
|
|
6550
|
+
this.logger.warn("CLI still starting after the delivery hold — the pane is not settled evidence");
|
|
6551
|
+
}
|
|
6552
|
+
return finished;
|
|
5998
6553
|
}
|
|
5999
6554
|
finally {
|
|
6000
6555
|
if (timer)
|
|
6001
6556
|
clearTimeout(timer);
|
|
6002
6557
|
}
|
|
6003
|
-
if (this.spawning) {
|
|
6004
|
-
this.logger.warn("CLI still starting after the delivery hold — delivering anyway");
|
|
6005
|
-
}
|
|
6006
6558
|
}
|
|
6007
6559
|
/** Spawn a CLI window. Returns true if --resume was used successfully. */
|
|
6008
6560
|
/**
|
|
@@ -6093,7 +6645,7 @@ export class Daemon extends EventEmitter {
|
|
|
6093
6645
|
await this.failStartupIfBackendUnreachable();
|
|
6094
6646
|
if (this.backend.retriesResumeOnStartupFailure?.() !== false) {
|
|
6095
6647
|
this.logger.warn("Resume startup failed — retrying resume once before abandoning the session");
|
|
6096
|
-
await this.killProcessTree();
|
|
6648
|
+
await this.killProcessTree("SIGTERM", "spawn: clearing the previous CLI process");
|
|
6097
6649
|
await this.tmux.killWindow();
|
|
6098
6650
|
alive = await this.trySpawn(false, resumeBudget);
|
|
6099
6651
|
if (!alive) {
|
|
@@ -6119,7 +6671,7 @@ export class Daemon extends EventEmitter {
|
|
|
6119
6671
|
if (this.unprovenResumeFailures < Daemon.MAX_UNPROVEN_RESUME_FAILURES) {
|
|
6120
6672
|
// Keep the session and fail this attempt; the fleet retries with
|
|
6121
6673
|
// backoff, which is also how the backend-outage path behaves.
|
|
6122
|
-
await this.killProcessTree();
|
|
6674
|
+
await this.killProcessTree("SIGTERM", "spawn: clearing the previous CLI process");
|
|
6123
6675
|
await this.tmux.killWindow();
|
|
6124
6676
|
throw new Error(`CLI startup failed with a session to resume (attempt ${this.unprovenResumeFailures}/${Daemon.MAX_UNPROVEN_RESUME_FAILURES}) `
|
|
6125
6677
|
+ "— session kept, will retry");
|
|
@@ -6136,11 +6688,11 @@ export class Daemon extends EventEmitter {
|
|
|
6136
6688
|
// A fresh start that also failed retries once, as before. It never clears
|
|
6137
6689
|
// a session: nothing about a failed fresh launch says the stored
|
|
6138
6690
|
// conversation is unusable.
|
|
6139
|
-
await this.killProcessTree();
|
|
6691
|
+
await this.killProcessTree("SIGTERM", "spawn: clearing the previous CLI process");
|
|
6140
6692
|
await this.tmux.killWindow();
|
|
6141
6693
|
const retryAlive = await this.trySpawn(false, this.startupBudgetFor(false));
|
|
6142
6694
|
if (!retryAlive) {
|
|
6143
|
-
await this.killProcessTree();
|
|
6695
|
+
await this.killProcessTree("SIGTERM", "spawn: clearing the previous CLI process");
|
|
6144
6696
|
await this.tmux.killWindow();
|
|
6145
6697
|
throw new Error("CLI failed to start after retry");
|
|
6146
6698
|
}
|
|
@@ -6212,7 +6764,7 @@ export class Daemon extends EventEmitter {
|
|
|
6212
6764
|
if (!this.backendOutage?.isActive(this.backendKey()))
|
|
6213
6765
|
return;
|
|
6214
6766
|
this.logger.warn("Backend unreachable — keeping the session and failing startup for a delayed retry");
|
|
6215
|
-
await this.killProcessTree();
|
|
6767
|
+
await this.killProcessTree("SIGTERM", "startup: backend unreachable");
|
|
6216
6768
|
await this.tmux.killWindow();
|
|
6217
6769
|
throw new BackendUnreachableStartupError(this.backendKey());
|
|
6218
6770
|
}
|
|
@@ -6249,8 +6801,12 @@ export class Daemon extends EventEmitter {
|
|
|
6249
6801
|
this.startupAborted = true;
|
|
6250
6802
|
this.freezeRuntimeMonitors();
|
|
6251
6803
|
this.pendingIpcRequests.clear();
|
|
6804
|
+
if (this.museUsageRelay) {
|
|
6805
|
+
await this.museUsageRelay.stop().catch(() => { });
|
|
6806
|
+
this.museUsageRelay = null;
|
|
6807
|
+
}
|
|
6252
6808
|
try {
|
|
6253
|
-
await this.killProcessTree();
|
|
6809
|
+
await this.killProcessTree("SIGTERM", "startup aborted");
|
|
6254
6810
|
}
|
|
6255
6811
|
catch { /* nothing running */ }
|
|
6256
6812
|
if (this.tmux) {
|
|
@@ -6277,12 +6833,13 @@ export class Daemon extends EventEmitter {
|
|
|
6277
6833
|
}
|
|
6278
6834
|
}
|
|
6279
6835
|
/** Kill the entire process tree of the current tmux pane (CLI + MCP server). */
|
|
6280
|
-
async killProcessTree(signal = "SIGTERM") {
|
|
6836
|
+
async killProcessTree(signal = "SIGTERM", reason = "unspecified") {
|
|
6281
6837
|
if (!this.tmux)
|
|
6282
6838
|
return;
|
|
6283
6839
|
try {
|
|
6284
6840
|
const pid = await TmuxManager.getPanePid(this.tmuxSessionName, this.tmux.getWindowId());
|
|
6285
6841
|
if (pid) {
|
|
6842
|
+
this.noteStopRequest(signal, reason);
|
|
6286
6843
|
process.kill(-pid, signal);
|
|
6287
6844
|
this.logger.debug({ pid, signal }, "Killed process group");
|
|
6288
6845
|
}
|
|
@@ -6304,6 +6861,129 @@ export class Daemon extends EventEmitter {
|
|
|
6304
6861
|
reason: reuseWindow ? "wake" : this.lastSpawnAt > 0 ? "recovery" : "startup",
|
|
6305
6862
|
}, () => this.trySpawnInsideGate(reuseWindow, startupTimeoutMs));
|
|
6306
6863
|
}
|
|
6864
|
+
/**
|
|
6865
|
+
* One fresh pane capture judged as ready && !busy && !dialog. Transcript
|
|
6866
|
+
* silence is deliberately NOT the gate: its monitor is inert for muse, so a
|
|
6867
|
+
* quiet-but-working pane would read as idle and the respawn below would kill
|
|
6868
|
+
* a live turn. Fail closed: an unreadable pane is never idle.
|
|
6869
|
+
*/
|
|
6870
|
+
async isPaneAuthoritativelyIdle() {
|
|
6871
|
+
const backend = this.backend;
|
|
6872
|
+
if (!this.tmux || !backend)
|
|
6873
|
+
return false;
|
|
6874
|
+
let pane;
|
|
6875
|
+
try {
|
|
6876
|
+
pane = await this.tmux.capturePane();
|
|
6877
|
+
}
|
|
6878
|
+
catch {
|
|
6879
|
+
return false;
|
|
6880
|
+
}
|
|
6881
|
+
if (backend.getBusyPattern?.()?.test(pane))
|
|
6882
|
+
return false;
|
|
6883
|
+
for (const dialog of this.deliveryBlockingDialogs()) {
|
|
6884
|
+
if (Daemon.dialogMatches(dialog, pane))
|
|
6885
|
+
return false;
|
|
6886
|
+
}
|
|
6887
|
+
if (!backend.getReadyPattern().test(pane))
|
|
6888
|
+
return false;
|
|
6889
|
+
return true;
|
|
6890
|
+
}
|
|
6891
|
+
/**
|
|
6892
|
+
* Poll the authoritative pane check until it reports idle or the budget runs
|
|
6893
|
+
* out. Each poll is a fresh capture, so a turn that ends mid-wait is seen.
|
|
6894
|
+
*/
|
|
6895
|
+
async waitForAuthoritativePaneIdle(budgetMs = MUSE_DIRECT_RESUME_IDLE_BUDGET_MS, pollMs = MUSE_DIRECT_RESUME_IDLE_POLL_MS) {
|
|
6896
|
+
const deadline = Date.now() + budgetMs;
|
|
6897
|
+
for (;;) {
|
|
6898
|
+
if (await this.isPaneAuthoritativelyIdle())
|
|
6899
|
+
return true;
|
|
6900
|
+
if (Date.now() >= deadline)
|
|
6901
|
+
return false;
|
|
6902
|
+
await new Promise(r => setTimeout(r, pollMs));
|
|
6903
|
+
}
|
|
6904
|
+
}
|
|
6905
|
+
/**
|
|
6906
|
+
* A relay crash must not make the daemon kill a live Muse process. The
|
|
6907
|
+
* supervisor normally reclaims the same port; when that bounded recovery
|
|
6908
|
+
* fails, move Muse back to a direct connection once it is authoritatively
|
|
6909
|
+
* idle (session preserved via the existing resume path) and mark usage
|
|
6910
|
+
* unavailable. The fallback flag also covers every later natural spawn.
|
|
6911
|
+
* Never throws: the floor is the old behavior (flag set, relay stopped,
|
|
6912
|
+
* next restart direct).
|
|
6913
|
+
*/
|
|
6914
|
+
switchMuseToDirect(idleBudgetMs = MUSE_DIRECT_RESUME_IDLE_BUDGET_MS, idlePollMs = MUSE_DIRECT_RESUME_IDLE_POLL_MS) {
|
|
6915
|
+
if (this.museRelayFallbackInFlight || this.startupAborted || this.backend?.binaryName !== "muse") {
|
|
6916
|
+
return this.museRelayFallbackInFlight ?? Promise.resolve();
|
|
6917
|
+
}
|
|
6918
|
+
this.museRelayFallbackInFlight = (async () => {
|
|
6919
|
+
this.museRelayFallback = true;
|
|
6920
|
+
this.logger.warn("Muse usage relay could not recover — usage is unavailable until Muse restarts direct");
|
|
6921
|
+
try {
|
|
6922
|
+
if (this.museUsageRelay) {
|
|
6923
|
+
await this.museUsageRelay.stop().catch(() => { });
|
|
6924
|
+
this.museUsageRelay = null;
|
|
6925
|
+
}
|
|
6926
|
+
}
|
|
6927
|
+
catch (err) {
|
|
6928
|
+
this.logger.warn({ reason: err instanceof Error ? err.message : "relay stop failed" }, "Muse usage relay cleanup failed");
|
|
6929
|
+
}
|
|
6930
|
+
// Say so where the operator is looking: without this, muse keeps running
|
|
6931
|
+
// with --base-url pointed at the dead relay port and every turn fails.
|
|
6932
|
+
this.emit("muse_relay_exhausted", { name: this.name });
|
|
6933
|
+
// An idle muse can be moved now; a busy, paused, or supervised-paused one
|
|
6934
|
+
// is left alone — pausing/wake respawns direct anyway via the fallback
|
|
6935
|
+
// flag, and interrupting a live turn (or fighting supervision) would be
|
|
6936
|
+
// worse than waiting for the next natural restart.
|
|
6937
|
+
if (this.isPaused || this.healthCheckPaused || this.getProcessStatus() === "stopped" || !this.tmux)
|
|
6938
|
+
return;
|
|
6939
|
+
const generation = this.spawnGeneration;
|
|
6940
|
+
const idle = await this.waitForAuthoritativePaneIdle(idleBudgetMs, idlePollMs);
|
|
6941
|
+
// Another spawn (wake/restart) completed while waiting — it owns the pane
|
|
6942
|
+
// now, and respawning here would kill its fresh process mid-turn.
|
|
6943
|
+
if (generation !== this.spawnGeneration)
|
|
6944
|
+
return;
|
|
6945
|
+
if (!idle)
|
|
6946
|
+
return;
|
|
6947
|
+
if (this.startupAborted || this.isPaused || this.healthCheckPaused || this.getProcessStatus() === "stopped")
|
|
6948
|
+
return;
|
|
6949
|
+
// Final re-verification under pane-write exclusion: no delivery can paste
|
|
6950
|
+
// a new turn between this verdict and the respawn it authorizes. The
|
|
6951
|
+
// spawn itself runs outside the lock (its dialog dismissal re-takes it).
|
|
6952
|
+
let claimed = false;
|
|
6953
|
+
await this.paneWriteLock.run(async () => {
|
|
6954
|
+
const stillIdle = await this.isPaneAuthoritativelyIdle();
|
|
6955
|
+
if (generation !== this.spawnGeneration)
|
|
6956
|
+
return;
|
|
6957
|
+
if (!stillIdle)
|
|
6958
|
+
return;
|
|
6959
|
+
// The final capture awaited above: re-check the full guards here, not
|
|
6960
|
+
// just generation and idle — pause needs no generation bump to take
|
|
6961
|
+
// effect, and respawning a freshly paused pane strands its state.
|
|
6962
|
+
if (this.startupAborted || this.isPaused || this.healthCheckPaused || this.getProcessStatus() === "stopped")
|
|
6963
|
+
return;
|
|
6964
|
+
this.beginSpawn();
|
|
6965
|
+
claimed = true;
|
|
6966
|
+
});
|
|
6967
|
+
if (!claimed)
|
|
6968
|
+
return;
|
|
6969
|
+
try {
|
|
6970
|
+
this.saveSessionId();
|
|
6971
|
+
const ready = await this.trySpawn(true, this.wakeBudgetMs(30_000));
|
|
6972
|
+
this.transcriptMonitor?.resetOffset();
|
|
6973
|
+
if (ready)
|
|
6974
|
+
this.logger.info("Muse resumed direct after relay exhaustion (session preserved)");
|
|
6975
|
+
else
|
|
6976
|
+
this.logger.warn("Muse direct resume after relay exhaustion did not become ready — next natural restart retries");
|
|
6977
|
+
}
|
|
6978
|
+
catch (err) {
|
|
6979
|
+
this.logger.warn({ reason: err instanceof Error ? err.message : String(err) }, "Muse direct resume after relay exhaustion failed — next natural restart retries");
|
|
6980
|
+
}
|
|
6981
|
+
finally {
|
|
6982
|
+
this.endSpawn();
|
|
6983
|
+
}
|
|
6984
|
+
})().finally(() => { this.museRelayFallbackInFlight = null; });
|
|
6985
|
+
return this.museRelayFallbackInFlight;
|
|
6986
|
+
}
|
|
6307
6987
|
/**
|
|
6308
6988
|
* Tell the agent its instructions moved, once the CLI is idle.
|
|
6309
6989
|
*
|
|
@@ -6439,7 +7119,30 @@ export class Daemon extends EventEmitter {
|
|
|
6439
7119
|
if (backendConfig.agentMode === "cli" && backendConfig.agentPort) {
|
|
6440
7120
|
envPrefix += ` AGEND_PORT=${backendConfig.agentPort}`;
|
|
6441
7121
|
}
|
|
6442
|
-
|
|
7122
|
+
// Muse's usage relay must be listening before its command is assembled.
|
|
7123
|
+
// Preparation is best-effort: if the loopback port cannot be bound, leave
|
|
7124
|
+
// museBaseUrl unset and let Muse connect directly so usage never blocks a
|
|
7125
|
+
// conversation.
|
|
7126
|
+
let launchConfig = backendConfig;
|
|
7127
|
+
if (this.backend?.binaryName === "muse" && !this.museRelayFallback) {
|
|
7128
|
+
try {
|
|
7129
|
+
this.museUsageRelay ??= new MuseUsageRelay({
|
|
7130
|
+
instanceDir: this.instanceDir,
|
|
7131
|
+
onUnavailable: () => { void this.switchMuseToDirect(); },
|
|
7132
|
+
});
|
|
7133
|
+
const baseUrl = await this.museUsageRelay.start();
|
|
7134
|
+
launchConfig = { ...backendConfig, museBaseUrl: baseUrl };
|
|
7135
|
+
}
|
|
7136
|
+
catch (err) {
|
|
7137
|
+
this.logger.warn({ reason: err instanceof Error ? err.message : "relay unavailable" }, "Muse usage relay unavailable — using direct connection");
|
|
7138
|
+
if (this.museUsageRelay) {
|
|
7139
|
+
await this.museUsageRelay.stop().catch(() => { });
|
|
7140
|
+
this.museUsageRelay = null;
|
|
7141
|
+
}
|
|
7142
|
+
clearMuseUsageSnapshot(this.instanceDir);
|
|
7143
|
+
}
|
|
7144
|
+
}
|
|
7145
|
+
const cmd = `${envPrefix} ` + this.backend.buildCommand(launchConfig);
|
|
6443
7146
|
// Ensure tmux session exists (may have been destroyed if all windows died)
|
|
6444
7147
|
await TmuxManager.ensureSession(this.tmuxSessionName);
|
|
6445
7148
|
if (this.stormWindow?.observeServerAlive(await TmuxManager.getServerPid(this.tmuxSessionName))) {
|
|
@@ -6553,6 +7256,10 @@ export class Daemon extends EventEmitter {
|
|
|
6553
7256
|
let lastTransient = null;
|
|
6554
7257
|
let captureFailures = 0;
|
|
6555
7258
|
let attempts = 0;
|
|
7259
|
+
// A safety-critical startup choice is one-shot. If the CLI has not
|
|
7260
|
+
// repainted after Enter, let the following hold-only entry report it
|
|
7261
|
+
// instead of sending a second Enter into a possibly changed screen.
|
|
7262
|
+
const attemptedSafetyChoices = new Set();
|
|
6556
7263
|
do {
|
|
6557
7264
|
attempts++;
|
|
6558
7265
|
let pane;
|
|
@@ -6589,6 +7296,11 @@ export class Daemon extends EventEmitter {
|
|
|
6589
7296
|
lastDialog = null;
|
|
6590
7297
|
for (const dialog of startupDialogs) {
|
|
6591
7298
|
if (Daemon.dialogMatches(dialog, pane)) {
|
|
7299
|
+
if (dialog.autoResolutionKey
|
|
7300
|
+
&& (attemptedSafetyChoices.has(dialog.autoResolutionKey)
|
|
7301
|
+
|| (this.autoResolvedDialogGeneration === this.spawnGeneration
|
|
7302
|
+
&& this.autoResolvedDialogKey === dialog.autoResolutionKey)))
|
|
7303
|
+
continue;
|
|
6592
7304
|
lastDialog = dialog;
|
|
6593
7305
|
cleanReadyPolls = 0;
|
|
6594
7306
|
// Start the parked clock for every delivery-blocking dialog, exact
|
|
@@ -6630,17 +7342,38 @@ export class Daemon extends EventEmitter {
|
|
|
6630
7342
|
// Restart is exactly when inbound messages pile up, and nothing gates
|
|
6631
7343
|
// delivery on `spawning`. Take the pane lock for the key sequence so a
|
|
6632
7344
|
// queued message cannot be pasted into a half-dismissed trust dialog.
|
|
6633
|
-
await this.paneWriteLock.run(async () => {
|
|
7345
|
+
const sent = await this.paneWriteLock.run(async () => {
|
|
7346
|
+
// The capture above may have gone stale while waiting for the
|
|
7347
|
+
// write lock. Trust/other safety prompts must still be the
|
|
7348
|
+
// CURRENT menu, with the same safe cursor, at the instant of
|
|
7349
|
+
// the key send. A changed pane falls through to the next scan.
|
|
7350
|
+
if (dialog.inputBlocked) {
|
|
7351
|
+
const currentPane = await this.tmux.capturePane();
|
|
7352
|
+
if (!Daemon.dialogMatches(dialog, currentPane))
|
|
7353
|
+
return false;
|
|
7354
|
+
}
|
|
7355
|
+
if (dialog.autoResolutionKey) {
|
|
7356
|
+
attemptedSafetyChoices.add(dialog.autoResolutionKey);
|
|
7357
|
+
this.autoResolvedDialogGeneration = this.spawnGeneration;
|
|
7358
|
+
this.autoResolvedDialogKey = dialog.autoResolutionKey;
|
|
7359
|
+
}
|
|
6634
7360
|
for (const key of dialog.keys) {
|
|
6635
7361
|
if (key === "Up" || key === "Down" || key === "Enter" || key === "Escape") {
|
|
6636
|
-
await this.tmux.sendSpecialKey(key)
|
|
7362
|
+
if (!await this.tmux.sendSpecialKey(key))
|
|
7363
|
+
return false;
|
|
6637
7364
|
}
|
|
6638
7365
|
else {
|
|
6639
|
-
await this.tmux.sendKeys(key)
|
|
7366
|
+
if (!await this.tmux.sendKeys(key))
|
|
7367
|
+
return false;
|
|
6640
7368
|
}
|
|
6641
7369
|
await new Promise(r => setTimeout(r, 200));
|
|
6642
7370
|
}
|
|
7371
|
+
return true;
|
|
6643
7372
|
});
|
|
7373
|
+
if (!sent) {
|
|
7374
|
+
matched = true;
|
|
7375
|
+
break;
|
|
7376
|
+
}
|
|
6644
7377
|
// Wait for next screen to render — bounded by what is left of the budget.
|
|
6645
7378
|
const renderWait = Math.max(0, Math.min(10_000, remaining()));
|
|
6646
7379
|
if (this.controlClient) {
|
|
@@ -6666,7 +7399,16 @@ export class Daemon extends EventEmitter {
|
|
|
6666
7399
|
// CLI is ready (pattern defined by each backend). Require it on two
|
|
6667
7400
|
// consecutive polls: the first ready frame is also the moment a late
|
|
6668
7401
|
// dialog is about to be painted over it.
|
|
6669
|
-
|
|
7402
|
+
const readyPatternMatches = this.backend.getReadyPattern().test(pane);
|
|
7403
|
+
// Codex's broad prompt/footer regex is necessary for inline layouts,
|
|
7404
|
+
// but startup must also prove that the current composer owns the
|
|
7405
|
+
// footer. This matters when a usage-limit picker is being resolved:
|
|
7406
|
+
// its old prompt can remain in scrollback while the reserve session is
|
|
7407
|
+
// still transitioning. Once the live composer is back, do not keep the
|
|
7408
|
+
// scan parked on the old limit text.
|
|
7409
|
+
const codexReady = this.backend.binaryName !== "codex"
|
|
7410
|
+
|| this.isCodexLivePaneSnapshot(pane);
|
|
7411
|
+
if (readyPatternMatches && codexReady) {
|
|
6670
7412
|
cleanReadyPolls++;
|
|
6671
7413
|
if (cleanReadyPolls >= 2 || remaining() <= 0) {
|
|
6672
7414
|
// A real ready prompt (with no fatal dialog on screen — those are
|