@songsid/agend 2.1.6-beta.11 → 2.1.6-beta.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -0
- package/dist/backend/codex-session.d.ts +45 -0
- package/dist/backend/codex-session.js +202 -0
- package/dist/backend/codex-session.js.map +1 -0
- package/dist/backend/codex.d.ts +48 -12
- package/dist/backend/codex.js +700 -65
- package/dist/backend/codex.js.map +1 -1
- package/dist/backend/kiro.d.ts +9 -0
- package/dist/backend/kiro.js +129 -0
- package/dist/backend/kiro.js.map +1 -1
- package/dist/backend/muse.d.ts +9 -4
- package/dist/backend/muse.js +89 -34
- package/dist/backend/muse.js.map +1 -1
- package/dist/backend/types.d.ts +25 -1
- package/dist/backend/types.js.map +1 -1
- package/dist/cli.js +23 -1
- package/dist/cli.js.map +1 -1
- package/dist/daemon.d.ts +41 -7
- package/dist/daemon.js +546 -91
- package/dist/daemon.js.map +1 -1
- package/dist/fleet-manager.d.ts +11 -1
- package/dist/fleet-manager.js +57 -3
- package/dist/fleet-manager.js.map +1 -1
- package/dist/instance-lifecycle.d.ts +7 -0
- package/dist/instance-lifecycle.js +126 -2
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/locale.js +6 -0
- package/dist/locale.js.map +1 -1
- package/dist/muse-usage-relay.js +14 -2
- package/dist/muse-usage-relay.js.map +1 -1
- package/dist/tmux-manager.d.ts +2 -0
- package/dist/tmux-manager.js +20 -0
- package/dist/tmux-manager.js.map +1 -1
- package/dist/usage/providers.js +10 -2
- package/dist/usage/providers.js.map +1 -1
- package/dist/usage/usage-api.d.ts +0 -12
- package/dist/usage/usage-api.js +24 -3
- package/dist/usage/usage-api.js.map +1 -1
- package/package.json +2 -1
package/dist/daemon.js
CHANGED
|
@@ -30,6 +30,8 @@ import { bottomRowIsReady, inputAreaText, inputShowsPastedText, pastedTextSignat
|
|
|
30
30
|
import { TurnReplyGuard } from "./turn-reply-guard.js";
|
|
31
31
|
import { t } from "./locale.js";
|
|
32
32
|
import { MuseUsageRelay, clearMuseUsageSnapshot } from "./muse-usage-relay.js";
|
|
33
|
+
import { CodexResumeConflictError, CodexResumeIdentityError, CodexResumeUnavailableError } from "./backend/codex-session.js";
|
|
34
|
+
import { codexResumeLockVisible } from "./backend/codex.js";
|
|
33
35
|
const __filename = fileURLToPath(import.meta.url);
|
|
34
36
|
const __dirname = dirname(__filename);
|
|
35
37
|
// Tool routing sets — module-level to avoid re-creation on every handleToolCall
|
|
@@ -399,6 +401,8 @@ export class PendingWorkTracker {
|
|
|
399
401
|
const NORMAL_ENTER_SETTLE_MS = 500;
|
|
400
402
|
/** How long to keep looking for proof after a retry Enter before failing it. */
|
|
401
403
|
const POST_ENTER_PROOF_WINDOW_MS = 3_000;
|
|
404
|
+
/** A final bounded observation before an ambiguous Codex submit can be classified. */
|
|
405
|
+
const CODEX_LATE_PROOF_MS = 5_000;
|
|
402
406
|
const POST_ENTER_PROOF_POLL_MS = 250;
|
|
403
407
|
/** Attempts to read the pre-paste pane before a delivery gives up on a baseline. */
|
|
404
408
|
const BASELINE_CAPTURE_ATTEMPTS = 3;
|
|
@@ -423,6 +427,16 @@ const STARTUP_DIALOG_BUDGET_MS = 30_000;
|
|
|
423
427
|
const DIALOG_PARKED_NOTIFY_MS = 60_000;
|
|
424
428
|
/** How many "dialog painted just before the write → wait → retry" rounds a delivery tolerates. */
|
|
425
429
|
const LATE_DIALOG_WRITE_ROUNDS = 3;
|
|
430
|
+
/** How many times a delivery redoes itself when a spawn starts between its settle wait and its pane write. */
|
|
431
|
+
const DELIVERY_SPAWN_RACE_MAX_ROUNDS = 3;
|
|
432
|
+
/**
|
|
433
|
+
* After relay exhaustion, how long to wait for an authoritatively idle pane
|
|
434
|
+
* before giving up the immediate resume-direct (the fallback flag and the
|
|
435
|
+
* notification already cover the next natural restart). Muse turns can run
|
|
436
|
+
* for many minutes; polling keeps each check a fresh capture.
|
|
437
|
+
*/
|
|
438
|
+
const MUSE_DIRECT_RESUME_IDLE_BUDGET_MS = 10 * 60_000;
|
|
439
|
+
const MUSE_DIRECT_RESUME_IDLE_POLL_MS = 5_000;
|
|
426
440
|
/**
|
|
427
441
|
* Startup failed because the CLI's backend is unreachable (see backend-outage.ts).
|
|
428
442
|
* The session-id is deliberately KEPT; the fleet schedules a delayed retry.
|
|
@@ -568,7 +582,7 @@ export async function waitForPasteSettle(client, windowId, pasteStartedAt, fallb
|
|
|
568
582
|
}
|
|
569
583
|
}
|
|
570
584
|
/** Redact likely credentials and control sequences before pane text reaches logs. */
|
|
571
|
-
export function sanitizePaneTail(pane, lineCount = 5) {
|
|
585
|
+
export function sanitizePaneTail(pane, lineCount = 5, excludeLine) {
|
|
572
586
|
const secretAssignment = /\b(token|secret|password|passwd|api[_-]?key|authorization)\b\s*[:=]\s*\S+/gi;
|
|
573
587
|
const bearer = /\bBearer\s+\S+/gi;
|
|
574
588
|
const knownToken = /\b(?:sk-[A-Za-z0-9_-]+|ghp_[A-Za-z0-9]+|github_pat_[A-Za-z0-9_]+|AKIA[A-Z0-9]{16})\b/g;
|
|
@@ -581,6 +595,10 @@ export function sanitizePaneTail(pane, lineCount = 5) {
|
|
|
581
595
|
lines.pop();
|
|
582
596
|
return lines
|
|
583
597
|
.slice(-lineCount)
|
|
598
|
+
// The proxy-reply chrome predicate sees only control-normalized lines,
|
|
599
|
+
// but must run BEFORE redaction. Redaction can replace a status-footer
|
|
600
|
+
// session UUID with [REDACTED], destroying its structural signature.
|
|
601
|
+
.filter(line => !excludeLine?.(line))
|
|
584
602
|
.map(line => line
|
|
585
603
|
.replace(bearer, "Bearer [REDACTED]")
|
|
586
604
|
.replace(secretAssignment, "$1=[REDACTED]")
|
|
@@ -597,12 +615,14 @@ export function sanitizePaneTail(pane, lineCount = 5) {
|
|
|
597
615
|
* final answer exists only on screen. Everything up to and including the last
|
|
598
616
|
* line of the inbound message we pasted is cut (the reply starts after it),
|
|
599
617
|
* lines with no letters or digits are dropped (borders, separators, spinners,
|
|
600
|
-
* bare prompts), and
|
|
601
|
-
* when
|
|
602
|
-
*
|
|
618
|
+
* bare prompts), and UI chrome is dropped by a backend-specific per-line
|
|
619
|
+
* filter when available (or a legacy single-line ready pattern). Whole-pane
|
|
620
|
+
* readiness regexes cannot identify individual prompt/footer lines. Returns
|
|
621
|
+
* null when what remains is trivial — a proxy message must carry an answer,
|
|
622
|
+
* not chrome. Secrets are redacted by sanitizePaneTail, same as stuck diagnostics.
|
|
603
623
|
*/
|
|
604
624
|
export function extractProxyReplyText(pane, opts = {}) {
|
|
605
|
-
const lines = sanitizePaneTail(pane, opts.maxLines ?? 40);
|
|
625
|
+
const lines = sanitizePaneTail(pane, opts.maxLines ?? 40, opts.isChromeLine);
|
|
606
626
|
const marker = opts.inboundMarker?.trim();
|
|
607
627
|
// Require a distinctive marker: a short one ("ok") would match agent text.
|
|
608
628
|
if (marker && marker.length >= 8) {
|
|
@@ -930,6 +950,8 @@ export class Daemon extends EventEmitter {
|
|
|
930
950
|
instanceStateBusyPattern = null;
|
|
931
951
|
instanceStateMonitorActive = false;
|
|
932
952
|
sessionCheckpointWarningEmitted = false;
|
|
953
|
+
codexSessionIdentityHoldNotified = false;
|
|
954
|
+
recoveredLegacyCodexSessionId = null;
|
|
933
955
|
statePollInFlight = false;
|
|
934
956
|
// One generation-scoped source of truth for human-turn reply completion.
|
|
935
957
|
// It also supplies the older dead-MCP/malformed-call recovery paths, so those
|
|
@@ -1185,6 +1207,11 @@ export class Daemon extends EventEmitter {
|
|
|
1185
1207
|
this.autoPauseController = new AutoPauseController(Math.max(0, autoPauseMinutes) * 60_000, readLastInboundAt(instanceDir) ?? Date.now());
|
|
1186
1208
|
}
|
|
1187
1209
|
async start() {
|
|
1210
|
+
// Do this before touching daemon.pid or a pre-existing window. An unknown
|
|
1211
|
+
// rollout format with an owned marker must not become a fresh session.
|
|
1212
|
+
if (this.backend?.hasInvalidSessionIdentity?.(this.config.working_directory)) {
|
|
1213
|
+
throw new CodexResumeIdentityError();
|
|
1214
|
+
}
|
|
1188
1215
|
mkdirSync(this.instanceDir, { recursive: true });
|
|
1189
1216
|
writeFileSync(join(this.instanceDir, "daemon.pid"), String(process.pid));
|
|
1190
1217
|
this.logger.info(`Starting ${this.name}`);
|
|
@@ -1194,8 +1221,16 @@ export class Daemon extends EventEmitter {
|
|
|
1194
1221
|
if (existsSync(crashStatePath)) {
|
|
1195
1222
|
const state = JSON.parse(readFileSync(crashStatePath, "utf-8"));
|
|
1196
1223
|
if (state.resumeDisabled) {
|
|
1197
|
-
this.
|
|
1198
|
-
|
|
1224
|
+
if (this.backend?.binaryName === "codex" && this.backend.canResume?.(this.config.working_directory)) {
|
|
1225
|
+
// A crash-loop heuristic cannot silently discard an explicitly
|
|
1226
|
+
// owned Codex conversation. Try once, then hold with a visible
|
|
1227
|
+
// reason rather than cycling or choosing a new session.
|
|
1228
|
+
this.logger.warn("Previous crash loop detected — preserving explicit Codex resume");
|
|
1229
|
+
}
|
|
1230
|
+
else {
|
|
1231
|
+
this.skipResume = true;
|
|
1232
|
+
this.logger.warn("Previous crash loop detected — starting without resume");
|
|
1233
|
+
}
|
|
1199
1234
|
}
|
|
1200
1235
|
unlinkSync(crashStatePath);
|
|
1201
1236
|
}
|
|
@@ -1323,7 +1358,12 @@ export class Daemon extends EventEmitter {
|
|
|
1323
1358
|
if (savedId) {
|
|
1324
1359
|
const oldTmux = new TmuxManager(this.tmuxSessionName, savedId);
|
|
1325
1360
|
if (await oldTmux.isWindowAlive()) {
|
|
1326
|
-
this.
|
|
1361
|
+
this.backend?.setActivePanePid?.(await TmuxManager.getPanePid(this.tmuxSessionName, savedId));
|
|
1362
|
+
const recoveredPane = this.backend?.binaryName === "codex"
|
|
1363
|
+
? await oldTmux.capturePane().catch(() => undefined) : undefined;
|
|
1364
|
+
const recoveredId = this.checkpointLivePaneBeforeReplacement(recoveredPane);
|
|
1365
|
+
if (this.backend?.binaryName === "codex")
|
|
1366
|
+
this.recoveredLegacyCodexSessionId = recoveredId;
|
|
1327
1367
|
await oldTmux.killWindow();
|
|
1328
1368
|
this.logger.info({ savedId }, "Killed old tmux window for fresh start");
|
|
1329
1369
|
}
|
|
@@ -1403,7 +1443,10 @@ export class Daemon extends EventEmitter {
|
|
|
1403
1443
|
this.guardian = new ContextGuardian(this.config.context_guardian, this.logger, statusFile);
|
|
1404
1444
|
this.guardian.startWatching();
|
|
1405
1445
|
this.guardian.on("status_update", () => {
|
|
1406
|
-
|
|
1446
|
+
// Codex's /new can leave multiple writer locks in one PID. A status
|
|
1447
|
+
// callback has no pane footer, so it cannot attest the current chat.
|
|
1448
|
+
if (this.backend?.binaryName !== "codex")
|
|
1449
|
+
this.saveSessionId();
|
|
1407
1450
|
});
|
|
1408
1451
|
// Context rotation removed: all CLI backends have built-in auto-compact.
|
|
1409
1452
|
// Crash recovery (health check + respawn with snapshot) is retained below.
|
|
@@ -1947,7 +1990,8 @@ export class Daemon extends EventEmitter {
|
|
|
1947
1990
|
await this.tmux.killWindow();
|
|
1948
1991
|
}
|
|
1949
1992
|
// Detect claude-code background session conflict — recover without counting as crash
|
|
1950
|
-
if (
|
|
1993
|
+
if (this.backend?.binaryName !== "codex" && lastOutput
|
|
1994
|
+
&& (lastOutput.includes("background agent") || lastOutput.includes("Session is currently running"))) {
|
|
1951
1995
|
if (!this.backgroundSessionRecoveryAttempted) {
|
|
1952
1996
|
this.backgroundSessionRecoveryAttempted = true;
|
|
1953
1997
|
this.logger.warn("Detected lingering background agent session — starting fresh (no resume)");
|
|
@@ -1988,6 +2032,14 @@ export class Daemon extends EventEmitter {
|
|
|
1988
2032
|
// and skip resume so the next spawn starts fresh. (skipResume also stops
|
|
1989
2033
|
// saveSessionId below from resurrecting the id from statusline.json.)
|
|
1990
2034
|
if (this.paneSaysNoConversation(lastOutput)) {
|
|
2035
|
+
if (this.backend?.binaryName === "codex" && this.backend.hasSessionIdentity?.()) {
|
|
2036
|
+
// Codex may have changed its rollout format or emitted an error
|
|
2037
|
+
// that only resembles this generic pattern. A persisted UUID is
|
|
2038
|
+
// never discarded by a text heuristic; preserve and hold.
|
|
2039
|
+
this.healthCheckPaused = true;
|
|
2040
|
+
this.emitSupervisionEnded("Codex could not resume its owned session", "Inspect the preserved session ID/rollout and restart this instance manually.");
|
|
2041
|
+
return;
|
|
2042
|
+
}
|
|
1991
2043
|
this.setSessionAside("cli_reported_no_conversation");
|
|
1992
2044
|
this.skipResume = true;
|
|
1993
2045
|
}
|
|
@@ -2045,7 +2097,10 @@ export class Daemon extends EventEmitter {
|
|
|
2045
2097
|
this.logger.warn({ crashCount: this.crashCount, delay }, `${cliLabel} window died — respawning after backoff`);
|
|
2046
2098
|
await new Promise(r => setTimeout(r, delay));
|
|
2047
2099
|
try {
|
|
2048
|
-
|
|
2100
|
+
// The dead Codex pane cannot supply current-session evidence.
|
|
2101
|
+
// Its last verified checkpoint stays as-is (or held if ambiguous).
|
|
2102
|
+
if (this.backend?.binaryName !== "codex")
|
|
2103
|
+
this.saveSessionId();
|
|
2049
2104
|
this.transcriptMonitor?.resetOffset();
|
|
2050
2105
|
// Kill orphan MCP server from the crashed CLI session.
|
|
2051
2106
|
// MCP server writes its PID to channel.mcp.pid on startup.
|
|
@@ -2093,7 +2148,16 @@ export class Daemon extends EventEmitter {
|
|
|
2093
2148
|
this.stormWindow?.markRecovered(this.name);
|
|
2094
2149
|
}
|
|
2095
2150
|
catch (err) {
|
|
2096
|
-
if (
|
|
2151
|
+
if (err instanceof CodexResumeConflictError || err instanceof CodexResumeUnavailableError) {
|
|
2152
|
+
// A live owner is not a broken conversation. Stop automatic
|
|
2153
|
+
// respawns and preserve the explicit session for a human retry.
|
|
2154
|
+
this.healthCheckPaused = true;
|
|
2155
|
+
this.logger.warn({ ownerPid: err instanceof CodexResumeConflictError ? err.ownerPid : null }, "Codex resume cannot continue — supervision stopped");
|
|
2156
|
+
this.emitSupervisionEnded(err.message, err instanceof CodexResumeConflictError
|
|
2157
|
+
? "Close the other Codex owner, then restart this instance."
|
|
2158
|
+
: "Inspect the Codex pane and restart this instance after resolving the startup failure.");
|
|
2159
|
+
}
|
|
2160
|
+
else if (!this.handOffBackendUnreachableRespawn(err)) {
|
|
2097
2161
|
this.logger.error({ err }, `Failed to respawn ${cliLabel} window`);
|
|
2098
2162
|
}
|
|
2099
2163
|
}
|
|
@@ -2642,7 +2706,9 @@ export class Daemon extends EventEmitter {
|
|
|
2642
2706
|
// Quit CLI FIRST — this kills MCP server child processes cleanly.
|
|
2643
2707
|
// IPC must stay open during quit so MCP servers receive the shutdown message.
|
|
2644
2708
|
if (this.tmux) {
|
|
2645
|
-
this.
|
|
2709
|
+
const codexPane = this.backend?.binaryName === "codex"
|
|
2710
|
+
? await this.tmux.capturePane().catch(() => undefined) : undefined;
|
|
2711
|
+
this.saveSessionId(codexPane);
|
|
2646
2712
|
this.healthCheckPaused = true;
|
|
2647
2713
|
let killed = false;
|
|
2648
2714
|
const quitSent = await this.sendQuitSequence();
|
|
@@ -2785,7 +2851,9 @@ export class Daemon extends EventEmitter {
|
|
|
2785
2851
|
this.freezeRuntimeMonitors();
|
|
2786
2852
|
const transition = (async () => {
|
|
2787
2853
|
try {
|
|
2788
|
-
this.
|
|
2854
|
+
const codexPane = this.backend?.binaryName === "codex"
|
|
2855
|
+
? await this.tmux?.capturePane().catch(() => undefined) : undefined;
|
|
2856
|
+
this.saveSessionId(codexPane);
|
|
2789
2857
|
await this.sendQuitSequence();
|
|
2790
2858
|
let exited = false;
|
|
2791
2859
|
for (let i = 0; i < 15; i++) {
|
|
@@ -2924,13 +2992,13 @@ export class Daemon extends EventEmitter {
|
|
|
2924
2992
|
return;
|
|
2925
2993
|
}
|
|
2926
2994
|
this.instanceState = snapshot.state;
|
|
2927
|
-
// OpenCode
|
|
2995
|
+
// OpenCode and Codex create their session lazily on the first submitted message.
|
|
2928
2996
|
// Waiting until stop/pause to persist that id loses resume state when the
|
|
2929
2997
|
// fleet is SIGKILLed or the host reboots.
|
|
2930
2998
|
// Idle observations are safe checkpoints; the 60s safety sweep also gives
|
|
2931
2999
|
// us a bounded retry if session creation produced no visible state edge.
|
|
2932
|
-
if (snapshot.state === "idle" && this.backend?.binaryName === "opencode") {
|
|
2933
|
-
this.saveSessionId();
|
|
3000
|
+
if (snapshot.state === "idle" && (this.backend?.binaryName === "opencode" || this.backend?.binaryName === "codex")) {
|
|
3001
|
+
this.saveSessionId(pane);
|
|
2934
3002
|
}
|
|
2935
3003
|
// Only a transition back to idle completes pending work. Repeated idle
|
|
2936
3004
|
// observations between enqueue and paste must not clear a newer inbound.
|
|
@@ -3214,7 +3282,11 @@ export class Daemon extends EventEmitter {
|
|
|
3214
3282
|
pane = await this.tmux?.capturePane();
|
|
3215
3283
|
if (!pane)
|
|
3216
3284
|
return;
|
|
3217
|
-
const text = extractProxyReplyText(pane, {
|
|
3285
|
+
const text = extractProxyReplyText(pane, {
|
|
3286
|
+
inboundMarker: target.inboundMarker,
|
|
3287
|
+
readyPattern: this.instanceStateReadyPattern,
|
|
3288
|
+
isChromeLine: line => this.backend?.isProxyReplyChromeLine?.(line) ?? false,
|
|
3289
|
+
});
|
|
3218
3290
|
if (!text) {
|
|
3219
3291
|
this.logger.debug("Dead-MCP proxy reply skipped — pane tail is trivial");
|
|
3220
3292
|
return;
|
|
@@ -3952,14 +4024,16 @@ export class Daemon extends EventEmitter {
|
|
|
3952
4024
|
* steer's verdict decide whether a queued message's sender is told its
|
|
3953
4025
|
* delivery failed — a false ❌ carrying the wrong correlation id.
|
|
3954
4026
|
*/
|
|
3955
|
-
failDelivery(verdict, status) {
|
|
4027
|
+
failDelivery(verdict, status, phase = "unknown", proof = "undelivered") {
|
|
3956
4028
|
verdict.reached = true;
|
|
4029
|
+
verdict.phase = phase;
|
|
4030
|
+
verdict.proof = proof;
|
|
3957
4031
|
if (status)
|
|
3958
4032
|
this.emit("message_failed", status); // ❌
|
|
3959
4033
|
return false;
|
|
3960
4034
|
}
|
|
3961
4035
|
/** Tell the fleet when an already-accepted cross-instance pane write failed. */
|
|
3962
|
-
reportCrossInstanceDeliveryFailure(meta, error) {
|
|
4036
|
+
reportCrossInstanceDeliveryFailure(meta, verdict, error) {
|
|
3963
4037
|
if (!meta.from_instance)
|
|
3964
4038
|
return;
|
|
3965
4039
|
this.ipcServer?.broadcast({
|
|
@@ -3967,7 +4041,10 @@ export class Daemon extends EventEmitter {
|
|
|
3967
4041
|
senderSession: meta.from_instance,
|
|
3968
4042
|
targetInstance: this.name,
|
|
3969
4043
|
correlationId: meta.correlation_id ?? "unknown",
|
|
3970
|
-
|
|
4044
|
+
// Never use getLastPasteError as a generic fallback: it may describe an
|
|
4045
|
+
// earlier paste, and #910 was a post-submit proof failure, not a paste
|
|
4046
|
+
// rejection. Keep the sender diagnostic phase-specific and non-secret.
|
|
4047
|
+
error: `delivery failed: phase=${verdict?.phase ?? (error ? "exception" : "unknown")}; proof=${verdict?.proof ?? (error ? "threw" : "none")}`,
|
|
3971
4048
|
});
|
|
3972
4049
|
}
|
|
3973
4050
|
/**
|
|
@@ -4021,12 +4098,12 @@ export class Daemon extends EventEmitter {
|
|
|
4021
4098
|
// Same rule as the queued path: a steer that never got to try is not a
|
|
4022
4099
|
// delivery failure. This holder is the steer's own, which is the point
|
|
4023
4100
|
// — it runs on steerLock while a queued delivery runs on pasteLock.
|
|
4024
|
-
this.reportCrossInstanceDeliveryFailure(meta);
|
|
4101
|
+
this.reportCrossInstanceDeliveryFailure(meta, verdict);
|
|
4025
4102
|
}
|
|
4026
4103
|
}).catch(err => {
|
|
4027
4104
|
this.logger.warn({ err: err.message }, "steer delivery error");
|
|
4028
4105
|
if (this.isDeliveryEpochCurrent(deliveryEpoch)) {
|
|
4029
|
-
this.reportCrossInstanceDeliveryFailure(meta, err.message);
|
|
4106
|
+
this.reportCrossInstanceDeliveryFailure(meta, undefined, err.message);
|
|
4030
4107
|
}
|
|
4031
4108
|
});
|
|
4032
4109
|
}
|
|
@@ -4160,7 +4237,7 @@ export class Daemon extends EventEmitter {
|
|
|
4160
4237
|
// not ready yet, a cancel, a storm hold or a shutdown all return
|
|
4161
4238
|
// false without one, and telling the sender its message was lost
|
|
4162
4239
|
// there would be the false ❌ of #826 in its other form.
|
|
4163
|
-
this.reportCrossInstanceDeliveryFailure(meta);
|
|
4240
|
+
this.reportCrossInstanceDeliveryFailure(meta, verdict);
|
|
4164
4241
|
}
|
|
4165
4242
|
}
|
|
4166
4243
|
finally {
|
|
@@ -4169,7 +4246,7 @@ export class Daemon extends EventEmitter {
|
|
|
4169
4246
|
}).catch(err => {
|
|
4170
4247
|
this.logger.warn({ err: err.message }, "pasteLock delivery error — chain continues");
|
|
4171
4248
|
if (this.isDeliveryEpochCurrent(deliveryEpoch)) {
|
|
4172
|
-
this.reportCrossInstanceDeliveryFailure(meta, err.message);
|
|
4249
|
+
this.reportCrossInstanceDeliveryFailure(meta, undefined, err.message);
|
|
4173
4250
|
}
|
|
4174
4251
|
});
|
|
4175
4252
|
this.logger.debug({ user: meta.user, text: content.slice(0, 100) }, "Queued channel message for delivery");
|
|
@@ -4214,7 +4291,14 @@ export class Daemon extends EventEmitter {
|
|
|
4214
4291
|
return false;
|
|
4215
4292
|
}
|
|
4216
4293
|
// Before anything reads the window id: a spawn in progress is about to change it.
|
|
4217
|
-
|
|
4294
|
+
// A false return means the cap expired with a spawn STILL running — the
|
|
4295
|
+
// generation captured below is in-progress evidence, not settled evidence,
|
|
4296
|
+
// so the critical section must still back out (see settledClean).
|
|
4297
|
+
const settledClean = await this.waitForSpawnToSettle();
|
|
4298
|
+
// Everything captured below (window id, readiness verdicts) belongs to
|
|
4299
|
+
// this spawn generation. A spawn that starts afterwards is detected inside
|
|
4300
|
+
// the critical section, where this delivery backs out and redoes itself.
|
|
4301
|
+
const settleGeneration = this.spawnGeneration;
|
|
4218
4302
|
if (cancelled())
|
|
4219
4303
|
return false;
|
|
4220
4304
|
if (this.refuseFatalStartupDelivery(verdict, status))
|
|
@@ -4239,7 +4323,8 @@ export class Daemon extends EventEmitter {
|
|
|
4239
4323
|
// fall through to the wait.
|
|
4240
4324
|
const canHandOff = (supportsQueuedInput || opts?.steer)
|
|
4241
4325
|
&& readiness === "busy"
|
|
4242
|
-
&& (await this.probeBlockingDialog()).state === "clear"
|
|
4326
|
+
&& (await this.probeBlockingDialog()).state === "clear"
|
|
4327
|
+
&& await this.hasPositiveDeliveryInput();
|
|
4243
4328
|
if (canHandOff) {
|
|
4244
4329
|
// Native queue (codex), or an explicit /steer: hand the complete
|
|
4245
4330
|
// paste+Enter transaction to the busy CLI now. For steer this is the
|
|
@@ -4259,7 +4344,7 @@ export class Daemon extends EventEmitter {
|
|
|
4259
4344
|
// wedged CLI (where the text would sit unsubmitted and the next message
|
|
4260
4345
|
// would land on top of it) — and instead of holding the queue silently.
|
|
4261
4346
|
this.logger.error("Pane still busy after the idle wait — reporting delivery failure");
|
|
4262
|
-
return this.failDelivery(verdict, status);
|
|
4347
|
+
return this.failDelivery(verdict, status, "readiness", "timeout-before-write");
|
|
4263
4348
|
}
|
|
4264
4349
|
}
|
|
4265
4350
|
}
|
|
@@ -4280,7 +4365,7 @@ export class Daemon extends EventEmitter {
|
|
|
4280
4365
|
// recovery Enter, so exactly STRANDED_INPUT_MAX_ROUNDS of them go out.
|
|
4281
4366
|
if (round >= STRANDED_INPUT_MAX_ROUNDS) {
|
|
4282
4367
|
this.logger.error({ round }, "Input row still not clear after the stranded-text recovery budget — reporting delivery failure");
|
|
4283
|
-
return this.failDelivery(verdict, status);
|
|
4368
|
+
return this.failDelivery(verdict, status, "stranded-input", "retry-budget-exhausted");
|
|
4284
4369
|
}
|
|
4285
4370
|
const stranded = await this.submitStrandedInputIfAny(windowId);
|
|
4286
4371
|
if (cancelled())
|
|
@@ -4289,14 +4374,14 @@ export class Daemon extends EventEmitter {
|
|
|
4289
4374
|
break;
|
|
4290
4375
|
if (stranded === "failed") {
|
|
4291
4376
|
this.logger.error({ round }, "Could not submit the stranded text — reporting delivery failure");
|
|
4292
|
-
return this.failDelivery(verdict, status);
|
|
4377
|
+
return this.failDelivery(verdict, status, "stranded-input", "enter-failed");
|
|
4293
4378
|
}
|
|
4294
4379
|
const readyAgain = await this.waitForPaneReadyForDelivery(windowId);
|
|
4295
4380
|
if (cancelled())
|
|
4296
4381
|
return false;
|
|
4297
4382
|
if (!readyAgain) {
|
|
4298
4383
|
this.logger.error("Pane never returned to its prompt after submitting stranded input — reporting delivery failure");
|
|
4299
|
-
return this.failDelivery(verdict, status);
|
|
4384
|
+
return this.failDelivery(verdict, status, "stranded-input", "prompt-timeout");
|
|
4300
4385
|
}
|
|
4301
4386
|
}
|
|
4302
4387
|
}
|
|
@@ -4308,13 +4393,22 @@ export class Daemon extends EventEmitter {
|
|
|
4308
4393
|
const outcome = await this.paneWriteLock.run(async () => {
|
|
4309
4394
|
if (cancelled())
|
|
4310
4395
|
return false;
|
|
4396
|
+
// A spawn that started after the settle wait above invalidates the
|
|
4397
|
+
// window id and every readiness verdict since: the pane about to be
|
|
4398
|
+
// written may already belong to a replacement process. The same holds
|
|
4399
|
+
// when the settle wait itself timed out (!settledClean) or a spawn is
|
|
4400
|
+
// running right now — none of those is settled evidence. Never wait
|
|
4401
|
+
// here (startup dismissal needs this lock) — back out and redo the
|
|
4402
|
+
// delivery from the top, where the settle wait runs again outside it.
|
|
4403
|
+
if (!settledClean || settleGeneration !== this.spawnGeneration || this.spawning)
|
|
4404
|
+
return "spawn-started";
|
|
4311
4405
|
if (this.refuseFatalStartupDelivery(verdict, status))
|
|
4312
4406
|
return false;
|
|
4313
4407
|
// A passive startup phase may paint after the outer readiness probe.
|
|
4314
4408
|
// It clears without input, so waiting under the pane lock cannot starve
|
|
4315
4409
|
// a dialog dismisser and closes the final clear→paste TOCTOU window.
|
|
4316
4410
|
if (!(await this.waitForInputTransientToClear("pre-write"))) {
|
|
4317
|
-
return this.failDelivery(verdict, status);
|
|
4411
|
+
return this.failDelivery(verdict, status, "pre-write", "input-transient-timeout");
|
|
4318
4412
|
}
|
|
4319
4413
|
// TOCTOU: the probes above ran outside this lock, and the CLI repaints
|
|
4320
4414
|
// whenever it likes — a resume prompt can be painted between "clear" and
|
|
@@ -4325,8 +4419,22 @@ export class Daemon extends EventEmitter {
|
|
|
4325
4419
|
if (probe.state !== "clear")
|
|
4326
4420
|
return "dialog";
|
|
4327
4421
|
}
|
|
4422
|
+
if (!(await this.hasPositiveDeliveryInput()))
|
|
4423
|
+
return "dialog";
|
|
4328
4424
|
return this.writeMessageToPane(formatted, windowId, handingOffToNativeQueue, status, opts?.submissionId, verdict);
|
|
4329
4425
|
});
|
|
4426
|
+
if (outcome === "spawn-started") {
|
|
4427
|
+
// The pane changed under this delivery: its queued paste must not land
|
|
4428
|
+
// in the replacement process's first screen. Redo the whole delivery
|
|
4429
|
+
// (fresh window id, fresh probes) — bounded, so spawn churn ends in an
|
|
4430
|
+
// honest failure rather than a message that never lands.
|
|
4431
|
+
const retries = (opts?.spawnRetry ?? 0) + 1;
|
|
4432
|
+
if (retries > DELIVERY_SPAWN_RACE_MAX_ROUNDS) {
|
|
4433
|
+
this.logger.error({ rounds: retries }, "A spawn kept starting before the pane write — reporting delivery failure");
|
|
4434
|
+
return this.failDelivery(verdict, status);
|
|
4435
|
+
}
|
|
4436
|
+
return this.deliverMessage(formatted, status, { ...opts, spawnRetry: retries });
|
|
4437
|
+
}
|
|
4330
4438
|
if (outcome !== "dialog")
|
|
4331
4439
|
return outcome;
|
|
4332
4440
|
// Wait OUTSIDE the lock (holding it would starve the runtime dismisser),
|
|
@@ -4334,14 +4442,14 @@ export class Daemon extends EventEmitter {
|
|
|
4334
4442
|
// an honest failure rather than a message that never lands.
|
|
4335
4443
|
if (round + 1 >= LATE_DIALOG_WRITE_ROUNDS) {
|
|
4336
4444
|
this.logger.error({ rounds: round + 1 }, "A dialog kept appearing before the pane write — reporting delivery failure");
|
|
4337
|
-
return this.failDelivery(verdict, status);
|
|
4445
|
+
return this.failDelivery(verdict, status, "pre-write", "dialog-or-input-not-ready");
|
|
4338
4446
|
}
|
|
4339
|
-
this.logger.info("Dialog appeared before the pane write — waiting for
|
|
4447
|
+
this.logger.info("Dialog or input transition appeared before the pane write — waiting for readiness");
|
|
4340
4448
|
const clear = gateWindowId ? await this.waitForPaneReadyForDelivery(gateWindowId) : false;
|
|
4341
4449
|
if (cancelled())
|
|
4342
4450
|
return false;
|
|
4343
4451
|
if (!clear) {
|
|
4344
|
-
return this.failDelivery(verdict, status);
|
|
4452
|
+
return this.failDelivery(verdict, status, "pre-write", "readiness-timeout");
|
|
4345
4453
|
}
|
|
4346
4454
|
}
|
|
4347
4455
|
}
|
|
@@ -4602,6 +4710,27 @@ export class Daemon extends EventEmitter {
|
|
|
4602
4710
|
async isPaneReadyForDelivery(windowId) {
|
|
4603
4711
|
return (await this.paneReadinessForDelivery(windowId)) === "ready";
|
|
4604
4712
|
}
|
|
4713
|
+
needsStartupInputProof() {
|
|
4714
|
+
return !!this.backend?.isDeliveryInputReadyPane
|
|
4715
|
+
&& this.inputTransientGuardGeneration === this.spawnGeneration;
|
|
4716
|
+
}
|
|
4717
|
+
/** Codex can paint a prompt before the TTY enters raw mode. Both are required. */
|
|
4718
|
+
async hasPositiveDeliveryInput() {
|
|
4719
|
+
if (!this.needsStartupInputProof())
|
|
4720
|
+
return true;
|
|
4721
|
+
const check = this.backend?.isDeliveryInputReadyPane;
|
|
4722
|
+
if (!check || !this.tmux)
|
|
4723
|
+
return !check;
|
|
4724
|
+
try {
|
|
4725
|
+
const mode = await this.tmux.getPaneInputMode?.();
|
|
4726
|
+
if (mode !== "raw")
|
|
4727
|
+
return false;
|
|
4728
|
+
return check.call(this.backend, await this.tmux.capturePane());
|
|
4729
|
+
}
|
|
4730
|
+
catch {
|
|
4731
|
+
return false;
|
|
4732
|
+
}
|
|
4733
|
+
}
|
|
4605
4734
|
/**
|
|
4606
4735
|
* Why a pane is not deliverable matters. "busy" may be handed to a native
|
|
4607
4736
|
* input queue or steered (after its own dialog probe); "dialog", "transient"
|
|
@@ -4620,6 +4749,9 @@ export class Daemon extends EventEmitter {
|
|
|
4620
4749
|
return "transient";
|
|
4621
4750
|
if (transient.state === "unknown")
|
|
4622
4751
|
return "unknown";
|
|
4752
|
+
if (this.needsStartupInputProof()) {
|
|
4753
|
+
return await this.hasPositiveDeliveryInput() ? "ready" : "transient";
|
|
4754
|
+
}
|
|
4623
4755
|
if (this.backend?.dropsEnterWhileBusy?.() !== true)
|
|
4624
4756
|
return "ready";
|
|
4625
4757
|
const prompt = this.backend.getBottomReadyPattern?.();
|
|
@@ -4664,7 +4796,8 @@ export class Daemon extends EventEmitter {
|
|
|
4664
4796
|
// screen", and a dialog disappears without any output edge the silence
|
|
4665
4797
|
// gate could see.
|
|
4666
4798
|
const deadline = Date.now() + timeoutMs;
|
|
4667
|
-
const bottomGated = this.backend?.dropsEnterWhileBusy?.() === true
|
|
4799
|
+
const bottomGated = this.backend?.dropsEnterWhileBusy?.() === true
|
|
4800
|
+
|| this.needsStartupInputProof();
|
|
4668
4801
|
let unknownStreak = 0;
|
|
4669
4802
|
let transientBudget = null;
|
|
4670
4803
|
for (;;) {
|
|
@@ -4787,11 +4920,12 @@ export class Daemon extends EventEmitter {
|
|
|
4787
4920
|
return "busy";
|
|
4788
4921
|
if (this.backend?.getBusyPattern?.()?.test(pane))
|
|
4789
4922
|
return "busy";
|
|
4923
|
+
if (this.backend?.isDeliveryInputReadyPane && !this.backend.isDeliveryInputReadyPane(pane))
|
|
4924
|
+
return "busy";
|
|
4790
4925
|
if (strandedAgendMessageInInput(pane, prompt))
|
|
4791
4926
|
return "stranded";
|
|
4792
|
-
// Kiro
|
|
4793
|
-
//
|
|
4794
|
-
// enabled solely to recover a positively identified old AgEnD strand.
|
|
4927
|
+
// Kiro uses the bottom row; Codex's positive prompt/footer check above
|
|
4928
|
+
// rules out a historical transcript echo or a transition/modal screen.
|
|
4795
4929
|
return !requireBottomReady || bottomRowIsReady(pane, prompt) ? "clear" : "busy";
|
|
4796
4930
|
}
|
|
4797
4931
|
/**
|
|
@@ -4841,7 +4975,7 @@ export class Daemon extends EventEmitter {
|
|
|
4841
4975
|
if (!this.fatalStartupBlocked)
|
|
4842
4976
|
return false;
|
|
4843
4977
|
this.logger.error("Delivery refused — CLI is parked on a fatal startup screen");
|
|
4844
|
-
this.failDelivery(verdict, status);
|
|
4978
|
+
this.failDelivery(verdict, status, "fatal-startup", "blocked-before-write");
|
|
4845
4979
|
return true;
|
|
4846
4980
|
}
|
|
4847
4981
|
/**
|
|
@@ -4944,7 +5078,7 @@ export class Daemon extends EventEmitter {
|
|
|
4944
5078
|
if (!recoverable) {
|
|
4945
5079
|
// A tmux error the window cannot be recovered from: the text will
|
|
4946
5080
|
// never reach this pane, so it is a verdict like the others.
|
|
4947
|
-
return this.failDelivery(verdict, status);
|
|
5081
|
+
return this.failDelivery(verdict, status, "paste", "non-retryable-tmux-error");
|
|
4948
5082
|
}
|
|
4949
5083
|
windowId = (await this.recoverWindow()) ?? windowId;
|
|
4950
5084
|
if (attempt < maxAttempts)
|
|
@@ -4974,7 +5108,7 @@ export class Daemon extends EventEmitter {
|
|
|
4974
5108
|
}
|
|
4975
5109
|
let enterAt = Date.now();
|
|
4976
5110
|
if (!(await this.sendDeliveryEnter("initial-submit"))) {
|
|
4977
|
-
return this.failDelivery(verdict, status);
|
|
5111
|
+
return this.failDelivery(verdict, status, "submit-enter", "tmux-send-keys-failed");
|
|
4978
5112
|
}
|
|
4979
5113
|
// Kiro's legacy TUI can swallow Enter while it is still processing a large
|
|
4980
5114
|
// paste — not only during the post-ready redraw (#479): on slower hosts it
|
|
@@ -5023,8 +5157,16 @@ export class Daemon extends EventEmitter {
|
|
|
5023
5157
|
// Our text is on the pane but cannot be shown to have left the input
|
|
5024
5158
|
// row — either the backend exposes no input row (only /steer reaches
|
|
5025
5159
|
// this path on such a backend) or the pre-paste pane was unreadable.
|
|
5026
|
-
//
|
|
5027
|
-
//
|
|
5160
|
+
// For Codex, this can also mean its current layout is not a trusted
|
|
5161
|
+
// input/queue frame. Do not turn that uncertainty into a false ✅.
|
|
5162
|
+
if (this.backend?.isDeliveryInputReadyPane) {
|
|
5163
|
+
this.logger.warn({ phase: "native-queue-proof", proof }, "Codex native-queue outcome uncertain — not re-pasting or confirming");
|
|
5164
|
+
verdict.phase = "native-queue-proof";
|
|
5165
|
+
verdict.proof = proof;
|
|
5166
|
+
return false;
|
|
5167
|
+
}
|
|
5168
|
+
// Re-pasting on a backend without structured Codex pane evidence
|
|
5169
|
+
// would risk duplication; retain its legacy best-effort behavior.
|
|
5028
5170
|
this.logger.warn("Paste reached the pane but could not be verified as submitted — accepting without proof");
|
|
5029
5171
|
if (status)
|
|
5030
5172
|
this.emit("message_confirmed", status); // ✅ (best-effort)
|
|
@@ -5057,9 +5199,14 @@ export class Daemon extends EventEmitter {
|
|
|
5057
5199
|
this.logger.warn("Message still in the input row after idle — submitting the existing text instead of pasting it again");
|
|
5058
5200
|
const strandedAt = Date.now();
|
|
5059
5201
|
if (!(await this.sendDeliveryEnter("native-queue-stranded-submit"))) {
|
|
5060
|
-
return this.failDelivery(verdict, status);
|
|
5202
|
+
return this.failDelivery(verdict, status, "native-queue-submit", "tmux-send-keys-failed");
|
|
5061
5203
|
}
|
|
5062
|
-
|
|
5204
|
+
// The recovery Enter can be accepted before Codex paints its new
|
|
5205
|
+
// transcript. One immediate capture is not a failure verdict: give
|
|
5206
|
+
// that echo/queue a bounded chance to appear, without re-pasting.
|
|
5207
|
+
const afterEnter = this.backend?.isDeliveryInputReadyPane
|
|
5208
|
+
? await this.lateCodexSubmissionProof(signature, pasteBaseline, true)
|
|
5209
|
+
: await this.confirmSubmitted(signature, pasteBaseline);
|
|
5063
5210
|
if (afterEnter === "submitted") {
|
|
5064
5211
|
if (status)
|
|
5065
5212
|
this.emit("message_confirmed", status); // ✅
|
|
@@ -5071,8 +5218,23 @@ export class Daemon extends EventEmitter {
|
|
|
5071
5218
|
// another turn's output — and treating that as proof re-confirms a
|
|
5072
5219
|
// message nobody submitted. Output is corroboration; text sitting in
|
|
5073
5220
|
// the input row is disqualifying, and disqualifying evidence wins.
|
|
5221
|
+
if (this.backend?.isDeliveryInputReadyPane && afterEnter !== "stranded") {
|
|
5222
|
+
this.logger.warn({ phase: "native-queue-submit", proof: afterEnter, strandedAt }, "Codex recovery Enter outcome uncertain — no hard failure or duplicate paste");
|
|
5223
|
+
verdict.phase = "native-queue-submit";
|
|
5224
|
+
verdict.proof = afterEnter;
|
|
5225
|
+
return false;
|
|
5226
|
+
}
|
|
5074
5227
|
this.logger.error({ afterEnter, strandedAt }, "Stranded message could not be submitted by Enter");
|
|
5075
|
-
return this.failDelivery(verdict, status);
|
|
5228
|
+
return this.failDelivery(verdict, status, "native-queue-submit", afterEnter);
|
|
5229
|
+
}
|
|
5230
|
+
if (this.backend?.isDeliveryInputReadyPane) {
|
|
5231
|
+
// In Codex a missing viewport echo is inconclusive, not a proof of
|
|
5232
|
+
// loss. Another paste could run the same request twice. Leave the
|
|
5233
|
+
// already-pasted delivery at 👀 and let the next observation decide.
|
|
5234
|
+
this.logger.warn({ phase: "native-queue-proof", proof: settled }, "Codex native-queue outcome uncertain — not re-pasting");
|
|
5235
|
+
verdict.phase = "native-queue-proof";
|
|
5236
|
+
verdict.proof = settled;
|
|
5237
|
+
return false;
|
|
5076
5238
|
}
|
|
5077
5239
|
// "unproven": nothing of ours is on screen — the paste itself was lost,
|
|
5078
5240
|
// so pasting it again cannot duplicate anything.
|
|
@@ -5081,12 +5243,12 @@ export class Daemon extends EventEmitter {
|
|
|
5081
5243
|
this.logger.error({
|
|
5082
5244
|
tmuxError: this.tmux.getLastPasteError?.() ?? "unknown tmux paste failure",
|
|
5083
5245
|
}, "Idle-gated redelivery paste failed after native-queue silent loss");
|
|
5084
|
-
return this.failDelivery(verdict, status);
|
|
5246
|
+
return this.failDelivery(verdict, status, "native-queue-redelivery", "tmux-paste-failed");
|
|
5085
5247
|
}
|
|
5086
5248
|
await new Promise(r => setTimeout(r, NORMAL_ENTER_SETTLE_MS));
|
|
5087
5249
|
const retryAt = Date.now();
|
|
5088
5250
|
if (!(await this.sendDeliveryEnter("native-queue-idle-redelivery"))) {
|
|
5089
|
-
return this.failDelivery(verdict, status);
|
|
5251
|
+
return this.failDelivery(verdict, status, "native-queue-redelivery", "tmux-send-keys-failed");
|
|
5090
5252
|
}
|
|
5091
5253
|
if (windowId && this.controlClient) {
|
|
5092
5254
|
if (await this.confirmAfterEnter(windowId, retryAt, signature, pasteBaseline, "native-queue-idle-redelivery-retry")) {
|
|
@@ -5101,7 +5263,7 @@ export class Daemon extends EventEmitter {
|
|
|
5101
5263
|
return true;
|
|
5102
5264
|
}
|
|
5103
5265
|
this.logger.error("Idle-gated redelivery also failed after native-queue silent loss");
|
|
5104
|
-
return this.failDelivery(verdict, status);
|
|
5266
|
+
return this.failDelivery(verdict, status, "native-queue-redelivery", "not-submitted");
|
|
5105
5267
|
}
|
|
5106
5268
|
if (windowId && this.controlClient && this.backend?.dropsEnterWhileBusy?.() === true) {
|
|
5107
5269
|
// F2: output after Enter is necessary but not sufficient — the paste
|
|
@@ -5115,7 +5277,7 @@ export class Daemon extends EventEmitter {
|
|
|
5115
5277
|
const promptBack = await this.waitForPaneReadyForDelivery(windowId, STRANDED_RETRY_READY_WAIT_MS);
|
|
5116
5278
|
const retryAt = Date.now();
|
|
5117
5279
|
if (promptBack && !(await this.sendDeliveryEnter("stranded-text-retry"))) {
|
|
5118
|
-
return this.failDelivery(verdict, status);
|
|
5280
|
+
return this.failDelivery(verdict, status, "stranded-text-retry", "tmux-send-keys-failed");
|
|
5119
5281
|
}
|
|
5120
5282
|
submitted = promptBack && await this.confirmSubmittedAfterEnter(windowId, retryAt, formatted);
|
|
5121
5283
|
}
|
|
@@ -5125,7 +5287,7 @@ export class Daemon extends EventEmitter {
|
|
|
5125
5287
|
}
|
|
5126
5288
|
else {
|
|
5127
5289
|
this.logger.error("Message pasted but never submitted (text still in the input row after Enter retry)");
|
|
5128
|
-
return this.failDelivery(verdict, status);
|
|
5290
|
+
return this.failDelivery(verdict, status, "stranded-text-retry", "stranded");
|
|
5129
5291
|
}
|
|
5130
5292
|
}
|
|
5131
5293
|
else if (windowId && this.controlClient) {
|
|
@@ -5145,12 +5307,26 @@ export class Daemon extends EventEmitter {
|
|
|
5145
5307
|
this.emit("message_confirmed", status); // ✅
|
|
5146
5308
|
}
|
|
5147
5309
|
else {
|
|
5148
|
-
|
|
5149
|
-
|
|
5150
|
-
|
|
5151
|
-
|
|
5152
|
-
|
|
5153
|
-
|
|
5310
|
+
const proof = this.backend?.isDeliveryInputReadyPane
|
|
5311
|
+
? await this.lateCodexSubmissionProof(signature, pasteBaseline)
|
|
5312
|
+
: await this.confirmSubmitted(signature, pasteBaseline);
|
|
5313
|
+
if (proof === "submitted") {
|
|
5314
|
+
if (status)
|
|
5315
|
+
this.emit("message_confirmed", status);
|
|
5316
|
+
return true;
|
|
5317
|
+
}
|
|
5318
|
+
if (this.backend?.isDeliveryInputReadyPane && proof !== "stranded") {
|
|
5319
|
+
// This is the observed #910 race: the CLI processed the message
|
|
5320
|
+
// although its echo had not appeared within the proof window. A
|
|
5321
|
+
// missing viewport signature cannot establish non-delivery; keep
|
|
5322
|
+
// 👀 and never emit the sender's hard ❌ or re-paste blindly.
|
|
5323
|
+
this.logger.warn({ phase: "post-submit-proof", proof }, "Codex delivery outcome uncertain — no hard failure or duplicate paste");
|
|
5324
|
+
verdict.phase = "post-submit-proof";
|
|
5325
|
+
verdict.proof = proof;
|
|
5326
|
+
return false;
|
|
5327
|
+
}
|
|
5328
|
+
this.logger.error({ phase: "post-submit-proof", proof }, "Message remains unsubmitted after Enter retry");
|
|
5329
|
+
return this.failDelivery(verdict, status, "post-submit-proof", proof);
|
|
5154
5330
|
}
|
|
5155
5331
|
}
|
|
5156
5332
|
else {
|
|
@@ -5163,7 +5339,7 @@ export class Daemon extends EventEmitter {
|
|
|
5163
5339
|
return true;
|
|
5164
5340
|
}
|
|
5165
5341
|
this.logger.error("Message delivery failed after retries — window not ready");
|
|
5166
|
-
return this.failDelivery(verdict, status);
|
|
5342
|
+
return this.failDelivery(verdict, status, "paste", "retry-budget-exhausted");
|
|
5167
5343
|
}
|
|
5168
5344
|
/**
|
|
5169
5345
|
* The single place a pasted message is judged submitted, shared by the
|
|
@@ -5221,6 +5397,42 @@ export class Daemon extends EventEmitter {
|
|
|
5221
5397
|
}
|
|
5222
5398
|
return busy;
|
|
5223
5399
|
}
|
|
5400
|
+
if (this.backend?.isDeliveryInputReadyPane) {
|
|
5401
|
+
// Codex's first post-wake redraw may hide the echo for a few seconds.
|
|
5402
|
+
// Absence from a viewport is NOT proof the paste was lost, so only a
|
|
5403
|
+
// positively identified strand authorizes another Enter. Never re-paste
|
|
5404
|
+
// here: the CLI may already be processing the unique message_id.
|
|
5405
|
+
let proof = "unproven";
|
|
5406
|
+
const firstDeadline = Date.now() + POST_ENTER_PROOF_WINDOW_MS;
|
|
5407
|
+
for (;;) {
|
|
5408
|
+
proof = await this.confirmSubmitted(signature, baseline);
|
|
5409
|
+
if (proof === "submitted")
|
|
5410
|
+
return true;
|
|
5411
|
+
if (proof === "stranded" || Date.now() >= firstDeadline)
|
|
5412
|
+
break;
|
|
5413
|
+
await new Promise(r => setTimeout(r, POST_ENTER_PROOF_POLL_MS));
|
|
5414
|
+
}
|
|
5415
|
+
if (proof !== "stranded")
|
|
5416
|
+
return false;
|
|
5417
|
+
if (!(await this.waitForPaneReadyForDelivery(windowId, STRANDED_RETRY_READY_WAIT_MS)))
|
|
5418
|
+
return false;
|
|
5419
|
+
proof = await this.confirmSubmitted(signature, baseline);
|
|
5420
|
+
if (proof === "submitted")
|
|
5421
|
+
return true;
|
|
5422
|
+
if (proof !== "stranded")
|
|
5423
|
+
return false;
|
|
5424
|
+
if (!(await this.sendDeliveryEnter(retryPhase)))
|
|
5425
|
+
return false;
|
|
5426
|
+
const retryDeadline = Date.now() + POST_ENTER_PROOF_WINDOW_MS;
|
|
5427
|
+
for (;;) {
|
|
5428
|
+
proof = await this.confirmSubmitted(signature, baseline);
|
|
5429
|
+
if (proof === "submitted")
|
|
5430
|
+
return true;
|
|
5431
|
+
if (Date.now() >= retryDeadline)
|
|
5432
|
+
return false;
|
|
5433
|
+
await new Promise(r => setTimeout(r, POST_ENTER_PROOF_POLL_MS));
|
|
5434
|
+
}
|
|
5435
|
+
}
|
|
5224
5436
|
if (await this.confirmSubmitted(signature, baseline) === "submitted")
|
|
5225
5437
|
return true;
|
|
5226
5438
|
// Retry once the prompt is back — an Enter sent into a mid-redraw TUI is
|
|
@@ -5270,6 +5482,14 @@ export class Daemon extends EventEmitter {
|
|
|
5270
5482
|
: "unproven";
|
|
5271
5483
|
}
|
|
5272
5484
|
const after = this.paneEvidence(pane, signature);
|
|
5485
|
+
// A Codex layout without its current input/footer pair may show a quoted
|
|
5486
|
+
// or historical `›` row. Unless its active Working banner corroborates a
|
|
5487
|
+
// real turn, text on that unknown screen cannot prove it left stdin.
|
|
5488
|
+
if (this.backend?.isDeliveryInputReadyPane
|
|
5489
|
+
&& !this.backend.isDeliveryInputReadyPane(pane)
|
|
5490
|
+
&& !this.backend.getBusyPattern?.()?.test(pane)) {
|
|
5491
|
+
return after.payload > 0 ? "unverifiable" : "unproven";
|
|
5492
|
+
}
|
|
5273
5493
|
// 1. Disqualifying evidence, checked FIRST and never overridden by the
|
|
5274
5494
|
// corroborating evidence below: our text is sitting in the input row, so
|
|
5275
5495
|
// it was not submitted — whatever else is on screen.
|
|
@@ -5308,6 +5528,35 @@ export class Daemon extends EventEmitter {
|
|
|
5308
5528
|
this.logger.warn("Could not read the pane before pasting — this delivery cannot be verified either way");
|
|
5309
5529
|
return "unverifiable";
|
|
5310
5530
|
}
|
|
5531
|
+
/** Only positive echo/queue evidence may turn an ambiguous Codex write into ✅. */
|
|
5532
|
+
async lateCodexSubmissionProof(signature, baseline, waitThroughStranded = false) {
|
|
5533
|
+
const deadline = Date.now() + CODEX_LATE_PROOF_MS;
|
|
5534
|
+
let proof = "unproven";
|
|
5535
|
+
for (;;) {
|
|
5536
|
+
proof = await this.confirmSubmitted(signature, baseline);
|
|
5537
|
+
// Ordinary idle-path proof may return a strand immediately. After a
|
|
5538
|
+
// native-queue recovery Enter, though, it can be the previous frame
|
|
5539
|
+
// still on screen; only a strand that survives the bounded repaint
|
|
5540
|
+
// window proves that Enter was swallowed.
|
|
5541
|
+
if (proof === "submitted" || (proof === "stranded" && !waitThroughStranded))
|
|
5542
|
+
return proof;
|
|
5543
|
+
if (signature.unique && this.tmux?.capturePaneWithHistory) {
|
|
5544
|
+
try {
|
|
5545
|
+
const history = await this.tmux.capturePaneWithHistory(300);
|
|
5546
|
+
const seen = this.paneEvidence(history, signature);
|
|
5547
|
+
const known = !this.backend?.isDeliveryInputReadyPane
|
|
5548
|
+
|| this.backend.isDeliveryInputReadyPane(history)
|
|
5549
|
+
|| this.backend.getBusyPattern?.()?.test(history);
|
|
5550
|
+
if (known && seen.payload > 0 && !seen.strandedInput)
|
|
5551
|
+
return this.submittedProof();
|
|
5552
|
+
}
|
|
5553
|
+
catch { /* a failed history read proves neither delivery nor loss */ }
|
|
5554
|
+
}
|
|
5555
|
+
if (Date.now() >= deadline)
|
|
5556
|
+
return proof;
|
|
5557
|
+
await new Promise(r => setTimeout(r, POST_ENTER_PROOF_POLL_MS));
|
|
5558
|
+
}
|
|
5559
|
+
}
|
|
5311
5560
|
/**
|
|
5312
5561
|
* What the pane currently shows of a given message. Taken once before the
|
|
5313
5562
|
* paste and once after, so confirmSubmitted can require a NEW marker or a NEW
|
|
@@ -5323,7 +5572,8 @@ export class Daemon extends EventEmitter {
|
|
|
5323
5572
|
const pane = await this.tmux.capturePane();
|
|
5324
5573
|
const evidence = this.paneEvidence(pane, signature);
|
|
5325
5574
|
const prompt = this.backend?.getBottomReadyPattern?.();
|
|
5326
|
-
if (prompt &&
|
|
5575
|
+
if (prompt && (!this.backend?.isDeliveryInputReadyPane || this.backend.isDeliveryInputReadyPane(pane))
|
|
5576
|
+
&& strandedAgendMessageInInput(pane, prompt)) {
|
|
5327
5577
|
// Whatever we paste now lands after it, and one Enter submits both as
|
|
5328
5578
|
// a single message. Nothing here can undo that; saying so beats
|
|
5329
5579
|
// letting two messages silently merge.
|
|
@@ -5344,7 +5594,8 @@ export class Daemon extends EventEmitter {
|
|
|
5344
5594
|
paneEvidence(pane, signature) {
|
|
5345
5595
|
const marker = this.backend?.getQueuedInputMarker?.();
|
|
5346
5596
|
const prompt = this.backend?.getBottomReadyPattern?.();
|
|
5347
|
-
const input = prompt
|
|
5597
|
+
const input = prompt && (!this.backend?.isDeliveryInputReadyPane || this.backend.isDeliveryInputReadyPane(pane))
|
|
5598
|
+
? inputAreaText(pane, prompt) : null;
|
|
5348
5599
|
return {
|
|
5349
5600
|
queued: marker ? pane.split(/\r?\n/).filter(row => marker.test(row)).length : 0,
|
|
5350
5601
|
payload: countOccurrences(pane.replace(/\s+/g, ""), signature.value),
|
|
@@ -5395,6 +5646,13 @@ export class Daemon extends EventEmitter {
|
|
|
5395
5646
|
return false;
|
|
5396
5647
|
let proof = await this.confirmSubmitted(signature, baseline);
|
|
5397
5648
|
if (proof === "unverifiable") {
|
|
5649
|
+
if (this.backend?.isDeliveryInputReadyPane) {
|
|
5650
|
+
// A Codex screen without a structurally current input/footer pair is
|
|
5651
|
+
// not a successful snapshot restore. The old best-effort rule below
|
|
5652
|
+
// is only for backends that expose no readable input row at all.
|
|
5653
|
+
this.logger.warn({ label }, "Codex system paste could not be verified");
|
|
5654
|
+
return false;
|
|
5655
|
+
}
|
|
5398
5656
|
// No input row to read: this backend gets exactly what the old pasteText
|
|
5399
5657
|
// path gave it, including the unconditional second Enter for queue-less
|
|
5400
5658
|
// TUIs that swallow the first. Narrowing that to one Enter on the grounds
|
|
@@ -5996,26 +6254,33 @@ export class Daemon extends EventEmitter {
|
|
|
5996
6254
|
* Bounded, and called BEFORE the pane lock is taken — waiting on the spawn while
|
|
5997
6255
|
* holding the lock the spawn itself needs would deadlock.
|
|
5998
6256
|
*/
|
|
5999
|
-
|
|
6257
|
+
/**
|
|
6258
|
+
* Wait for an in-flight spawn to finish, up to the cap. Returns true when no
|
|
6259
|
+
* spawn is in flight afterwards; false when the cap expired with one still
|
|
6260
|
+
* running. A false return is NOT settled evidence — the caller must not
|
|
6261
|
+
* treat the current generation as a completed spawn.
|
|
6262
|
+
*/
|
|
6263
|
+
async waitForSpawnToSettle(capMs = SPAWN_SETTLE_MAX_WAIT_MS) {
|
|
6000
6264
|
const settled = this.spawnSettled;
|
|
6001
6265
|
if (!settled)
|
|
6002
|
-
return;
|
|
6266
|
+
return true;
|
|
6003
6267
|
this.logger.debug("Holding delivery until the CLI has finished starting up");
|
|
6004
6268
|
let timer;
|
|
6005
6269
|
const cap = new Promise(resolve => {
|
|
6006
|
-
timer = setTimeout(resolve,
|
|
6270
|
+
timer = setTimeout(() => resolve(false), capMs);
|
|
6007
6271
|
timer.unref?.();
|
|
6008
6272
|
});
|
|
6009
6273
|
try {
|
|
6010
|
-
await Promise.race([settled, cap]);
|
|
6274
|
+
const finished = await Promise.race([settled.then(() => true), cap]);
|
|
6275
|
+
if (!finished) {
|
|
6276
|
+
this.logger.warn("CLI still starting after the delivery hold — the pane is not settled evidence");
|
|
6277
|
+
}
|
|
6278
|
+
return finished;
|
|
6011
6279
|
}
|
|
6012
6280
|
finally {
|
|
6013
6281
|
if (timer)
|
|
6014
6282
|
clearTimeout(timer);
|
|
6015
6283
|
}
|
|
6016
|
-
if (this.spawning) {
|
|
6017
|
-
this.logger.warn("CLI still starting after the delivery hold — delivering anyway");
|
|
6018
|
-
}
|
|
6019
6284
|
}
|
|
6020
6285
|
/** Spawn a CLI window. Returns true if --resume was used successfully. */
|
|
6021
6286
|
/**
|
|
@@ -6088,11 +6353,25 @@ export class Daemon extends EventEmitter {
|
|
|
6088
6353
|
if (!this.backend) {
|
|
6089
6354
|
throw new Error("No backend configured — cannot spawn CLI window");
|
|
6090
6355
|
}
|
|
6091
|
-
const attemptedResume = !this.skipResume
|
|
6356
|
+
const attemptedResume = !this.skipResume
|
|
6357
|
+
&& (this.backend.canResume?.(this.config.working_directory) ?? true);
|
|
6092
6358
|
// A resume launch may get a longer budget than a fresh one (kiro: the
|
|
6093
6359
|
// conversation must come back from the backend before anything paints).
|
|
6094
6360
|
const resumeBudget = attemptedResume ? this.startupBudgetFor(true) : undefined;
|
|
6095
6361
|
let alive = await this.trySpawn(false, resumeBudget);
|
|
6362
|
+
if (!alive && attemptedResume && this.backend.binaryName === "codex") {
|
|
6363
|
+
const status = await this.tmux?.getPaneStatus().catch(() => null);
|
|
6364
|
+
if (status?.exitCode === 75)
|
|
6365
|
+
throw new CodexResumeConflictError(null);
|
|
6366
|
+
const pane = await this.tmux?.capturePane().catch(() => "");
|
|
6367
|
+
if (pane?.includes("[agend:codex-session-held]"))
|
|
6368
|
+
throw new CodexResumeConflictError(null);
|
|
6369
|
+
if (pane && codexResumeLockVisible(pane))
|
|
6370
|
+
throw new CodexResumeConflictError(null);
|
|
6371
|
+
// A failed explicit resume is not evidence that the conversation is
|
|
6372
|
+
// unusable. Never enter the generic abandon/--fresh recovery path.
|
|
6373
|
+
throw new CodexResumeUnavailableError();
|
|
6374
|
+
}
|
|
6096
6375
|
if (!alive && attemptedResume) {
|
|
6097
6376
|
// Resume failed. Before abandoning the session:
|
|
6098
6377
|
// 1. If the backend is unreachable (fleet-level memory, or this very pane
|
|
@@ -6286,7 +6565,9 @@ export class Daemon extends EventEmitter {
|
|
|
6286
6565
|
this.logger.debug({ err }, "IPC server close failed during startup abort");
|
|
6287
6566
|
}
|
|
6288
6567
|
this.ipcServer = null;
|
|
6289
|
-
|
|
6568
|
+
// A rejection before tmux setup did not create a replacement window; its
|
|
6569
|
+
// pre-existing window-id still belongs to the old pane and must survive.
|
|
6570
|
+
for (const file of this.tmux ? ["daemon.pid", "window-id"] : ["daemon.pid"]) {
|
|
6290
6571
|
try {
|
|
6291
6572
|
unlinkSync(join(this.instanceDir, file));
|
|
6292
6573
|
}
|
|
@@ -6313,28 +6594,79 @@ export class Daemon extends EventEmitter {
|
|
|
6313
6594
|
* Returns true if CLI is ready, false if it failed or got stuck.
|
|
6314
6595
|
*/
|
|
6315
6596
|
async trySpawn(reuseWindow = false, startupTimeoutMs) {
|
|
6316
|
-
|
|
6317
|
-
|
|
6318
|
-
|
|
6319
|
-
|
|
6320
|
-
|
|
6321
|
-
|
|
6322
|
-
|
|
6597
|
+
const ready = !this.spawnGate
|
|
6598
|
+
? await this.trySpawnInsideGate(reuseWindow, startupTimeoutMs)
|
|
6599
|
+
: await this.spawnGate.run({
|
|
6600
|
+
instanceName: this.name,
|
|
6601
|
+
workingDirectory: this.config.working_directory,
|
|
6602
|
+
reason: reuseWindow ? "wake" : this.lastSpawnAt > 0 ? "recovery" : "startup",
|
|
6603
|
+
}, () => this.trySpawnInsideGate(reuseWindow, startupTimeoutMs));
|
|
6604
|
+
// All successful launches, including wake's trySpawn(true), cross this
|
|
6605
|
+
// point. The previous checkpoint in spawnClaudeWindow missed wake and
|
|
6606
|
+
// kept reading the dead pane PID after a resumed session changed.
|
|
6607
|
+
if (ready && !this.fatalStartupBlocked && this.backend?.binaryName === "codex" && this.tmux) {
|
|
6608
|
+
this.backend.setActivePanePid?.(await TmuxManager.getPanePid(this.tmuxSessionName, this.tmux.getWindowId()));
|
|
6609
|
+
this.saveSessionId(await this.tmux.capturePane().catch(() => undefined));
|
|
6610
|
+
}
|
|
6611
|
+
return ready;
|
|
6612
|
+
}
|
|
6613
|
+
/**
|
|
6614
|
+
* One fresh pane capture judged as ready && !busy && !dialog. Transcript
|
|
6615
|
+
* silence is deliberately NOT the gate: its monitor is inert for muse, so a
|
|
6616
|
+
* quiet-but-working pane would read as idle and the respawn below would kill
|
|
6617
|
+
* a live turn. Fail closed: an unreadable pane is never idle.
|
|
6618
|
+
*/
|
|
6619
|
+
async isPaneAuthoritativelyIdle() {
|
|
6620
|
+
const backend = this.backend;
|
|
6621
|
+
if (!this.tmux || !backend)
|
|
6622
|
+
return false;
|
|
6623
|
+
let pane;
|
|
6624
|
+
try {
|
|
6625
|
+
pane = await this.tmux.capturePane();
|
|
6626
|
+
}
|
|
6627
|
+
catch {
|
|
6628
|
+
return false;
|
|
6629
|
+
}
|
|
6630
|
+
if (backend.getBusyPattern?.()?.test(pane))
|
|
6631
|
+
return false;
|
|
6632
|
+
for (const dialog of this.deliveryBlockingDialogs()) {
|
|
6633
|
+
if (Daemon.dialogMatches(dialog, pane))
|
|
6634
|
+
return false;
|
|
6635
|
+
}
|
|
6636
|
+
if (!backend.getReadyPattern().test(pane))
|
|
6637
|
+
return false;
|
|
6638
|
+
return true;
|
|
6639
|
+
}
|
|
6640
|
+
/**
|
|
6641
|
+
* Poll the authoritative pane check until it reports idle or the budget runs
|
|
6642
|
+
* out. Each poll is a fresh capture, so a turn that ends mid-wait is seen.
|
|
6643
|
+
*/
|
|
6644
|
+
async waitForAuthoritativePaneIdle(budgetMs = MUSE_DIRECT_RESUME_IDLE_BUDGET_MS, pollMs = MUSE_DIRECT_RESUME_IDLE_POLL_MS) {
|
|
6645
|
+
const deadline = Date.now() + budgetMs;
|
|
6646
|
+
for (;;) {
|
|
6647
|
+
if (await this.isPaneAuthoritativelyIdle())
|
|
6648
|
+
return true;
|
|
6649
|
+
if (Date.now() >= deadline)
|
|
6650
|
+
return false;
|
|
6651
|
+
await new Promise(r => setTimeout(r, pollMs));
|
|
6652
|
+
}
|
|
6323
6653
|
}
|
|
6324
6654
|
/**
|
|
6325
6655
|
* A relay crash must not make the daemon kill a live Muse process. The
|
|
6326
6656
|
* supervisor normally reclaims the same port; when that bounded recovery
|
|
6327
|
-
* fails,
|
|
6328
|
-
*
|
|
6329
|
-
*
|
|
6657
|
+
* fails, move Muse back to a direct connection once it is authoritatively
|
|
6658
|
+
* idle (session preserved via the existing resume path) and mark usage
|
|
6659
|
+
* unavailable. The fallback flag also covers every later natural spawn.
|
|
6660
|
+
* Never throws: the floor is the old behavior (flag set, relay stopped,
|
|
6661
|
+
* next restart direct).
|
|
6330
6662
|
*/
|
|
6331
|
-
switchMuseToDirect() {
|
|
6663
|
+
switchMuseToDirect(idleBudgetMs = MUSE_DIRECT_RESUME_IDLE_BUDGET_MS, idlePollMs = MUSE_DIRECT_RESUME_IDLE_POLL_MS) {
|
|
6332
6664
|
if (this.museRelayFallbackInFlight || this.startupAborted || this.backend?.binaryName !== "muse") {
|
|
6333
6665
|
return this.museRelayFallbackInFlight ?? Promise.resolve();
|
|
6334
6666
|
}
|
|
6335
6667
|
this.museRelayFallbackInFlight = (async () => {
|
|
6336
6668
|
this.museRelayFallback = true;
|
|
6337
|
-
this.logger.warn("Muse usage relay could not recover — usage is unavailable until
|
|
6669
|
+
this.logger.warn("Muse usage relay could not recover — usage is unavailable until Muse restarts direct");
|
|
6338
6670
|
try {
|
|
6339
6671
|
if (this.museUsageRelay) {
|
|
6340
6672
|
await this.museUsageRelay.stop().catch(() => { });
|
|
@@ -6344,6 +6676,60 @@ export class Daemon extends EventEmitter {
|
|
|
6344
6676
|
catch (err) {
|
|
6345
6677
|
this.logger.warn({ reason: err instanceof Error ? err.message : "relay stop failed" }, "Muse usage relay cleanup failed");
|
|
6346
6678
|
}
|
|
6679
|
+
// Say so where the operator is looking: without this, muse keeps running
|
|
6680
|
+
// with --base-url pointed at the dead relay port and every turn fails.
|
|
6681
|
+
this.emit("muse_relay_exhausted", { name: this.name });
|
|
6682
|
+
// An idle muse can be moved now; a busy, paused, or supervised-paused one
|
|
6683
|
+
// is left alone — pausing/wake respawns direct anyway via the fallback
|
|
6684
|
+
// flag, and interrupting a live turn (or fighting supervision) would be
|
|
6685
|
+
// worse than waiting for the next natural restart.
|
|
6686
|
+
if (this.isPaused || this.healthCheckPaused || this.getProcessStatus() === "stopped" || !this.tmux)
|
|
6687
|
+
return;
|
|
6688
|
+
const generation = this.spawnGeneration;
|
|
6689
|
+
const idle = await this.waitForAuthoritativePaneIdle(idleBudgetMs, idlePollMs);
|
|
6690
|
+
// Another spawn (wake/restart) completed while waiting — it owns the pane
|
|
6691
|
+
// now, and respawning here would kill its fresh process mid-turn.
|
|
6692
|
+
if (generation !== this.spawnGeneration)
|
|
6693
|
+
return;
|
|
6694
|
+
if (!idle)
|
|
6695
|
+
return;
|
|
6696
|
+
if (this.startupAborted || this.isPaused || this.healthCheckPaused || this.getProcessStatus() === "stopped")
|
|
6697
|
+
return;
|
|
6698
|
+
// Final re-verification under pane-write exclusion: no delivery can paste
|
|
6699
|
+
// a new turn between this verdict and the respawn it authorizes. The
|
|
6700
|
+
// spawn itself runs outside the lock (its dialog dismissal re-takes it).
|
|
6701
|
+
let claimed = false;
|
|
6702
|
+
await this.paneWriteLock.run(async () => {
|
|
6703
|
+
const stillIdle = await this.isPaneAuthoritativelyIdle();
|
|
6704
|
+
if (generation !== this.spawnGeneration)
|
|
6705
|
+
return;
|
|
6706
|
+
if (!stillIdle)
|
|
6707
|
+
return;
|
|
6708
|
+
// The final capture awaited above: re-check the full guards here, not
|
|
6709
|
+
// just generation and idle — pause needs no generation bump to take
|
|
6710
|
+
// effect, and respawning a freshly paused pane strands its state.
|
|
6711
|
+
if (this.startupAborted || this.isPaused || this.healthCheckPaused || this.getProcessStatus() === "stopped")
|
|
6712
|
+
return;
|
|
6713
|
+
this.beginSpawn();
|
|
6714
|
+
claimed = true;
|
|
6715
|
+
});
|
|
6716
|
+
if (!claimed)
|
|
6717
|
+
return;
|
|
6718
|
+
try {
|
|
6719
|
+
this.saveSessionId();
|
|
6720
|
+
const ready = await this.trySpawn(true, this.wakeBudgetMs(30_000));
|
|
6721
|
+
this.transcriptMonitor?.resetOffset();
|
|
6722
|
+
if (ready)
|
|
6723
|
+
this.logger.info("Muse resumed direct after relay exhaustion (session preserved)");
|
|
6724
|
+
else
|
|
6725
|
+
this.logger.warn("Muse direct resume after relay exhaustion did not become ready — next natural restart retries");
|
|
6726
|
+
}
|
|
6727
|
+
catch (err) {
|
|
6728
|
+
this.logger.warn({ reason: err instanceof Error ? err.message : String(err) }, "Muse direct resume after relay exhaustion failed — next natural restart retries");
|
|
6729
|
+
}
|
|
6730
|
+
finally {
|
|
6731
|
+
this.endSpawn();
|
|
6732
|
+
}
|
|
6347
6733
|
})().finally(() => { this.museRelayFallbackInFlight = null; });
|
|
6348
6734
|
return this.museRelayFallbackInFlight;
|
|
6349
6735
|
}
|
|
@@ -6458,6 +6844,21 @@ export class Daemon extends EventEmitter {
|
|
|
6458
6844
|
}
|
|
6459
6845
|
this.backend.writeConfig(backendConfig);
|
|
6460
6846
|
this.backend.preTrust?.(this.config.working_directory);
|
|
6847
|
+
// Bounded preflight for an owner left by a restart or another app. Codex
|
|
6848
|
+
// must not be launched as a competing resumer. The command's flock is the
|
|
6849
|
+
// atomic second fence for a race after this observation.
|
|
6850
|
+
if (!backendConfig.skipResume && this.backend.resumeOwner) {
|
|
6851
|
+
let owner = null;
|
|
6852
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
6853
|
+
owner = this.backend.resumeOwner(this.config.working_directory);
|
|
6854
|
+
if (owner === null)
|
|
6855
|
+
break;
|
|
6856
|
+
if (attempt < 2)
|
|
6857
|
+
await new Promise(r => setTimeout(r, 1_000));
|
|
6858
|
+
}
|
|
6859
|
+
if (owner !== null)
|
|
6860
|
+
throw new CodexResumeConflictError(owner);
|
|
6861
|
+
}
|
|
6461
6862
|
// Resolve working directory (e.g. symlink for hidden paths)
|
|
6462
6863
|
const resolvedCwd = this.backend.resolveWorkingDirectory?.(this.config.working_directory, this.name) ?? this.config.working_directory;
|
|
6463
6864
|
// Generate a fresh per-instance agent token each spawn. agent-cli reads
|
|
@@ -6619,6 +7020,10 @@ export class Daemon extends EventEmitter {
|
|
|
6619
7020
|
let lastTransient = null;
|
|
6620
7021
|
let captureFailures = 0;
|
|
6621
7022
|
let attempts = 0;
|
|
7023
|
+
// A safety-critical startup choice is one-shot. If the CLI has not
|
|
7024
|
+
// repainted after Enter, let the following hold-only entry report it
|
|
7025
|
+
// instead of sending a second Enter into a possibly changed screen.
|
|
7026
|
+
const attemptedSafetyChoices = new Set();
|
|
6622
7027
|
do {
|
|
6623
7028
|
attempts++;
|
|
6624
7029
|
let pane;
|
|
@@ -6655,6 +7060,11 @@ export class Daemon extends EventEmitter {
|
|
|
6655
7060
|
lastDialog = null;
|
|
6656
7061
|
for (const dialog of startupDialogs) {
|
|
6657
7062
|
if (Daemon.dialogMatches(dialog, pane)) {
|
|
7063
|
+
if (dialog.autoResolutionKey
|
|
7064
|
+
&& (attemptedSafetyChoices.has(dialog.autoResolutionKey)
|
|
7065
|
+
|| (this.autoResolvedDialogGeneration === this.spawnGeneration
|
|
7066
|
+
&& this.autoResolvedDialogKey === dialog.autoResolutionKey)))
|
|
7067
|
+
continue;
|
|
6658
7068
|
lastDialog = dialog;
|
|
6659
7069
|
cleanReadyPolls = 0;
|
|
6660
7070
|
// Start the parked clock for every delivery-blocking dialog, exact
|
|
@@ -6696,17 +7106,38 @@ export class Daemon extends EventEmitter {
|
|
|
6696
7106
|
// Restart is exactly when inbound messages pile up, and nothing gates
|
|
6697
7107
|
// delivery on `spawning`. Take the pane lock for the key sequence so a
|
|
6698
7108
|
// queued message cannot be pasted into a half-dismissed trust dialog.
|
|
6699
|
-
await this.paneWriteLock.run(async () => {
|
|
7109
|
+
const sent = await this.paneWriteLock.run(async () => {
|
|
7110
|
+
// The capture above may have gone stale while waiting for the
|
|
7111
|
+
// write lock. Trust/other safety prompts must still be the
|
|
7112
|
+
// CURRENT menu, with the same safe cursor, at the instant of
|
|
7113
|
+
// the key send. A changed pane falls through to the next scan.
|
|
7114
|
+
if (dialog.inputBlocked) {
|
|
7115
|
+
const currentPane = await this.tmux.capturePane();
|
|
7116
|
+
if (!Daemon.dialogMatches(dialog, currentPane))
|
|
7117
|
+
return false;
|
|
7118
|
+
}
|
|
7119
|
+
if (dialog.autoResolutionKey) {
|
|
7120
|
+
attemptedSafetyChoices.add(dialog.autoResolutionKey);
|
|
7121
|
+
this.autoResolvedDialogGeneration = this.spawnGeneration;
|
|
7122
|
+
this.autoResolvedDialogKey = dialog.autoResolutionKey;
|
|
7123
|
+
}
|
|
6700
7124
|
for (const key of dialog.keys) {
|
|
6701
7125
|
if (key === "Up" || key === "Down" || key === "Enter" || key === "Escape") {
|
|
6702
|
-
await this.tmux.sendSpecialKey(key)
|
|
7126
|
+
if (!await this.tmux.sendSpecialKey(key))
|
|
7127
|
+
return false;
|
|
6703
7128
|
}
|
|
6704
7129
|
else {
|
|
6705
|
-
await this.tmux.sendKeys(key)
|
|
7130
|
+
if (!await this.tmux.sendKeys(key))
|
|
7131
|
+
return false;
|
|
6706
7132
|
}
|
|
6707
7133
|
await new Promise(r => setTimeout(r, 200));
|
|
6708
7134
|
}
|
|
7135
|
+
return true;
|
|
6709
7136
|
});
|
|
7137
|
+
if (!sent) {
|
|
7138
|
+
matched = true;
|
|
7139
|
+
break;
|
|
7140
|
+
}
|
|
6710
7141
|
// Wait for next screen to render — bounded by what is left of the budget.
|
|
6711
7142
|
const renderWait = Math.max(0, Math.min(10_000, remaining()));
|
|
6712
7143
|
if (this.controlClient) {
|
|
@@ -6816,26 +7247,34 @@ export class Daemon extends EventEmitter {
|
|
|
6816
7247
|
}
|
|
6817
7248
|
catch { /* best effort */ }
|
|
6818
7249
|
}
|
|
6819
|
-
|
|
7250
|
+
/** The returned ID was durably checkpointed or already matched on disk. */
|
|
7251
|
+
saveSessionId(pane) {
|
|
6820
7252
|
// When a resume failure has forced a fresh start, don't persist the stale id
|
|
6821
7253
|
// back from statusline.json — that would re-arm --continue and re-loop.
|
|
6822
7254
|
if (this.skipResume)
|
|
6823
|
-
return;
|
|
7255
|
+
return null;
|
|
6824
7256
|
try {
|
|
6825
|
-
const sid = this.backend?.getSessionId();
|
|
6826
|
-
if (!sid)
|
|
6827
|
-
|
|
7257
|
+
const sid = this.backend?.getSessionId(pane);
|
|
7258
|
+
if (!sid) {
|
|
7259
|
+
if (this.backend?.hasUnconfirmedSessionIdentity?.() && !this.codexSessionIdentityHoldNotified) {
|
|
7260
|
+
this.codexSessionIdentityHoldNotified = true;
|
|
7261
|
+
this.emit("codex_session_identity_unconfirmed", { name: this.name });
|
|
7262
|
+
}
|
|
7263
|
+
return null;
|
|
7264
|
+
}
|
|
7265
|
+
this.codexSessionIdentityHoldNotified = false;
|
|
6828
7266
|
const path = join(this.instanceDir, "session-id");
|
|
6829
7267
|
// Idle observations happen after turns. Avoid needless writes (and mtime
|
|
6830
7268
|
// churn) while still updating the marker when /new changes the session.
|
|
6831
7269
|
try {
|
|
6832
7270
|
if (readFileSync(path, "utf-8").trim() === sid)
|
|
6833
|
-
return;
|
|
7271
|
+
return sid;
|
|
6834
7272
|
}
|
|
6835
7273
|
catch { /* first checkpoint */ }
|
|
6836
7274
|
writeFileSync(path, sid);
|
|
6837
7275
|
this.sessionCheckpointWarningEmitted = false;
|
|
6838
7276
|
this.logger.debug("Session id checkpointed");
|
|
7277
|
+
return sid;
|
|
6839
7278
|
}
|
|
6840
7279
|
catch (err) {
|
|
6841
7280
|
// Session discovery is best-effort and must never break the pane state
|
|
@@ -6844,8 +7283,24 @@ export class Daemon extends EventEmitter {
|
|
|
6844
7283
|
this.sessionCheckpointWarningEmitted = true;
|
|
6845
7284
|
this.logger.warn({ err: err.message }, "Session id checkpoint failed");
|
|
6846
7285
|
}
|
|
7286
|
+
return null;
|
|
6847
7287
|
}
|
|
6848
7288
|
}
|
|
7289
|
+
checkpointLivePaneBeforeReplacement(pane) {
|
|
7290
|
+
// A pre-upgrade Codex pane can provide stronger, exact ownership
|
|
7291
|
+
// evidence than an old crash-loop heuristic. Discover it before
|
|
7292
|
+
// saveSessionId's skip-resume guard, then arm resume only if the
|
|
7293
|
+
// checkpoint was actually durable. Never fall back to --last.
|
|
7294
|
+
const exactCodexId = this.backend?.binaryName === "codex" ? this.backend.getSessionId(pane) : null;
|
|
7295
|
+
if (exactCodexId)
|
|
7296
|
+
this.skipResume = false;
|
|
7297
|
+
const recoveredId = this.saveSessionId(pane);
|
|
7298
|
+
if (exactCodexId && !recoveredId)
|
|
7299
|
+
this.skipResume = true;
|
|
7300
|
+
return recoveredId;
|
|
7301
|
+
}
|
|
7302
|
+
/** Only a live pre-upgrade pane can provide this positive migration proof. */
|
|
7303
|
+
getRecoveredLegacyCodexSessionId() { return this.recoveredLegacyCodexSessionId; }
|
|
6849
7304
|
readContextPercentage() {
|
|
6850
7305
|
return this.backend?.getContextUsage() ?? 0;
|
|
6851
7306
|
}
|