@songsid/agend 2.1.6-beta.15 → 2.1.6-beta.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backend/codex.d.ts +1 -0
- package/dist/backend/codex.js +106 -7
- package/dist/backend/codex.js.map +1 -1
- package/dist/backend/muse.d.ts +28 -0
- package/dist/backend/muse.js +52 -0
- package/dist/backend/muse.js.map +1 -1
- package/dist/backend/types.d.ts +1 -1
- package/dist/daemon.d.ts +49 -6
- package/dist/daemon.js +221 -30
- package/dist/daemon.js.map +1 -1
- package/dist/instance-lifecycle.d.ts +17 -0
- package/dist/instance-lifecycle.js +111 -5
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/locale.js +6 -0
- package/dist/locale.js.map +1 -1
- package/dist/usage/providers.js +19 -6
- package/dist/usage/providers.js.map +1 -1
- package/package.json +1 -1
package/dist/daemon.d.ts
CHANGED
|
@@ -217,9 +217,13 @@ export declare class BackendUnreachableStartupError extends Error {
|
|
|
217
217
|
* prompt. A TUI that is still completing its first redraw can swallow Enter.
|
|
218
218
|
*
|
|
219
219
|
* Since the adaptive settle (see {@link waitForPasteSettle}) this value is the
|
|
220
|
-
* *
|
|
221
|
-
*
|
|
222
|
-
*
|
|
220
|
+
* *minimum* settle window for the first delivery — it prevents Enter from firing
|
|
221
|
+
* before the minimum elapses even when the paste's own render goes quiet early.
|
|
222
|
+
* For observed deliveries the quiet-exit path in waitForPasteSettle now checks
|
|
223
|
+
* `now >= fallbackDeadline`, so 1750ms governs every first delivery regardless
|
|
224
|
+
* of whether output is observed. For unobserved deliveries (no control mode,
|
|
225
|
+
* native-queue handoff, mid-wait reconnect, or a paste that never visibly
|
|
226
|
+
* renders) it is the flat fallback delay, same as before.
|
|
223
227
|
*/
|
|
224
228
|
export declare class FirstDeliveryDelay {
|
|
225
229
|
private readyAt;
|
|
@@ -256,9 +260,15 @@ export interface PasteSettleResult {
|
|
|
256
260
|
* So: watch `lastOutputAt` and send Enter only once the paste's own render has
|
|
257
261
|
* been quiet for {@link PASTE_QUIET_MS}. Bounded both ways —
|
|
258
262
|
*
|
|
259
|
-
* - never earlier than
|
|
260
|
-
*
|
|
261
|
-
*
|
|
263
|
+
* - never earlier than `fallbackMs` from settle start: `fallbackMs` is a
|
|
264
|
+
* **minimum** settle window, not merely a fallback. For first deliveries
|
|
265
|
+
* (fallbackMs = 1750ms) the compositor may render quickly then go quiet at
|
|
266
|
+
* ~550ms while still initialising — the minimum ensures Enter is held until
|
|
267
|
+
* 1750ms regardless of observed output. This applies to every backend's first
|
|
268
|
+
* delivery after ready, not only codex/Luna Reserve. For normal deliveries
|
|
269
|
+
* (fallbackMs = 500ms ≈ PASTE_QUIET_MS) the minimum adds at most a few ms
|
|
270
|
+
* and is effectively unchanged. When the paste produces no observable output
|
|
271
|
+
* at all (`usedFallback`), the same `fallbackMs` deadline governs;
|
|
262
272
|
* - never later than {@link PASTE_SETTLE_CAP_MS} after the paste (`capHit`),
|
|
263
273
|
* so a chatty pane cannot stall delivery.
|
|
264
274
|
*
|
|
@@ -610,6 +620,14 @@ export declare class Daemon extends EventEmitter {
|
|
|
610
620
|
getProcessStatus(): "running" | "crashed" | "stopped";
|
|
611
621
|
get lastPausedAt(): number | null;
|
|
612
622
|
private getPauseWakeState;
|
|
623
|
+
/**
|
|
624
|
+
* Codex can show an exhausted-account notice while the user is switching to
|
|
625
|
+
* a reserve account. Once the live composer and Context footer are back, the
|
|
626
|
+
* notice is scrollback, not a reason to pause and send /quit.
|
|
627
|
+
*/
|
|
628
|
+
private isCodexLivePaneSnapshot;
|
|
629
|
+
/** Re-check the current pane after an async quota probe before pausing it. */
|
|
630
|
+
isCodexLivePane(): Promise<boolean>;
|
|
613
631
|
/** Whether this instance is in a crash loop (3+ consecutive crashes). */
|
|
614
632
|
get isCrashLoop(): boolean;
|
|
615
633
|
/** The most recent error type detected by the error monitor (e.g. "rate_limit", "auth_error"). */
|
|
@@ -811,8 +829,33 @@ export declare class Daemon extends EventEmitter {
|
|
|
811
829
|
* `y` as a brand-new user message.
|
|
812
830
|
*/
|
|
813
831
|
private confirmBackendClearDialog;
|
|
832
|
+
/**
|
|
833
|
+
* The last time AgEnD itself asked the CLI to go away, and why (#927). Read
|
|
834
|
+
* when a pane dies, to tell "AgEnD stopped it" from "the CLI exited on its
|
|
835
|
+
* own" — a codex that exits 0 on its own and one AgEnD sent /quit look the
|
|
836
|
+
* same from the pane.
|
|
837
|
+
*/
|
|
838
|
+
private lastStopRequest;
|
|
839
|
+
/** Record, and log, that AgEnD is about to stop the CLI. */
|
|
840
|
+
private noteStopRequest;
|
|
841
|
+
/** Capture the pane's recent output with escape codes stripped. Never throws. */
|
|
842
|
+
private capturePaneOutput;
|
|
843
|
+
/**
|
|
844
|
+
* One record per pane death: exit status, when, who, and what the CLI printed
|
|
845
|
+
* last — the line that was missing when a codex on Luna Reserve died with
|
|
846
|
+
* status 0 and daemon.log said only "exited normally" (#927). Logging only;
|
|
847
|
+
* the caller decides what happens next.
|
|
848
|
+
*/
|
|
849
|
+
private logPaneDeath;
|
|
814
850
|
/** Send the backend-specific graceful quit command/key sequence. */
|
|
815
851
|
private sendQuitSequence;
|
|
852
|
+
/**
|
|
853
|
+
* Kiro drops `/quit` while a turn is running. Stop that turn first, then use
|
|
854
|
+
* the same delivery readiness gate that protects Enter from the busy pane.
|
|
855
|
+
* Returning false is fail-closed: the caller skips `/quit` and enters the
|
|
856
|
+
* bounded SIGTERM -> SIGKILL fallback instead of typing into an unknown pane.
|
|
857
|
+
*/
|
|
858
|
+
private drainBusyKiroForStop;
|
|
816
859
|
stop(): Promise<void>;
|
|
817
860
|
getHangDetector(): HangDetector | null;
|
|
818
861
|
getInstanceState(): InstanceState | "paused";
|
package/dist/daemon.js
CHANGED
|
@@ -435,6 +435,27 @@ const DELIVERY_SPAWN_RACE_MAX_ROUNDS = 3;
|
|
|
435
435
|
*/
|
|
436
436
|
const MUSE_DIRECT_RESUME_IDLE_BUDGET_MS = 10 * 60_000;
|
|
437
437
|
const MUSE_DIRECT_RESUME_IDLE_POLL_MS = 5_000;
|
|
438
|
+
/** Kiro can drop /quit while a turn is running; cancel it and wait for its
|
|
439
|
+
* verified prompt before asking the native process to exit. Each instance is
|
|
440
|
+
* bounded so a fleet update can stop all daemons in parallel. */
|
|
441
|
+
const KIRO_STOP_IDLE_BUDGET_MS = 15_000;
|
|
442
|
+
const KIRO_STOP_QUIT_GRACE_MS = 5_000;
|
|
443
|
+
const KIRO_STOP_SIGTERM_GRACE_MS = 2_000;
|
|
444
|
+
/**
|
|
445
|
+
* How recently AgEnD must have asked the CLI to stop for a pane death to be
|
|
446
|
+
* attributed to AgEnD rather than to the CLI itself (#927). Covers the longest
|
|
447
|
+
* quit grace plus the SIGTERM/SIGKILL fallback, and one health tick more.
|
|
448
|
+
*/
|
|
449
|
+
const STOP_ATTRIBUTION_WINDOW_MS = 90_000;
|
|
450
|
+
/** Lines of pane output kept with a death record: enough for a vendor error. */
|
|
451
|
+
const DEATH_OUTPUT_LINES = 20;
|
|
452
|
+
/** The last `lines` non-empty lines of a captured pane, escape codes already stripped. */
|
|
453
|
+
function paneTail(output, lines = DEATH_OUTPUT_LINES) {
|
|
454
|
+
if (!output)
|
|
455
|
+
return undefined;
|
|
456
|
+
const tail = output.split("\n").filter(line => line.trim()).slice(-lines).join("\n");
|
|
457
|
+
return tail ? tail.slice(-2_000) : undefined;
|
|
458
|
+
}
|
|
438
459
|
/**
|
|
439
460
|
* Startup failed because the CLI's backend is unreachable (see backend-outage.ts).
|
|
440
461
|
* The session-id is deliberately KEPT; the fleet schedules a delayed retry.
|
|
@@ -498,9 +519,13 @@ const SPAWN_SETTLE_MAX_WAIT_MS = 60_000;
|
|
|
498
519
|
* prompt. A TUI that is still completing its first redraw can swallow Enter.
|
|
499
520
|
*
|
|
500
521
|
* Since the adaptive settle (see {@link waitForPasteSettle}) this value is the
|
|
501
|
-
* *
|
|
502
|
-
*
|
|
503
|
-
*
|
|
522
|
+
* *minimum* settle window for the first delivery — it prevents Enter from firing
|
|
523
|
+
* before the minimum elapses even when the paste's own render goes quiet early.
|
|
524
|
+
* For observed deliveries the quiet-exit path in waitForPasteSettle now checks
|
|
525
|
+
* `now >= fallbackDeadline`, so 1750ms governs every first delivery regardless
|
|
526
|
+
* of whether output is observed. For unobserved deliveries (no control mode,
|
|
527
|
+
* native-queue handoff, mid-wait reconnect, or a paste that never visibly
|
|
528
|
+
* renders) it is the flat fallback delay, same as before.
|
|
504
529
|
*/
|
|
505
530
|
export class FirstDeliveryDelay {
|
|
506
531
|
readyAt = 0;
|
|
@@ -537,9 +562,15 @@ const PASTE_SETTLE_POLL_MS = 100;
|
|
|
537
562
|
* So: watch `lastOutputAt` and send Enter only once the paste's own render has
|
|
538
563
|
* been quiet for {@link PASTE_QUIET_MS}. Bounded both ways —
|
|
539
564
|
*
|
|
540
|
-
* - never earlier than
|
|
541
|
-
*
|
|
542
|
-
*
|
|
565
|
+
* - never earlier than `fallbackMs` from settle start: `fallbackMs` is a
|
|
566
|
+
* **minimum** settle window, not merely a fallback. For first deliveries
|
|
567
|
+
* (fallbackMs = 1750ms) the compositor may render quickly then go quiet at
|
|
568
|
+
* ~550ms while still initialising — the minimum ensures Enter is held until
|
|
569
|
+
* 1750ms regardless of observed output. This applies to every backend's first
|
|
570
|
+
* delivery after ready, not only codex/Luna Reserve. For normal deliveries
|
|
571
|
+
* (fallbackMs = 500ms ≈ PASTE_QUIET_MS) the minimum adds at most a few ms
|
|
572
|
+
* and is effectively unchanged. When the paste produces no observable output
|
|
573
|
+
* at all (`usedFallback`), the same `fallbackMs` deadline governs;
|
|
543
574
|
* - never later than {@link PASTE_SETTLE_CAP_MS} after the paste (`capHit`),
|
|
544
575
|
* so a chatty pane cannot stall delivery.
|
|
545
576
|
*
|
|
@@ -563,7 +594,13 @@ export async function waitForPasteSettle(client, windowId, pasteStartedAt, fallb
|
|
|
563
594
|
const last = client.getLastOutputAt(windowId);
|
|
564
595
|
if (last != null && last > pasteStartedAt) {
|
|
565
596
|
observed = true;
|
|
566
|
-
|
|
597
|
+
// Quiet for PASTE_QUIET_MS AND the minimum settle window has elapsed.
|
|
598
|
+
// The minimum (fallbackDeadline) is the key fix for Luna Reserve: the
|
|
599
|
+
// TUI's first compositor render produces output quickly then goes quiet,
|
|
600
|
+
// but Enter sent at the 500ms quiet point lands while the compositor is
|
|
601
|
+
// still initialising and is swallowed. Holding until fallbackDeadline
|
|
602
|
+
// (1750ms for first deliveries, 500ms for normal ones) eliminates the race.
|
|
603
|
+
if (now - last >= PASTE_QUIET_MS && now >= fallbackDeadline) {
|
|
567
604
|
return { settleMs: now - settleStart, observedPostPasteOutput: true, capHit: false, usedFallback: false };
|
|
568
605
|
}
|
|
569
606
|
}
|
|
@@ -575,9 +612,12 @@ export async function waitForPasteSettle(client, windowId, pasteStartedAt, fallb
|
|
|
575
612
|
}
|
|
576
613
|
// Sleep to the next decision point, at most one poll tick — so the normal
|
|
577
614
|
// paths return at their exact deadlines instead of a poll-width late.
|
|
615
|
+
// When output has been observed but we are still inside the minimum window,
|
|
616
|
+
// sleep to the later of (last quiet deadline) and (fallback deadline) so
|
|
617
|
+
// the loop does not busy-spin between quiet and minimum.
|
|
578
618
|
let wake = capDeadline;
|
|
579
619
|
if (last != null && last > pasteStartedAt)
|
|
580
|
-
wake = Math.min(wake, last + PASTE_QUIET_MS);
|
|
620
|
+
wake = Math.min(wake, Math.max(last + PASTE_QUIET_MS, fallbackDeadline));
|
|
581
621
|
else
|
|
582
622
|
wake = Math.min(wake, fallbackDeadline);
|
|
583
623
|
await new Promise(r => setTimeout(r, Math.min(PASTE_SETTLE_POLL_MS, Math.max(1, wake - now))));
|
|
@@ -1117,6 +1157,42 @@ export class Daemon extends EventEmitter {
|
|
|
1117
1157
|
}
|
|
1118
1158
|
get lastPausedAt() { return this.autoPauseController.lastPausedAt; }
|
|
1119
1159
|
getPauseWakeState() { return this.pauseWakeState; }
|
|
1160
|
+
/**
|
|
1161
|
+
* Codex can show an exhausted-account notice while the user is switching to
|
|
1162
|
+
* a reserve account. Once the live composer and Context footer are back, the
|
|
1163
|
+
* notice is scrollback, not a reason to pause and send /quit.
|
|
1164
|
+
*/
|
|
1165
|
+
isCodexLivePaneSnapshot(pane) {
|
|
1166
|
+
if (this.backend?.binaryName !== "codex")
|
|
1167
|
+
return false;
|
|
1168
|
+
try {
|
|
1169
|
+
if (this.backend.isDeliveryInputReadyPane?.(pane) !== true)
|
|
1170
|
+
return false;
|
|
1171
|
+
const busy = this.backend.getBusyPattern?.();
|
|
1172
|
+
if (busy) {
|
|
1173
|
+
busy.lastIndex = 0;
|
|
1174
|
+
if (busy.test(pane))
|
|
1175
|
+
return false;
|
|
1176
|
+
}
|
|
1177
|
+
const ready = this.backend.getReadyPattern();
|
|
1178
|
+
ready.lastIndex = 0;
|
|
1179
|
+
return ready.test(pane);
|
|
1180
|
+
}
|
|
1181
|
+
catch {
|
|
1182
|
+
return false;
|
|
1183
|
+
}
|
|
1184
|
+
}
|
|
1185
|
+
/** Re-check the current pane after an async quota probe before pausing it. */
|
|
1186
|
+
async isCodexLivePane() {
|
|
1187
|
+
if (this.backend?.binaryName !== "codex" || !this.tmux)
|
|
1188
|
+
return false;
|
|
1189
|
+
try {
|
|
1190
|
+
return this.isCodexLivePaneSnapshot(await this.tmux.capturePane());
|
|
1191
|
+
}
|
|
1192
|
+
catch {
|
|
1193
|
+
return false;
|
|
1194
|
+
}
|
|
1195
|
+
}
|
|
1120
1196
|
/** Whether this instance is in a crash loop (3+ consecutive crashes). */
|
|
1121
1197
|
get isCrashLoop() {
|
|
1122
1198
|
return this.crashCount >= 3;
|
|
@@ -1888,6 +1964,9 @@ export class Daemon extends EventEmitter {
|
|
|
1888
1964
|
// Normal exit (e.g. user Ctrl+C or /exit) — no crash, no respawn
|
|
1889
1965
|
if (paneStatus && exitCode === 0) {
|
|
1890
1966
|
this.setProcessStatus("stopped");
|
|
1967
|
+
// Status 0 is not proof of a clean exit: a codex that hits a quota
|
|
1968
|
+
// wall exits 0 too. Capture what it printed before the window goes.
|
|
1969
|
+
this.logPaneDeath(cliLabel, exitCode, await this.capturePaneOutput());
|
|
1891
1970
|
this.logger.info("CLI exited normally (code 0) — pausing health check");
|
|
1892
1971
|
await this.tmux.killWindow();
|
|
1893
1972
|
this.healthCheckPaused = true;
|
|
@@ -1964,6 +2043,7 @@ export class Daemon extends EventEmitter {
|
|
|
1964
2043
|
lastOutput = cleaned.trimEnd() || undefined;
|
|
1965
2044
|
}
|
|
1966
2045
|
catch { /* best effort — pane may already be gone */ }
|
|
2046
|
+
this.logPaneDeath(cliLabel, exitCode, lastOutput);
|
|
1967
2047
|
// Kill the dead window (remain-on-exit keeps it around) before respawn
|
|
1968
2048
|
if (paneStatus) {
|
|
1969
2049
|
await this.tmux.killWindow();
|
|
@@ -2035,7 +2115,7 @@ export class Daemon extends EventEmitter {
|
|
|
2035
2115
|
this.crashTimestamps = this.crashTimestamps.filter(t => t > Date.now() - crashWindowMs);
|
|
2036
2116
|
if (this.crashTimestamps.length >= 3) {
|
|
2037
2117
|
this.healthCheckPaused = true;
|
|
2038
|
-
this.logger.error({ crashesInWindow: this.crashTimestamps.length }, "3+ crashes in 5 minutes — pausing respawn");
|
|
2118
|
+
this.logger.error({ crashesInWindow: this.crashTimestamps.length, lastExitCode: exitCode ?? null, lastOutput: paneTail(lastOutput) ?? null }, "3+ crashes in 5 minutes — pausing respawn");
|
|
2039
2119
|
// P1: Persist crash state so next process restart skips resume
|
|
2040
2120
|
try {
|
|
2041
2121
|
writeFileSync(join(this.instanceDir, "crash-state.json"), JSON.stringify({
|
|
@@ -2055,7 +2135,7 @@ export class Daemon extends EventEmitter {
|
|
|
2055
2135
|
this.crashCount++;
|
|
2056
2136
|
this.lastCrashAt = Date.now();
|
|
2057
2137
|
if (this.crashCount > max_retries) {
|
|
2058
|
-
this.logger.error({ crashCount: this.crashCount, maxRetries: max_retries }, "Max crash retries exceeded — not respawning");
|
|
2138
|
+
this.logger.error({ crashCount: this.crashCount, maxRetries: max_retries, lastExitCode: exitCode ?? null, lastOutput: paneTail(lastOutput) ?? null }, "Max crash retries exceeded — not respawning");
|
|
2059
2139
|
this.healthCheckPaused = true;
|
|
2060
2140
|
this.emitSupervisionEnded(`it crashed ${this.crashCount} times, exceeding restart_policy.max_retries (${max_retries})`, "Check the logs for the cause, then restart it.");
|
|
2061
2141
|
return; // don't schedule next — given up
|
|
@@ -2359,6 +2439,7 @@ export class Daemon extends EventEmitter {
|
|
|
2359
2439
|
// rebaselining the occurrence count while the error is still displayed — one
|
|
2360
2440
|
// notification, a false recovery log, then silence.
|
|
2361
2441
|
const looksReady = () => !busyPattern?.test(pane) && readyPattern.test(pane);
|
|
2442
|
+
const codexLivePane = this.isCodexLivePaneSnapshot(pane);
|
|
2362
2443
|
// State: waiting for recovery. A missing/outdated ready pattern must not
|
|
2363
2444
|
// suppress every future error forever, so the gate has a hard deadline.
|
|
2364
2445
|
if (this.errorWaitingForRecovery) {
|
|
@@ -2422,6 +2503,17 @@ export class Daemon extends EventEmitter {
|
|
|
2422
2503
|
const key = Daemon.errorPatternKey(ep);
|
|
2423
2504
|
const count = countMatches(ep.pattern);
|
|
2424
2505
|
const seen = this.lastErrorCount.get(key) ?? 0;
|
|
2506
|
+
// The generic Codex usage-limit line remains in scrollback after the
|
|
2507
|
+
// user selects Luna Reserve. A live composer + Context footer is positive
|
|
2508
|
+
// evidence that the pane recovered; baseline the stale occurrence so it
|
|
2509
|
+
// cannot reach the destructive quota pause path.
|
|
2510
|
+
if (codexLivePane && ep.type === "quota" && ep.action === "pause"
|
|
2511
|
+
&& ep.message.startsWith("Codex usage limit reached")) {
|
|
2512
|
+
if (count > 0)
|
|
2513
|
+
this.lastErrorCount.set(key, count);
|
|
2514
|
+
this.logger.debug("Codex usage-limit text is stale — live reserve pane is running");
|
|
2515
|
+
continue;
|
|
2516
|
+
}
|
|
2425
2517
|
if (count <= seen) {
|
|
2426
2518
|
// Occurrences scrolled out of the capture buffer → lower the baseline
|
|
2427
2519
|
// so a future re-occurrence still counts as new (no permanent suppress).
|
|
@@ -2616,12 +2708,61 @@ export class Daemon extends EventEmitter {
|
|
|
2616
2708
|
// is a successful no-op here, not an error.
|
|
2617
2709
|
this.logger.debug({ dialog: dialog.description }, "Backend clear dialog did not appear");
|
|
2618
2710
|
}
|
|
2711
|
+
/**
|
|
2712
|
+
* The last time AgEnD itself asked the CLI to go away, and why (#927). Read
|
|
2713
|
+
* when a pane dies, to tell "AgEnD stopped it" from "the CLI exited on its
|
|
2714
|
+
* own" — a codex that exits 0 on its own and one AgEnD sent /quit look the
|
|
2715
|
+
* same from the pane.
|
|
2716
|
+
*/
|
|
2717
|
+
lastStopRequest = null;
|
|
2718
|
+
/** Record, and log, that AgEnD is about to stop the CLI. */
|
|
2719
|
+
noteStopRequest(via, reason) {
|
|
2720
|
+
this.lastStopRequest = { via, reason, at: Date.now() };
|
|
2721
|
+
this.logger.info({ via, reason }, `AgEnD is stopping the CLI: ${via} (${reason})`);
|
|
2722
|
+
}
|
|
2723
|
+
/** Capture the pane's recent output with escape codes stripped. Never throws. */
|
|
2724
|
+
async capturePaneOutput() {
|
|
2725
|
+
try {
|
|
2726
|
+
const raw = await this.tmux?.capturePaneWithHistory(50);
|
|
2727
|
+
const cleaned = raw?.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
|
|
2728
|
+
return cleaned?.trimEnd() || undefined;
|
|
2729
|
+
}
|
|
2730
|
+
catch {
|
|
2731
|
+
return undefined;
|
|
2732
|
+
}
|
|
2733
|
+
}
|
|
2734
|
+
/**
|
|
2735
|
+
* One record per pane death: exit status, when, who, and what the CLI printed
|
|
2736
|
+
* last — the line that was missing when a codex on Luna Reserve died with
|
|
2737
|
+
* status 0 and daemon.log said only "exited normally" (#927). Logging only;
|
|
2738
|
+
* the caller decides what happens next.
|
|
2739
|
+
*/
|
|
2740
|
+
logPaneDeath(cliLabel, exitCode, lastOutput) {
|
|
2741
|
+
const request = this.lastStopRequest;
|
|
2742
|
+
const sinceStopMs = request ? Date.now() - request.at : null;
|
|
2743
|
+
const byAgend = request !== null && sinceStopMs !== null && sinceStopMs <= STOP_ATTRIBUTION_WINDOW_MS;
|
|
2744
|
+
const record = {
|
|
2745
|
+
exitCode: exitCode ?? null,
|
|
2746
|
+
diedAt: new Date().toISOString(),
|
|
2747
|
+
initiatedBy: byAgend ? "agend" : "cli",
|
|
2748
|
+
...(byAgend ? { stopVia: request.via, stopReason: request.reason, sinceStopMs } : {}),
|
|
2749
|
+
lastOutput: paneTail(lastOutput) ?? null,
|
|
2750
|
+
};
|
|
2751
|
+
if (byAgend) {
|
|
2752
|
+
this.logger.info(record, `${cliLabel} exited after AgEnD stopped it (${request.via}: ${request.reason})`);
|
|
2753
|
+
}
|
|
2754
|
+
else {
|
|
2755
|
+
this.logger.warn(record, `${cliLabel} exited on its own (exit code ${exitCode ?? "unknown"}) — last output recorded`);
|
|
2756
|
+
}
|
|
2757
|
+
}
|
|
2619
2758
|
/** Send the backend-specific graceful quit command/key sequence. */
|
|
2620
|
-
async sendQuitSequence() {
|
|
2759
|
+
async sendQuitSequence(reason = "unspecified") {
|
|
2621
2760
|
if (!this.tmux || !this.backend)
|
|
2622
2761
|
return false;
|
|
2623
2762
|
const quitCmd = this.backend.getQuitCommand();
|
|
2624
2763
|
const quitKey = this.backend.getQuitKey?.();
|
|
2764
|
+
if (quitCmd || quitKey)
|
|
2765
|
+
this.noteStopRequest(quitCmd ? `quit command ${quitCmd}` : `quit key ${quitKey}`, reason);
|
|
2625
2766
|
if (quitCmd) {
|
|
2626
2767
|
if (!await this.tmux.sendKeys(quitCmd))
|
|
2627
2768
|
return false;
|
|
@@ -2642,6 +2783,26 @@ export class Daemon extends EventEmitter {
|
|
|
2642
2783
|
}
|
|
2643
2784
|
return true;
|
|
2644
2785
|
}
|
|
2786
|
+
/**
|
|
2787
|
+
* Kiro drops `/quit` while a turn is running. Stop that turn first, then use
|
|
2788
|
+
* the same delivery readiness gate that protects Enter from the busy pane.
|
|
2789
|
+
* Returning false is fail-closed: the caller skips `/quit` and enters the
|
|
2790
|
+
* bounded SIGTERM -> SIGKILL fallback instead of typing into an unknown pane.
|
|
2791
|
+
*/
|
|
2792
|
+
async drainBusyKiroForStop(windowId) {
|
|
2793
|
+
if (this.backend?.binaryName !== "kiro-cli")
|
|
2794
|
+
return true;
|
|
2795
|
+
const readiness = await this.paneReadinessForDelivery(windowId);
|
|
2796
|
+
if (readiness === "ready")
|
|
2797
|
+
return true;
|
|
2798
|
+
if (readiness !== "busy")
|
|
2799
|
+
return false;
|
|
2800
|
+
const cancelKey = this.backend.getCancelKey?.() ?? "Escape";
|
|
2801
|
+
const sent = await this.tmux?.sendSpecialKey(cancelKey);
|
|
2802
|
+
if (!sent)
|
|
2803
|
+
return false;
|
|
2804
|
+
return this.waitForPaneReadyForDelivery(windowId, KIRO_STOP_IDLE_BUDGET_MS);
|
|
2805
|
+
}
|
|
2645
2806
|
async stop() {
|
|
2646
2807
|
this.logger.info("Stopping daemon instance");
|
|
2647
2808
|
this.turnReplyGuard.reset();
|
|
@@ -2667,11 +2828,18 @@ export class Daemon extends EventEmitter {
|
|
|
2667
2828
|
this.saveSessionId();
|
|
2668
2829
|
this.healthCheckPaused = true;
|
|
2669
2830
|
let killed = false;
|
|
2670
|
-
const
|
|
2831
|
+
const windowId = this.tmux.getWindowId();
|
|
2832
|
+
const kiroReady = windowId ? await this.drainBusyKiroForStop(windowId) : true;
|
|
2833
|
+
const quitSent = kiroReady && await this.sendQuitSequence("graceful stop");
|
|
2834
|
+
const quitGraceMs = this.backend?.binaryName === "kiro-cli"
|
|
2835
|
+
? KIRO_STOP_QUIT_GRACE_MS : 3_000;
|
|
2836
|
+
const sigtermGraceMs = this.backend?.binaryName === "kiro-cli"
|
|
2837
|
+
? KIRO_STOP_SIGTERM_GRACE_MS : 1_000;
|
|
2671
2838
|
if (quitSent) {
|
|
2672
|
-
// Wait up to
|
|
2839
|
+
// Wait up to the backend's bounded graceful-exit window, polling every
|
|
2840
|
+
// 200ms. A healthy CLI
|
|
2673
2841
|
// exits within ~1s; a longer wait just delays the force-kill fallback.
|
|
2674
|
-
for (let
|
|
2842
|
+
for (let elapsed = 0; elapsed < quitGraceMs; elapsed += 200) {
|
|
2675
2843
|
await new Promise(r => setTimeout(r, 200));
|
|
2676
2844
|
const status = await this.tmux.getPaneStatus();
|
|
2677
2845
|
if (!status || !status.alive) {
|
|
@@ -2681,9 +2849,9 @@ export class Daemon extends EventEmitter {
|
|
|
2681
2849
|
}
|
|
2682
2850
|
}
|
|
2683
2851
|
if (!killed) {
|
|
2684
|
-
this.logger.warn({ quitSent }, "CLI did not exit gracefully within
|
|
2685
|
-
await this.killProcessTree("SIGTERM");
|
|
2686
|
-
for (let
|
|
2852
|
+
this.logger.warn({ quitSent, quitGraceMs }, "CLI did not exit gracefully within its bounded quit grace — falling back to SIGTERM");
|
|
2853
|
+
await this.killProcessTree("SIGTERM", "graceful stop: the CLI outlived its quit grace");
|
|
2854
|
+
for (let elapsed = 0; elapsed < sigtermGraceMs; elapsed += 200) {
|
|
2687
2855
|
await new Promise(r => setTimeout(r, 200));
|
|
2688
2856
|
const status = await this.tmux.getPaneStatus();
|
|
2689
2857
|
if (!status || !status.alive) {
|
|
@@ -2694,7 +2862,7 @@ export class Daemon extends EventEmitter {
|
|
|
2694
2862
|
}
|
|
2695
2863
|
if (!killed) {
|
|
2696
2864
|
this.logger.warn("CLI process tree survived SIGTERM — falling back to SIGKILL");
|
|
2697
|
-
await this.killProcessTree("SIGKILL");
|
|
2865
|
+
await this.killProcessTree("SIGKILL", "graceful stop: the CLI survived SIGTERM");
|
|
2698
2866
|
await new Promise(r => setTimeout(r, 200));
|
|
2699
2867
|
}
|
|
2700
2868
|
// Always kill window — remain-on-exit keeps dead panes around after CLI exits
|
|
@@ -2796,6 +2964,9 @@ export class Daemon extends EventEmitter {
|
|
|
2796
2964
|
// into an ordinary idle-timeout pause of a busy instance.
|
|
2797
2965
|
const allowStuck = this.pauseAllowStuck;
|
|
2798
2966
|
this.pauseAllowStuck = false;
|
|
2967
|
+
// Why this pause, for the stop record (#927): an idle pause and an
|
|
2968
|
+
// auth-deferred one look the same from the pane.
|
|
2969
|
+
const pauseReason = allowStuck ? "pause (auth-deferred, stuck pane)" : "pause (idle)";
|
|
2799
2970
|
const pausableState = this.instanceState === "idle"
|
|
2800
2971
|
|| (allowStuck && this.instanceState === "stuck");
|
|
2801
2972
|
if (!pausableState || this.pasteQueueDepth > 0) {
|
|
@@ -2808,7 +2979,7 @@ export class Daemon extends EventEmitter {
|
|
|
2808
2979
|
const transition = (async () => {
|
|
2809
2980
|
try {
|
|
2810
2981
|
this.saveSessionId();
|
|
2811
|
-
await this.sendQuitSequence();
|
|
2982
|
+
await this.sendQuitSequence(pauseReason);
|
|
2812
2983
|
let exited = false;
|
|
2813
2984
|
for (let i = 0; i < 15; i++) {
|
|
2814
2985
|
await new Promise(r => setTimeout(r, 200));
|
|
@@ -2819,11 +2990,11 @@ export class Daemon extends EventEmitter {
|
|
|
2819
2990
|
}
|
|
2820
2991
|
}
|
|
2821
2992
|
if (!exited) {
|
|
2822
|
-
await this.killProcessTree("SIGTERM");
|
|
2993
|
+
await this.killProcessTree("SIGTERM", `${pauseReason}: the CLI outlived its quit grace`);
|
|
2823
2994
|
await new Promise(r => setTimeout(r, 1_000));
|
|
2824
2995
|
const status = await this.tmux?.getPaneStatus();
|
|
2825
2996
|
if (status?.alive) {
|
|
2826
|
-
await this.killProcessTree("SIGKILL");
|
|
2997
|
+
await this.killProcessTree("SIGKILL", `${pauseReason}: the CLI survived SIGTERM`);
|
|
2827
2998
|
await new Promise(r => setTimeout(r, 200));
|
|
2828
2999
|
}
|
|
2829
3000
|
}
|
|
@@ -4516,8 +4687,18 @@ export class Daemon extends EventEmitter {
|
|
|
4516
4687
|
this.logger.warn({ phase, generation }, "Spawn changed during the input-availability probe — refusing to send Enter into the replacement pane");
|
|
4517
4688
|
return false;
|
|
4518
4689
|
}
|
|
4519
|
-
if (probe.state === "clear")
|
|
4690
|
+
if (probe.state === "clear") {
|
|
4691
|
+
// Re-arm the first-delivery delay when a transient was actually observed
|
|
4692
|
+
// and then cleared, but ONLY from the pre-write phase. That phase runs
|
|
4693
|
+
// before consume() in writeMessageToPane, so the re-arm benefits the
|
|
4694
|
+
// current delivery's paste settle. Enter-path callers (initial-submit,
|
|
4695
|
+
// retries) run after consume() — re-arming there would leave the flag
|
|
4696
|
+
// for the NEXT unrelated delivery, which must not be slowed.
|
|
4697
|
+
if (observedDescription !== null && phase === "pre-write") {
|
|
4698
|
+
this.firstDeliveryDelay.recordReady();
|
|
4699
|
+
}
|
|
4520
4700
|
return true;
|
|
4701
|
+
}
|
|
4521
4702
|
if (probe.state === "active" && observedDescription !== probe.transient.description) {
|
|
4522
4703
|
observedDescription = probe.transient.description;
|
|
4523
4704
|
this.logger.info({ phase, transient: probe.transient.description, generation }, "CLI is still completing startup — waiting before sending Enter");
|
|
@@ -6464,7 +6645,7 @@ export class Daemon extends EventEmitter {
|
|
|
6464
6645
|
await this.failStartupIfBackendUnreachable();
|
|
6465
6646
|
if (this.backend.retriesResumeOnStartupFailure?.() !== false) {
|
|
6466
6647
|
this.logger.warn("Resume startup failed — retrying resume once before abandoning the session");
|
|
6467
|
-
await this.killProcessTree();
|
|
6648
|
+
await this.killProcessTree("SIGTERM", "spawn: clearing the previous CLI process");
|
|
6468
6649
|
await this.tmux.killWindow();
|
|
6469
6650
|
alive = await this.trySpawn(false, resumeBudget);
|
|
6470
6651
|
if (!alive) {
|
|
@@ -6490,7 +6671,7 @@ export class Daemon extends EventEmitter {
|
|
|
6490
6671
|
if (this.unprovenResumeFailures < Daemon.MAX_UNPROVEN_RESUME_FAILURES) {
|
|
6491
6672
|
// Keep the session and fail this attempt; the fleet retries with
|
|
6492
6673
|
// backoff, which is also how the backend-outage path behaves.
|
|
6493
|
-
await this.killProcessTree();
|
|
6674
|
+
await this.killProcessTree("SIGTERM", "spawn: clearing the previous CLI process");
|
|
6494
6675
|
await this.tmux.killWindow();
|
|
6495
6676
|
throw new Error(`CLI startup failed with a session to resume (attempt ${this.unprovenResumeFailures}/${Daemon.MAX_UNPROVEN_RESUME_FAILURES}) `
|
|
6496
6677
|
+ "— session kept, will retry");
|
|
@@ -6507,11 +6688,11 @@ export class Daemon extends EventEmitter {
|
|
|
6507
6688
|
// A fresh start that also failed retries once, as before. It never clears
|
|
6508
6689
|
// a session: nothing about a failed fresh launch says the stored
|
|
6509
6690
|
// conversation is unusable.
|
|
6510
|
-
await this.killProcessTree();
|
|
6691
|
+
await this.killProcessTree("SIGTERM", "spawn: clearing the previous CLI process");
|
|
6511
6692
|
await this.tmux.killWindow();
|
|
6512
6693
|
const retryAlive = await this.trySpawn(false, this.startupBudgetFor(false));
|
|
6513
6694
|
if (!retryAlive) {
|
|
6514
|
-
await this.killProcessTree();
|
|
6695
|
+
await this.killProcessTree("SIGTERM", "spawn: clearing the previous CLI process");
|
|
6515
6696
|
await this.tmux.killWindow();
|
|
6516
6697
|
throw new Error("CLI failed to start after retry");
|
|
6517
6698
|
}
|
|
@@ -6583,7 +6764,7 @@ export class Daemon extends EventEmitter {
|
|
|
6583
6764
|
if (!this.backendOutage?.isActive(this.backendKey()))
|
|
6584
6765
|
return;
|
|
6585
6766
|
this.logger.warn("Backend unreachable — keeping the session and failing startup for a delayed retry");
|
|
6586
|
-
await this.killProcessTree();
|
|
6767
|
+
await this.killProcessTree("SIGTERM", "startup: backend unreachable");
|
|
6587
6768
|
await this.tmux.killWindow();
|
|
6588
6769
|
throw new BackendUnreachableStartupError(this.backendKey());
|
|
6589
6770
|
}
|
|
@@ -6625,7 +6806,7 @@ export class Daemon extends EventEmitter {
|
|
|
6625
6806
|
this.museUsageRelay = null;
|
|
6626
6807
|
}
|
|
6627
6808
|
try {
|
|
6628
|
-
await this.killProcessTree();
|
|
6809
|
+
await this.killProcessTree("SIGTERM", "startup aborted");
|
|
6629
6810
|
}
|
|
6630
6811
|
catch { /* nothing running */ }
|
|
6631
6812
|
if (this.tmux) {
|
|
@@ -6652,12 +6833,13 @@ export class Daemon extends EventEmitter {
|
|
|
6652
6833
|
}
|
|
6653
6834
|
}
|
|
6654
6835
|
/** Kill the entire process tree of the current tmux pane (CLI + MCP server). */
|
|
6655
|
-
async killProcessTree(signal = "SIGTERM") {
|
|
6836
|
+
async killProcessTree(signal = "SIGTERM", reason = "unspecified") {
|
|
6656
6837
|
if (!this.tmux)
|
|
6657
6838
|
return;
|
|
6658
6839
|
try {
|
|
6659
6840
|
const pid = await TmuxManager.getPanePid(this.tmuxSessionName, this.tmux.getWindowId());
|
|
6660
6841
|
if (pid) {
|
|
6842
|
+
this.noteStopRequest(signal, reason);
|
|
6661
6843
|
process.kill(-pid, signal);
|
|
6662
6844
|
this.logger.debug({ pid, signal }, "Killed process group");
|
|
6663
6845
|
}
|
|
@@ -7217,7 +7399,16 @@ export class Daemon extends EventEmitter {
|
|
|
7217
7399
|
// CLI is ready (pattern defined by each backend). Require it on two
|
|
7218
7400
|
// consecutive polls: the first ready frame is also the moment a late
|
|
7219
7401
|
// dialog is about to be painted over it.
|
|
7220
|
-
|
|
7402
|
+
const readyPatternMatches = this.backend.getReadyPattern().test(pane);
|
|
7403
|
+
// Codex's broad prompt/footer regex is necessary for inline layouts,
|
|
7404
|
+
// but startup must also prove that the current composer owns the
|
|
7405
|
+
// footer. This matters when a usage-limit picker is being resolved:
|
|
7406
|
+
// its old prompt can remain in scrollback while the reserve session is
|
|
7407
|
+
// still transitioning. Once the live composer is back, do not keep the
|
|
7408
|
+
// scan parked on the old limit text.
|
|
7409
|
+
const codexReady = this.backend.binaryName !== "codex"
|
|
7410
|
+
|| this.isCodexLivePaneSnapshot(pane);
|
|
7411
|
+
if (readyPatternMatches && codexReady) {
|
|
7221
7412
|
cleanReadyPolls++;
|
|
7222
7413
|
if (cleanReadyPolls >= 2 || remaining() <= 0) {
|
|
7223
7414
|
// A real ready prompt (with no fatal dialog on screen — those are
|