@songsid/agend 2.1.6-beta.14 → 2.1.6-beta.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backend/codex.d.ts +1 -0
- package/dist/backend/codex.js +106 -7
- package/dist/backend/codex.js.map +1 -1
- package/dist/backend/muse.d.ts +28 -0
- package/dist/backend/muse.js +52 -0
- package/dist/backend/muse.js.map +1 -1
- package/dist/backend/types.d.ts +1 -1
- package/dist/daemon.d.ts +53 -6
- package/dist/daemon.js +368 -34
- package/dist/daemon.js.map +1 -1
- package/dist/instance-lifecycle.d.ts +17 -0
- package/dist/instance-lifecycle.js +111 -5
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/locale.js +6 -0
- package/dist/locale.js.map +1 -1
- package/dist/usage/providers.js +19 -6
- package/dist/usage/providers.js.map +1 -1
- package/package.json +1 -1
package/dist/daemon.js
CHANGED
|
@@ -435,6 +435,27 @@ const DELIVERY_SPAWN_RACE_MAX_ROUNDS = 3;
|
|
|
435
435
|
*/
|
|
436
436
|
const MUSE_DIRECT_RESUME_IDLE_BUDGET_MS = 10 * 60_000;
|
|
437
437
|
const MUSE_DIRECT_RESUME_IDLE_POLL_MS = 5_000;
|
|
438
|
+
/** Kiro can drop /quit while a turn is running; cancel it and wait for its
|
|
439
|
+
* verified prompt before asking the native process to exit. Each instance is
|
|
440
|
+
* bounded so a fleet update can stop all daemons in parallel. */
|
|
441
|
+
const KIRO_STOP_IDLE_BUDGET_MS = 15_000;
|
|
442
|
+
const KIRO_STOP_QUIT_GRACE_MS = 5_000;
|
|
443
|
+
const KIRO_STOP_SIGTERM_GRACE_MS = 2_000;
|
|
444
|
+
/**
|
|
445
|
+
* How recently AgEnD must have asked the CLI to stop for a pane death to be
|
|
446
|
+
* attributed to AgEnD rather than to the CLI itself (#927). Covers the longest
|
|
447
|
+
* quit grace plus the SIGTERM/SIGKILL fallback, and one health tick more.
|
|
448
|
+
*/
|
|
449
|
+
const STOP_ATTRIBUTION_WINDOW_MS = 90_000;
|
|
450
|
+
/** Lines of pane output kept with a death record: enough for a vendor error. */
|
|
451
|
+
const DEATH_OUTPUT_LINES = 20;
|
|
452
|
+
/** The last `lines` non-empty lines of a captured pane, escape codes already stripped. */
|
|
453
|
+
function paneTail(output, lines = DEATH_OUTPUT_LINES) {
|
|
454
|
+
if (!output)
|
|
455
|
+
return undefined;
|
|
456
|
+
const tail = output.split("\n").filter(line => line.trim()).slice(-lines).join("\n");
|
|
457
|
+
return tail ? tail.slice(-2_000) : undefined;
|
|
458
|
+
}
|
|
438
459
|
/**
|
|
439
460
|
* Startup failed because the CLI's backend is unreachable (see backend-outage.ts).
|
|
440
461
|
* The session-id is deliberately KEPT; the fleet schedules a delayed retry.
|
|
@@ -449,6 +470,10 @@ export class BackendUnreachableStartupError extends Error {
|
|
|
449
470
|
}
|
|
450
471
|
/** Bounded wait (under the pane lock) for the prompt to return before retrying a dropped Enter. */
|
|
451
472
|
const STRANDED_RETRY_READY_WAIT_MS = 30_000;
|
|
473
|
+
/** A pasted Kiro message may be processing for longer than the retry window. */
|
|
474
|
+
const KIRO_SUBMISSION_OBSERVE_MS = 10 * 60_000;
|
|
475
|
+
const KIRO_SUBMISSION_POLL_MS = 1_000;
|
|
476
|
+
const KIRO_SUBMISSION_UNREADABLE_POLLS = 40;
|
|
452
477
|
/** A passive startup phase must clear within this bound before an Enter is sent. */
|
|
453
478
|
/**
|
|
454
479
|
* How long a passive startup/resume transient may sit WITHOUT REPAINTING before
|
|
@@ -494,9 +519,13 @@ const SPAWN_SETTLE_MAX_WAIT_MS = 60_000;
|
|
|
494
519
|
* prompt. A TUI that is still completing its first redraw can swallow Enter.
|
|
495
520
|
*
|
|
496
521
|
* Since the adaptive settle (see {@link waitForPasteSettle}) this value is the
|
|
497
|
-
* *
|
|
498
|
-
*
|
|
499
|
-
*
|
|
522
|
+
* *minimum* settle window for the first delivery — it prevents Enter from firing
|
|
523
|
+
* before the minimum elapses even when the paste's own render goes quiet early.
|
|
524
|
+
* For observed deliveries the quiet-exit path in waitForPasteSettle now checks
|
|
525
|
+
* `now >= fallbackDeadline`, so 1750ms governs every first delivery regardless
|
|
526
|
+
* of whether output is observed. For unobserved deliveries (no control mode,
|
|
527
|
+
* native-queue handoff, mid-wait reconnect, or a paste that never visibly
|
|
528
|
+
* renders) it is the flat fallback delay, same as before.
|
|
500
529
|
*/
|
|
501
530
|
export class FirstDeliveryDelay {
|
|
502
531
|
readyAt = 0;
|
|
@@ -533,9 +562,15 @@ const PASTE_SETTLE_POLL_MS = 100;
|
|
|
533
562
|
* So: watch `lastOutputAt` and send Enter only once the paste's own render has
|
|
534
563
|
* been quiet for {@link PASTE_QUIET_MS}. Bounded both ways —
|
|
535
564
|
*
|
|
536
|
-
* - never earlier than
|
|
537
|
-
*
|
|
538
|
-
*
|
|
565
|
+
* - never earlier than `fallbackMs` from settle start: `fallbackMs` is a
|
|
566
|
+
* **minimum** settle window, not merely a fallback. For first deliveries
|
|
567
|
+
* (fallbackMs = 1750ms) the compositor may render quickly then go quiet at
|
|
568
|
+
* ~550ms while still initialising — the minimum ensures Enter is held until
|
|
569
|
+
* 1750ms regardless of observed output. This applies to every backend's first
|
|
570
|
+
* delivery after ready, not only codex/Luna Reserve. For normal deliveries
|
|
571
|
+
* (fallbackMs = 500ms ≈ PASTE_QUIET_MS) the minimum adds at most a few ms
|
|
572
|
+
* and is effectively unchanged. When the paste produces no observable output
|
|
573
|
+
* at all (`usedFallback`), the same `fallbackMs` deadline governs;
|
|
539
574
|
* - never later than {@link PASTE_SETTLE_CAP_MS} after the paste (`capHit`),
|
|
540
575
|
* so a chatty pane cannot stall delivery.
|
|
541
576
|
*
|
|
@@ -559,7 +594,13 @@ export async function waitForPasteSettle(client, windowId, pasteStartedAt, fallb
|
|
|
559
594
|
const last = client.getLastOutputAt(windowId);
|
|
560
595
|
if (last != null && last > pasteStartedAt) {
|
|
561
596
|
observed = true;
|
|
562
|
-
|
|
597
|
+
// Quiet for PASTE_QUIET_MS AND the minimum settle window has elapsed.
|
|
598
|
+
// The minimum (fallbackDeadline) is the key fix for Luna Reserve: the
|
|
599
|
+
// TUI's first compositor render produces output quickly then goes quiet,
|
|
600
|
+
// but Enter sent at the 500ms quiet point lands while the compositor is
|
|
601
|
+
// still initialising and is swallowed. Holding until fallbackDeadline
|
|
602
|
+
// (1750ms for first deliveries, 500ms for normal ones) eliminates the race.
|
|
603
|
+
if (now - last >= PASTE_QUIET_MS && now >= fallbackDeadline) {
|
|
563
604
|
return { settleMs: now - settleStart, observedPostPasteOutput: true, capHit: false, usedFallback: false };
|
|
564
605
|
}
|
|
565
606
|
}
|
|
@@ -571,9 +612,12 @@ export async function waitForPasteSettle(client, windowId, pasteStartedAt, fallb
|
|
|
571
612
|
}
|
|
572
613
|
// Sleep to the next decision point, at most one poll tick — so the normal
|
|
573
614
|
// paths return at their exact deadlines instead of a poll-width late.
|
|
615
|
+
// When output has been observed but we are still inside the minimum window,
|
|
616
|
+
// sleep to the later of (last quiet deadline) and (fallback deadline) so
|
|
617
|
+
// the loop does not busy-spin between quiet and minimum.
|
|
574
618
|
let wake = capDeadline;
|
|
575
619
|
if (last != null && last > pasteStartedAt)
|
|
576
|
-
wake = Math.min(wake, last + PASTE_QUIET_MS);
|
|
620
|
+
wake = Math.min(wake, Math.max(last + PASTE_QUIET_MS, fallbackDeadline));
|
|
577
621
|
else
|
|
578
622
|
wake = Math.min(wake, fallbackDeadline);
|
|
579
623
|
await new Promise(r => setTimeout(r, Math.min(PASTE_SETTLE_POLL_MS, Math.max(1, wake - now))));
|
|
@@ -1113,6 +1157,42 @@ export class Daemon extends EventEmitter {
|
|
|
1113
1157
|
}
|
|
1114
1158
|
get lastPausedAt() { return this.autoPauseController.lastPausedAt; }
|
|
1115
1159
|
getPauseWakeState() { return this.pauseWakeState; }
|
|
1160
|
+
/**
|
|
1161
|
+
* Codex can show an exhausted-account notice while the user is switching to
|
|
1162
|
+
* a reserve account. Once the live composer and Context footer are back, the
|
|
1163
|
+
* notice is scrollback, not a reason to pause and send /quit.
|
|
1164
|
+
*/
|
|
1165
|
+
isCodexLivePaneSnapshot(pane) {
|
|
1166
|
+
if (this.backend?.binaryName !== "codex")
|
|
1167
|
+
return false;
|
|
1168
|
+
try {
|
|
1169
|
+
if (this.backend.isDeliveryInputReadyPane?.(pane) !== true)
|
|
1170
|
+
return false;
|
|
1171
|
+
const busy = this.backend.getBusyPattern?.();
|
|
1172
|
+
if (busy) {
|
|
1173
|
+
busy.lastIndex = 0;
|
|
1174
|
+
if (busy.test(pane))
|
|
1175
|
+
return false;
|
|
1176
|
+
}
|
|
1177
|
+
const ready = this.backend.getReadyPattern();
|
|
1178
|
+
ready.lastIndex = 0;
|
|
1179
|
+
return ready.test(pane);
|
|
1180
|
+
}
|
|
1181
|
+
catch {
|
|
1182
|
+
return false;
|
|
1183
|
+
}
|
|
1184
|
+
}
|
|
1185
|
+
/** Re-check the current pane after an async quota probe before pausing it. */
|
|
1186
|
+
async isCodexLivePane() {
|
|
1187
|
+
if (this.backend?.binaryName !== "codex" || !this.tmux)
|
|
1188
|
+
return false;
|
|
1189
|
+
try {
|
|
1190
|
+
return this.isCodexLivePaneSnapshot(await this.tmux.capturePane());
|
|
1191
|
+
}
|
|
1192
|
+
catch {
|
|
1193
|
+
return false;
|
|
1194
|
+
}
|
|
1195
|
+
}
|
|
1116
1196
|
/** Whether this instance is in a crash loop (3+ consecutive crashes). */
|
|
1117
1197
|
get isCrashLoop() {
|
|
1118
1198
|
return this.crashCount >= 3;
|
|
@@ -1884,6 +1964,9 @@ export class Daemon extends EventEmitter {
|
|
|
1884
1964
|
// Normal exit (e.g. user Ctrl+C or /exit) — no crash, no respawn
|
|
1885
1965
|
if (paneStatus && exitCode === 0) {
|
|
1886
1966
|
this.setProcessStatus("stopped");
|
|
1967
|
+
// Status 0 is not proof of a clean exit: a codex that hits a quota
|
|
1968
|
+
// wall exits 0 too. Capture what it printed before the window goes.
|
|
1969
|
+
this.logPaneDeath(cliLabel, exitCode, await this.capturePaneOutput());
|
|
1887
1970
|
this.logger.info("CLI exited normally (code 0) — pausing health check");
|
|
1888
1971
|
await this.tmux.killWindow();
|
|
1889
1972
|
this.healthCheckPaused = true;
|
|
@@ -1960,6 +2043,7 @@ export class Daemon extends EventEmitter {
|
|
|
1960
2043
|
lastOutput = cleaned.trimEnd() || undefined;
|
|
1961
2044
|
}
|
|
1962
2045
|
catch { /* best effort — pane may already be gone */ }
|
|
2046
|
+
this.logPaneDeath(cliLabel, exitCode, lastOutput);
|
|
1963
2047
|
// Kill the dead window (remain-on-exit keeps it around) before respawn
|
|
1964
2048
|
if (paneStatus) {
|
|
1965
2049
|
await this.tmux.killWindow();
|
|
@@ -2031,7 +2115,7 @@ export class Daemon extends EventEmitter {
|
|
|
2031
2115
|
this.crashTimestamps = this.crashTimestamps.filter(t => t > Date.now() - crashWindowMs);
|
|
2032
2116
|
if (this.crashTimestamps.length >= 3) {
|
|
2033
2117
|
this.healthCheckPaused = true;
|
|
2034
|
-
this.logger.error({ crashesInWindow: this.crashTimestamps.length }, "3+ crashes in 5 minutes — pausing respawn");
|
|
2118
|
+
this.logger.error({ crashesInWindow: this.crashTimestamps.length, lastExitCode: exitCode ?? null, lastOutput: paneTail(lastOutput) ?? null }, "3+ crashes in 5 minutes — pausing respawn");
|
|
2035
2119
|
// P1: Persist crash state so next process restart skips resume
|
|
2036
2120
|
try {
|
|
2037
2121
|
writeFileSync(join(this.instanceDir, "crash-state.json"), JSON.stringify({
|
|
@@ -2051,7 +2135,7 @@ export class Daemon extends EventEmitter {
|
|
|
2051
2135
|
this.crashCount++;
|
|
2052
2136
|
this.lastCrashAt = Date.now();
|
|
2053
2137
|
if (this.crashCount > max_retries) {
|
|
2054
|
-
this.logger.error({ crashCount: this.crashCount, maxRetries: max_retries }, "Max crash retries exceeded — not respawning");
|
|
2138
|
+
this.logger.error({ crashCount: this.crashCount, maxRetries: max_retries, lastExitCode: exitCode ?? null, lastOutput: paneTail(lastOutput) ?? null }, "Max crash retries exceeded — not respawning");
|
|
2055
2139
|
this.healthCheckPaused = true;
|
|
2056
2140
|
this.emitSupervisionEnded(`it crashed ${this.crashCount} times, exceeding restart_policy.max_retries (${max_retries})`, "Check the logs for the cause, then restart it.");
|
|
2057
2141
|
return; // don't schedule next — given up
|
|
@@ -2355,6 +2439,7 @@ export class Daemon extends EventEmitter {
|
|
|
2355
2439
|
// rebaselining the occurrence count while the error is still displayed — one
|
|
2356
2440
|
// notification, a false recovery log, then silence.
|
|
2357
2441
|
const looksReady = () => !busyPattern?.test(pane) && readyPattern.test(pane);
|
|
2442
|
+
const codexLivePane = this.isCodexLivePaneSnapshot(pane);
|
|
2358
2443
|
// State: waiting for recovery. A missing/outdated ready pattern must not
|
|
2359
2444
|
// suppress every future error forever, so the gate has a hard deadline.
|
|
2360
2445
|
if (this.errorWaitingForRecovery) {
|
|
@@ -2418,6 +2503,17 @@ export class Daemon extends EventEmitter {
|
|
|
2418
2503
|
const key = Daemon.errorPatternKey(ep);
|
|
2419
2504
|
const count = countMatches(ep.pattern);
|
|
2420
2505
|
const seen = this.lastErrorCount.get(key) ?? 0;
|
|
2506
|
+
// The generic Codex usage-limit line remains in scrollback after the
|
|
2507
|
+
// user selects Luna Reserve. A live composer + Context footer is positive
|
|
2508
|
+
// evidence that the pane recovered; baseline the stale occurrence so it
|
|
2509
|
+
// cannot reach the destructive quota pause path.
|
|
2510
|
+
if (codexLivePane && ep.type === "quota" && ep.action === "pause"
|
|
2511
|
+
&& ep.message.startsWith("Codex usage limit reached")) {
|
|
2512
|
+
if (count > 0)
|
|
2513
|
+
this.lastErrorCount.set(key, count);
|
|
2514
|
+
this.logger.debug("Codex usage-limit text is stale — live reserve pane is running");
|
|
2515
|
+
continue;
|
|
2516
|
+
}
|
|
2421
2517
|
if (count <= seen) {
|
|
2422
2518
|
// Occurrences scrolled out of the capture buffer → lower the baseline
|
|
2423
2519
|
// so a future re-occurrence still counts as new (no permanent suppress).
|
|
@@ -2612,12 +2708,61 @@ export class Daemon extends EventEmitter {
|
|
|
2612
2708
|
// is a successful no-op here, not an error.
|
|
2613
2709
|
this.logger.debug({ dialog: dialog.description }, "Backend clear dialog did not appear");
|
|
2614
2710
|
}
|
|
2711
|
+
/**
|
|
2712
|
+
* The last time AgEnD itself asked the CLI to go away, and why (#927). Read
|
|
2713
|
+
* when a pane dies, to tell "AgEnD stopped it" from "the CLI exited on its
|
|
2714
|
+
* own" — a codex that exits 0 on its own and one AgEnD sent /quit look the
|
|
2715
|
+
* same from the pane.
|
|
2716
|
+
*/
|
|
2717
|
+
lastStopRequest = null;
|
|
2718
|
+
/** Record, and log, that AgEnD is about to stop the CLI. */
|
|
2719
|
+
noteStopRequest(via, reason) {
|
|
2720
|
+
this.lastStopRequest = { via, reason, at: Date.now() };
|
|
2721
|
+
this.logger.info({ via, reason }, `AgEnD is stopping the CLI: ${via} (${reason})`);
|
|
2722
|
+
}
|
|
2723
|
+
/** Capture the pane's recent output with escape codes stripped. Never throws. */
|
|
2724
|
+
async capturePaneOutput() {
|
|
2725
|
+
try {
|
|
2726
|
+
const raw = await this.tmux?.capturePaneWithHistory(50);
|
|
2727
|
+
const cleaned = raw?.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
|
|
2728
|
+
return cleaned?.trimEnd() || undefined;
|
|
2729
|
+
}
|
|
2730
|
+
catch {
|
|
2731
|
+
return undefined;
|
|
2732
|
+
}
|
|
2733
|
+
}
|
|
2734
|
+
/**
|
|
2735
|
+
* One record per pane death: exit status, when, who, and what the CLI printed
|
|
2736
|
+
* last — the line that was missing when a codex on Luna Reserve died with
|
|
2737
|
+
* status 0 and daemon.log said only "exited normally" (#927). Logging only;
|
|
2738
|
+
* the caller decides what happens next.
|
|
2739
|
+
*/
|
|
2740
|
+
logPaneDeath(cliLabel, exitCode, lastOutput) {
|
|
2741
|
+
const request = this.lastStopRequest;
|
|
2742
|
+
const sinceStopMs = request ? Date.now() - request.at : null;
|
|
2743
|
+
const byAgend = request !== null && sinceStopMs !== null && sinceStopMs <= STOP_ATTRIBUTION_WINDOW_MS;
|
|
2744
|
+
const record = {
|
|
2745
|
+
exitCode: exitCode ?? null,
|
|
2746
|
+
diedAt: new Date().toISOString(),
|
|
2747
|
+
initiatedBy: byAgend ? "agend" : "cli",
|
|
2748
|
+
...(byAgend ? { stopVia: request.via, stopReason: request.reason, sinceStopMs } : {}),
|
|
2749
|
+
lastOutput: paneTail(lastOutput) ?? null,
|
|
2750
|
+
};
|
|
2751
|
+
if (byAgend) {
|
|
2752
|
+
this.logger.info(record, `${cliLabel} exited after AgEnD stopped it (${request.via}: ${request.reason})`);
|
|
2753
|
+
}
|
|
2754
|
+
else {
|
|
2755
|
+
this.logger.warn(record, `${cliLabel} exited on its own (exit code ${exitCode ?? "unknown"}) — last output recorded`);
|
|
2756
|
+
}
|
|
2757
|
+
}
|
|
2615
2758
|
/** Send the backend-specific graceful quit command/key sequence. */
|
|
2616
|
-
async sendQuitSequence() {
|
|
2759
|
+
async sendQuitSequence(reason = "unspecified") {
|
|
2617
2760
|
if (!this.tmux || !this.backend)
|
|
2618
2761
|
return false;
|
|
2619
2762
|
const quitCmd = this.backend.getQuitCommand();
|
|
2620
2763
|
const quitKey = this.backend.getQuitKey?.();
|
|
2764
|
+
if (quitCmd || quitKey)
|
|
2765
|
+
this.noteStopRequest(quitCmd ? `quit command ${quitCmd}` : `quit key ${quitKey}`, reason);
|
|
2621
2766
|
if (quitCmd) {
|
|
2622
2767
|
if (!await this.tmux.sendKeys(quitCmd))
|
|
2623
2768
|
return false;
|
|
@@ -2638,6 +2783,26 @@ export class Daemon extends EventEmitter {
|
|
|
2638
2783
|
}
|
|
2639
2784
|
return true;
|
|
2640
2785
|
}
|
|
2786
|
+
/**
|
|
2787
|
+
* Kiro drops `/quit` while a turn is running. Stop that turn first, then use
|
|
2788
|
+
* the same delivery readiness gate that protects Enter from the busy pane.
|
|
2789
|
+
* Returning false is fail-closed: the caller skips `/quit` and enters the
|
|
2790
|
+
* bounded SIGTERM -> SIGKILL fallback instead of typing into an unknown pane.
|
|
2791
|
+
*/
|
|
2792
|
+
async drainBusyKiroForStop(windowId) {
|
|
2793
|
+
if (this.backend?.binaryName !== "kiro-cli")
|
|
2794
|
+
return true;
|
|
2795
|
+
const readiness = await this.paneReadinessForDelivery(windowId);
|
|
2796
|
+
if (readiness === "ready")
|
|
2797
|
+
return true;
|
|
2798
|
+
if (readiness !== "busy")
|
|
2799
|
+
return false;
|
|
2800
|
+
const cancelKey = this.backend.getCancelKey?.() ?? "Escape";
|
|
2801
|
+
const sent = await this.tmux?.sendSpecialKey(cancelKey);
|
|
2802
|
+
if (!sent)
|
|
2803
|
+
return false;
|
|
2804
|
+
return this.waitForPaneReadyForDelivery(windowId, KIRO_STOP_IDLE_BUDGET_MS);
|
|
2805
|
+
}
|
|
2641
2806
|
async stop() {
|
|
2642
2807
|
this.logger.info("Stopping daemon instance");
|
|
2643
2808
|
this.turnReplyGuard.reset();
|
|
@@ -2663,11 +2828,18 @@ export class Daemon extends EventEmitter {
|
|
|
2663
2828
|
this.saveSessionId();
|
|
2664
2829
|
this.healthCheckPaused = true;
|
|
2665
2830
|
let killed = false;
|
|
2666
|
-
const
|
|
2831
|
+
const windowId = this.tmux.getWindowId();
|
|
2832
|
+
const kiroReady = windowId ? await this.drainBusyKiroForStop(windowId) : true;
|
|
2833
|
+
const quitSent = kiroReady && await this.sendQuitSequence("graceful stop");
|
|
2834
|
+
const quitGraceMs = this.backend?.binaryName === "kiro-cli"
|
|
2835
|
+
? KIRO_STOP_QUIT_GRACE_MS : 3_000;
|
|
2836
|
+
const sigtermGraceMs = this.backend?.binaryName === "kiro-cli"
|
|
2837
|
+
? KIRO_STOP_SIGTERM_GRACE_MS : 1_000;
|
|
2667
2838
|
if (quitSent) {
|
|
2668
|
-
// Wait up to
|
|
2839
|
+
// Wait up to the backend's bounded graceful-exit window, polling every
|
|
2840
|
+
// 200ms. A healthy CLI
|
|
2669
2841
|
// exits within ~1s; a longer wait just delays the force-kill fallback.
|
|
2670
|
-
for (let
|
|
2842
|
+
for (let elapsed = 0; elapsed < quitGraceMs; elapsed += 200) {
|
|
2671
2843
|
await new Promise(r => setTimeout(r, 200));
|
|
2672
2844
|
const status = await this.tmux.getPaneStatus();
|
|
2673
2845
|
if (!status || !status.alive) {
|
|
@@ -2677,9 +2849,9 @@ export class Daemon extends EventEmitter {
|
|
|
2677
2849
|
}
|
|
2678
2850
|
}
|
|
2679
2851
|
if (!killed) {
|
|
2680
|
-
this.logger.warn({ quitSent }, "CLI did not exit gracefully within
|
|
2681
|
-
await this.killProcessTree("SIGTERM");
|
|
2682
|
-
for (let
|
|
2852
|
+
this.logger.warn({ quitSent, quitGraceMs }, "CLI did not exit gracefully within its bounded quit grace — falling back to SIGTERM");
|
|
2853
|
+
await this.killProcessTree("SIGTERM", "graceful stop: the CLI outlived its quit grace");
|
|
2854
|
+
for (let elapsed = 0; elapsed < sigtermGraceMs; elapsed += 200) {
|
|
2683
2855
|
await new Promise(r => setTimeout(r, 200));
|
|
2684
2856
|
const status = await this.tmux.getPaneStatus();
|
|
2685
2857
|
if (!status || !status.alive) {
|
|
@@ -2690,7 +2862,7 @@ export class Daemon extends EventEmitter {
|
|
|
2690
2862
|
}
|
|
2691
2863
|
if (!killed) {
|
|
2692
2864
|
this.logger.warn("CLI process tree survived SIGTERM — falling back to SIGKILL");
|
|
2693
|
-
await this.killProcessTree("SIGKILL");
|
|
2865
|
+
await this.killProcessTree("SIGKILL", "graceful stop: the CLI survived SIGTERM");
|
|
2694
2866
|
await new Promise(r => setTimeout(r, 200));
|
|
2695
2867
|
}
|
|
2696
2868
|
// Always kill window — remain-on-exit keeps dead panes around after CLI exits
|
|
@@ -2792,6 +2964,9 @@ export class Daemon extends EventEmitter {
|
|
|
2792
2964
|
// into an ordinary idle-timeout pause of a busy instance.
|
|
2793
2965
|
const allowStuck = this.pauseAllowStuck;
|
|
2794
2966
|
this.pauseAllowStuck = false;
|
|
2967
|
+
// Why this pause, for the stop record (#927): an idle pause and an
|
|
2968
|
+
// auth-deferred one look the same from the pane.
|
|
2969
|
+
const pauseReason = allowStuck ? "pause (auth-deferred, stuck pane)" : "pause (idle)";
|
|
2795
2970
|
const pausableState = this.instanceState === "idle"
|
|
2796
2971
|
|| (allowStuck && this.instanceState === "stuck");
|
|
2797
2972
|
if (!pausableState || this.pasteQueueDepth > 0) {
|
|
@@ -2804,7 +2979,7 @@ export class Daemon extends EventEmitter {
|
|
|
2804
2979
|
const transition = (async () => {
|
|
2805
2980
|
try {
|
|
2806
2981
|
this.saveSessionId();
|
|
2807
|
-
await this.sendQuitSequence();
|
|
2982
|
+
await this.sendQuitSequence(pauseReason);
|
|
2808
2983
|
let exited = false;
|
|
2809
2984
|
for (let i = 0; i < 15; i++) {
|
|
2810
2985
|
await new Promise(r => setTimeout(r, 200));
|
|
@@ -2815,11 +2990,11 @@ export class Daemon extends EventEmitter {
|
|
|
2815
2990
|
}
|
|
2816
2991
|
}
|
|
2817
2992
|
if (!exited) {
|
|
2818
|
-
await this.killProcessTree("SIGTERM");
|
|
2993
|
+
await this.killProcessTree("SIGTERM", `${pauseReason}: the CLI outlived its quit grace`);
|
|
2819
2994
|
await new Promise(r => setTimeout(r, 1_000));
|
|
2820
2995
|
const status = await this.tmux?.getPaneStatus();
|
|
2821
2996
|
if (status?.alive) {
|
|
2822
|
-
await this.killProcessTree("SIGKILL");
|
|
2997
|
+
await this.killProcessTree("SIGKILL", `${pauseReason}: the CLI survived SIGTERM`);
|
|
2823
2998
|
await new Promise(r => setTimeout(r, 200));
|
|
2824
2999
|
}
|
|
2825
3000
|
}
|
|
@@ -4244,6 +4419,8 @@ export class Daemon extends EventEmitter {
|
|
|
4244
4419
|
// A false return means the cap expired with a spawn STILL running — the
|
|
4245
4420
|
// generation captured below is in-progress evidence, not settled evidence,
|
|
4246
4421
|
// so the critical section must still back out (see settledClean).
|
|
4422
|
+
if (this.spawnSettled && status)
|
|
4423
|
+
this.emit("message_queued", status);
|
|
4247
4424
|
const settledClean = await this.waitForSpawnToSettle();
|
|
4248
4425
|
// Everything captured below (window id, readiness verdicts) belongs to
|
|
4249
4426
|
// this spawn generation. A spawn that starts afterwards is detected inside
|
|
@@ -4385,6 +4562,9 @@ export class Daemon extends EventEmitter {
|
|
|
4385
4562
|
}
|
|
4386
4563
|
return this.deliverMessage(formatted, status, { ...opts, spawnRetry: retries });
|
|
4387
4564
|
}
|
|
4565
|
+
if (typeof outcome === "object") {
|
|
4566
|
+
return this.observeKiroSubmission(formatted, outcome, status, verdict, cancelled);
|
|
4567
|
+
}
|
|
4388
4568
|
if (outcome !== "dialog")
|
|
4389
4569
|
return outcome;
|
|
4390
4570
|
// Wait OUTSIDE the lock (holding it would starve the runtime dismisser),
|
|
@@ -4507,8 +4687,18 @@ export class Daemon extends EventEmitter {
|
|
|
4507
4687
|
this.logger.warn({ phase, generation }, "Spawn changed during the input-availability probe — refusing to send Enter into the replacement pane");
|
|
4508
4688
|
return false;
|
|
4509
4689
|
}
|
|
4510
|
-
if (probe.state === "clear")
|
|
4690
|
+
if (probe.state === "clear") {
|
|
4691
|
+
// Re-arm the first-delivery delay when a transient was actually observed
|
|
4692
|
+
// and then cleared, but ONLY from the pre-write phase. That phase runs
|
|
4693
|
+
// before consume() in writeMessageToPane, so the re-arm benefits the
|
|
4694
|
+
// current delivery's paste settle. Enter-path callers (initial-submit,
|
|
4695
|
+
// retries) run after consume() — re-arming there would leave the flag
|
|
4696
|
+
// for the NEXT unrelated delivery, which must not be slowed.
|
|
4697
|
+
if (observedDescription !== null && phase === "pre-write") {
|
|
4698
|
+
this.firstDeliveryDelay.recordReady();
|
|
4699
|
+
}
|
|
4511
4700
|
return true;
|
|
4701
|
+
}
|
|
4512
4702
|
if (probe.state === "active" && observedDescription !== probe.transient.description) {
|
|
4513
4703
|
observedDescription = probe.transient.description;
|
|
4514
4704
|
this.logger.info({ phase, transient: probe.transient.description, generation }, "CLI is still completing startup — waiting before sending Enter");
|
|
@@ -4913,6 +5103,126 @@ export class Daemon extends EventEmitter {
|
|
|
4913
5103
|
this.logger.warn("Could not read the pane after Enter — not confirming the submission");
|
|
4914
5104
|
return sawOutput && residue === "absent";
|
|
4915
5105
|
}
|
|
5106
|
+
/** Once Kiro returns to its prompt, its unique envelope id in history proves submission. */
|
|
5107
|
+
async kiroSubmissionEvidence(windowId, formatted, signature) {
|
|
5108
|
+
if (!this.tmux)
|
|
5109
|
+
return "unreadable";
|
|
5110
|
+
let pane;
|
|
5111
|
+
try {
|
|
5112
|
+
pane = await this.tmux.capturePane();
|
|
5113
|
+
}
|
|
5114
|
+
catch {
|
|
5115
|
+
return "unreadable";
|
|
5116
|
+
}
|
|
5117
|
+
const prompt = this.backend?.getBottomReadyPattern?.();
|
|
5118
|
+
if (prompt && bottomRowIsReady(pane, prompt)
|
|
5119
|
+
&& inputShowsPastedText(inputAreaText(pane, prompt) ?? "", signature.value))
|
|
5120
|
+
return "stranded";
|
|
5121
|
+
if (prompt && pasteLeftInInput(pane, prompt, formatted))
|
|
5122
|
+
return "stranded";
|
|
5123
|
+
// During generation the same text can still be typeahead for an older
|
|
5124
|
+
// turn. A history hit is safe only after this pane returns to its prompt.
|
|
5125
|
+
if (!signature.unique || !this.tmux.capturePaneWithHistory
|
|
5126
|
+
|| !(await this.isPaneReadyForDelivery(windowId)))
|
|
5127
|
+
return "unknown";
|
|
5128
|
+
try {
|
|
5129
|
+
const history = await this.tmux.capturePaneWithHistory(300);
|
|
5130
|
+
// History includes the CURRENT input row. The earlier viewport capture
|
|
5131
|
+
// and readiness probe may belong to an older frame: Kiro can finish a
|
|
5132
|
+
// turn and expose our unsent typeahead between those reads. Judge both
|
|
5133
|
+
// readiness and input residue on this same, final snapshot before its
|
|
5134
|
+
// message id can count as a submitted transcript echo.
|
|
5135
|
+
if (!prompt || !bottomRowIsReady(history, prompt))
|
|
5136
|
+
return "unknown";
|
|
5137
|
+
if (inputShowsPastedText(inputAreaText(history, prompt) ?? "", signature.value)
|
|
5138
|
+
|| pasteLeftInInput(history, prompt, formatted))
|
|
5139
|
+
return "stranded";
|
|
5140
|
+
if (countOccurrences(history.replace(/\s+/g, ""), signature.value) > 0)
|
|
5141
|
+
return "submitted";
|
|
5142
|
+
}
|
|
5143
|
+
catch { /* missing history cannot establish a verdict */ }
|
|
5144
|
+
return "unknown";
|
|
5145
|
+
}
|
|
5146
|
+
/** Observe the Kiro paste outside paneWriteLock; a long agent turn must not block dialog handling. */
|
|
5147
|
+
async observeKiroSubmission(formatted, pending, status, verdict, cancelled) {
|
|
5148
|
+
const deadline = Date.now() + KIRO_SUBMISSION_OBSERVE_MS;
|
|
5149
|
+
let retriedStrand = false;
|
|
5150
|
+
let unreadable = 0;
|
|
5151
|
+
const current = () => !cancelled()
|
|
5152
|
+
&& !this.spawning
|
|
5153
|
+
&& this.spawnGeneration === pending.spawnGeneration
|
|
5154
|
+
&& this.getWindowId() === pending.windowId
|
|
5155
|
+
&& this.tmux?.getWindowId() === pending.windowId;
|
|
5156
|
+
for (;;) {
|
|
5157
|
+
if (!current())
|
|
5158
|
+
return false;
|
|
5159
|
+
const evidence = await this.kiroSubmissionEvidence(pending.windowId, formatted, pending.signature);
|
|
5160
|
+
if (!current())
|
|
5161
|
+
return false;
|
|
5162
|
+
if (evidence === "submitted") {
|
|
5163
|
+
if (status)
|
|
5164
|
+
this.emit("message_confirmed", status);
|
|
5165
|
+
return true;
|
|
5166
|
+
}
|
|
5167
|
+
unreadable = evidence === "unreadable" ? unreadable + 1 : 0;
|
|
5168
|
+
if (unreadable >= KIRO_SUBMISSION_UNREADABLE_POLLS) {
|
|
5169
|
+
return this.failDelivery(verdict, status, "post-submit-proof", "pane-unreadable");
|
|
5170
|
+
}
|
|
5171
|
+
if (evidence === "stranded" && await this.isPaneReadyForDelivery(pending.windowId)) {
|
|
5172
|
+
if (!current())
|
|
5173
|
+
return false;
|
|
5174
|
+
if (retriedStrand)
|
|
5175
|
+
return this.failDelivery(verdict, status, "stranded-text-retry", "stranded");
|
|
5176
|
+
const retry = await this.paneWriteLock.run(async () => {
|
|
5177
|
+
if (!current())
|
|
5178
|
+
return "cancelled";
|
|
5179
|
+
const fresh = await this.kiroSubmissionEvidence(pending.windowId, formatted, pending.signature);
|
|
5180
|
+
if (!current())
|
|
5181
|
+
return "cancelled";
|
|
5182
|
+
if (fresh === "submitted")
|
|
5183
|
+
return "submitted";
|
|
5184
|
+
if (fresh !== "stranded" || !(await this.isPaneReadyForDelivery(pending.windowId)))
|
|
5185
|
+
return "changed";
|
|
5186
|
+
if (!current())
|
|
5187
|
+
return "cancelled";
|
|
5188
|
+
const sent = await this.sendDeliveryEnter("stranded-text-retry", current);
|
|
5189
|
+
if (!current())
|
|
5190
|
+
return "cancelled";
|
|
5191
|
+
return sent ? "sent" : "failed";
|
|
5192
|
+
});
|
|
5193
|
+
if (!current())
|
|
5194
|
+
return false;
|
|
5195
|
+
if (retry === "cancelled")
|
|
5196
|
+
return false;
|
|
5197
|
+
if (retry === "submitted") {
|
|
5198
|
+
if (status)
|
|
5199
|
+
this.emit("message_confirmed", status);
|
|
5200
|
+
return true;
|
|
5201
|
+
}
|
|
5202
|
+
if (retry === "failed")
|
|
5203
|
+
return this.failDelivery(verdict, status, "stranded-text-retry", "tmux-send-keys-failed");
|
|
5204
|
+
if (retry === "sent") {
|
|
5205
|
+
retriedStrand = true;
|
|
5206
|
+
await new Promise(r => setTimeout(r, POST_ENTER_PROOF_WINDOW_MS));
|
|
5207
|
+
continue;
|
|
5208
|
+
}
|
|
5209
|
+
}
|
|
5210
|
+
const alive = await this.tmux?.isWindowAlive().catch(() => false);
|
|
5211
|
+
if (!current())
|
|
5212
|
+
return false;
|
|
5213
|
+
if (!alive)
|
|
5214
|
+
return this.failDelivery(verdict, status, "post-submit-proof", "window-gone");
|
|
5215
|
+
if (Date.now() >= deadline) {
|
|
5216
|
+
// A live, busy pane and no positive strand are still ambiguous. The
|
|
5217
|
+
// hang detector owns hung CLI recovery; elapsed time alone is not ❌.
|
|
5218
|
+
this.logger.warn("Kiro submission remains unproven; keeping the delivery queued without a false failure");
|
|
5219
|
+
verdict.phase = "post-submit-proof";
|
|
5220
|
+
verdict.proof = "unproven";
|
|
5221
|
+
return false;
|
|
5222
|
+
}
|
|
5223
|
+
await new Promise(r => setTimeout(r, KIRO_SUBMISSION_POLL_MS));
|
|
5224
|
+
}
|
|
5225
|
+
}
|
|
4916
5226
|
/**
|
|
4917
5227
|
* While the CLI is parked on a fatal startup screen (see fatalStartupBlocked),
|
|
4918
5228
|
* no pane write may happen: paste+Enter into the corrupt-config modal would
|
|
@@ -4992,9 +5302,13 @@ export class Daemon extends EventEmitter {
|
|
|
4992
5302
|
* precisely so the lock's scope is visible at the call site rather than being
|
|
4993
5303
|
* an invariant maintained by comments.
|
|
4994
5304
|
*/
|
|
4995
|
-
async sendDeliveryEnter(phase) {
|
|
5305
|
+
async sendDeliveryEnter(phase, stillCurrent) {
|
|
4996
5306
|
if (!(await this.waitForInputTransientToClear(phase)))
|
|
4997
5307
|
return false;
|
|
5308
|
+
// A recovery Enter may have waited for a transient while cancel or spawn
|
|
5309
|
+
// replaced the delivery. Check at the last point before the tmux write.
|
|
5310
|
+
if (stillCurrent && !stillCurrent())
|
|
5311
|
+
return false;
|
|
4998
5312
|
const sent = await this.tmux.sendSpecialKey("Enter");
|
|
4999
5313
|
if (!sent) {
|
|
5000
5314
|
this.logger.error({
|
|
@@ -5219,10 +5533,20 @@ export class Daemon extends EventEmitter {
|
|
|
5219
5533
|
// F2: output after Enter is necessary but not sufficient — the paste
|
|
5220
5534
|
// must also have LEFT the input row.
|
|
5221
5535
|
let submitted = await this.confirmSubmittedAfterEnter(windowId, enterAt, formatted);
|
|
5536
|
+
if (!submitted && signature.unique) {
|
|
5537
|
+
submitted = await this.kiroSubmissionEvidence(windowId, formatted, signature) === "submitted";
|
|
5538
|
+
if (!submitted) {
|
|
5539
|
+
// No output edge is not proof of a lost message. Leave the pane
|
|
5540
|
+
// lock before waiting through a long Kiro turn, so runtime dialog
|
|
5541
|
+
// handling can still make progress.
|
|
5542
|
+
if (status)
|
|
5543
|
+
this.emit("message_queued", status);
|
|
5544
|
+
return { kind: "kiro-pending", windowId, spawnGeneration: this.spawnGeneration, signature };
|
|
5545
|
+
}
|
|
5546
|
+
}
|
|
5222
5547
|
if (!submitted) {
|
|
5223
|
-
//
|
|
5224
|
-
//
|
|
5225
|
-
// bounded, since we hold the pane lock here.
|
|
5548
|
+
// Legacy system pastes have no unique envelope id. Preserve their
|
|
5549
|
+
// existing bounded retry until they can be tied to a specific turn.
|
|
5226
5550
|
this.logger.warn("Message not submitted after Enter — waiting for the prompt, then re-sending Enter once");
|
|
5227
5551
|
const promptBack = await this.waitForPaneReadyForDelivery(windowId, STRANDED_RETRY_READY_WAIT_MS);
|
|
5228
5552
|
const retryAt = Date.now();
|
|
@@ -6321,7 +6645,7 @@ export class Daemon extends EventEmitter {
|
|
|
6321
6645
|
await this.failStartupIfBackendUnreachable();
|
|
6322
6646
|
if (this.backend.retriesResumeOnStartupFailure?.() !== false) {
|
|
6323
6647
|
this.logger.warn("Resume startup failed — retrying resume once before abandoning the session");
|
|
6324
|
-
await this.killProcessTree();
|
|
6648
|
+
await this.killProcessTree("SIGTERM", "spawn: clearing the previous CLI process");
|
|
6325
6649
|
await this.tmux.killWindow();
|
|
6326
6650
|
alive = await this.trySpawn(false, resumeBudget);
|
|
6327
6651
|
if (!alive) {
|
|
@@ -6347,7 +6671,7 @@ export class Daemon extends EventEmitter {
|
|
|
6347
6671
|
if (this.unprovenResumeFailures < Daemon.MAX_UNPROVEN_RESUME_FAILURES) {
|
|
6348
6672
|
// Keep the session and fail this attempt; the fleet retries with
|
|
6349
6673
|
// backoff, which is also how the backend-outage path behaves.
|
|
6350
|
-
await this.killProcessTree();
|
|
6674
|
+
await this.killProcessTree("SIGTERM", "spawn: clearing the previous CLI process");
|
|
6351
6675
|
await this.tmux.killWindow();
|
|
6352
6676
|
throw new Error(`CLI startup failed with a session to resume (attempt ${this.unprovenResumeFailures}/${Daemon.MAX_UNPROVEN_RESUME_FAILURES}) `
|
|
6353
6677
|
+ "— session kept, will retry");
|
|
@@ -6364,11 +6688,11 @@ export class Daemon extends EventEmitter {
|
|
|
6364
6688
|
// A fresh start that also failed retries once, as before. It never clears
|
|
6365
6689
|
// a session: nothing about a failed fresh launch says the stored
|
|
6366
6690
|
// conversation is unusable.
|
|
6367
|
-
await this.killProcessTree();
|
|
6691
|
+
await this.killProcessTree("SIGTERM", "spawn: clearing the previous CLI process");
|
|
6368
6692
|
await this.tmux.killWindow();
|
|
6369
6693
|
const retryAlive = await this.trySpawn(false, this.startupBudgetFor(false));
|
|
6370
6694
|
if (!retryAlive) {
|
|
6371
|
-
await this.killProcessTree();
|
|
6695
|
+
await this.killProcessTree("SIGTERM", "spawn: clearing the previous CLI process");
|
|
6372
6696
|
await this.tmux.killWindow();
|
|
6373
6697
|
throw new Error("CLI failed to start after retry");
|
|
6374
6698
|
}
|
|
@@ -6440,7 +6764,7 @@ export class Daemon extends EventEmitter {
|
|
|
6440
6764
|
if (!this.backendOutage?.isActive(this.backendKey()))
|
|
6441
6765
|
return;
|
|
6442
6766
|
this.logger.warn("Backend unreachable — keeping the session and failing startup for a delayed retry");
|
|
6443
|
-
await this.killProcessTree();
|
|
6767
|
+
await this.killProcessTree("SIGTERM", "startup: backend unreachable");
|
|
6444
6768
|
await this.tmux.killWindow();
|
|
6445
6769
|
throw new BackendUnreachableStartupError(this.backendKey());
|
|
6446
6770
|
}
|
|
@@ -6482,7 +6806,7 @@ export class Daemon extends EventEmitter {
|
|
|
6482
6806
|
this.museUsageRelay = null;
|
|
6483
6807
|
}
|
|
6484
6808
|
try {
|
|
6485
|
-
await this.killProcessTree();
|
|
6809
|
+
await this.killProcessTree("SIGTERM", "startup aborted");
|
|
6486
6810
|
}
|
|
6487
6811
|
catch { /* nothing running */ }
|
|
6488
6812
|
if (this.tmux) {
|
|
@@ -6509,12 +6833,13 @@ export class Daemon extends EventEmitter {
|
|
|
6509
6833
|
}
|
|
6510
6834
|
}
|
|
6511
6835
|
/** Kill the entire process tree of the current tmux pane (CLI + MCP server). */
|
|
6512
|
-
async killProcessTree(signal = "SIGTERM") {
|
|
6836
|
+
async killProcessTree(signal = "SIGTERM", reason = "unspecified") {
|
|
6513
6837
|
if (!this.tmux)
|
|
6514
6838
|
return;
|
|
6515
6839
|
try {
|
|
6516
6840
|
const pid = await TmuxManager.getPanePid(this.tmuxSessionName, this.tmux.getWindowId());
|
|
6517
6841
|
if (pid) {
|
|
6842
|
+
this.noteStopRequest(signal, reason);
|
|
6518
6843
|
process.kill(-pid, signal);
|
|
6519
6844
|
this.logger.debug({ pid, signal }, "Killed process group");
|
|
6520
6845
|
}
|
|
@@ -7074,7 +7399,16 @@ export class Daemon extends EventEmitter {
|
|
|
7074
7399
|
// CLI is ready (pattern defined by each backend). Require it on two
|
|
7075
7400
|
// consecutive polls: the first ready frame is also the moment a late
|
|
7076
7401
|
// dialog is about to be painted over it.
|
|
7077
|
-
|
|
7402
|
+
const readyPatternMatches = this.backend.getReadyPattern().test(pane);
|
|
7403
|
+
// Codex's broad prompt/footer regex is necessary for inline layouts,
|
|
7404
|
+
// but startup must also prove that the current composer owns the
|
|
7405
|
+
// footer. This matters when a usage-limit picker is being resolved:
|
|
7406
|
+
// its old prompt can remain in scrollback while the reserve session is
|
|
7407
|
+
// still transitioning. Once the live composer is back, do not keep the
|
|
7408
|
+
// scan parked on the old limit text.
|
|
7409
|
+
const codexReady = this.backend.binaryName !== "codex"
|
|
7410
|
+
|| this.isCodexLivePaneSnapshot(pane);
|
|
7411
|
+
if (readyPatternMatches && codexReady) {
|
|
7078
7412
|
cleanReadyPolls++;
|
|
7079
7413
|
if (cleanReadyPolls >= 2 || remaining() <= 0) {
|
|
7080
7414
|
// A real ready prompt (with no fatal dialog on screen — those are
|