@songsid/agend 2.1.8 → 2.1.9

package/dist/daemon.js

@@ -454,6 +454,11 @@ const MUSE_DIRECT_RESUME_IDLE_POLL_MS = 5_000;
  * verified prompt before asking the native process to exit. Each instance is
  * bounded so a fleet update can stop all daemons in parallel. */
 const KIRO_STOP_IDLE_BUDGET_MS = 15_000;
+/**
+ * Bound after a non-stop quit before a dead pane with no relaunch is a wedge
+ * (#1030). Exceeds the wake budget (30s+) so a slow relaunch never trips it.
+ */
+const QUIT_RELAUNCH_WATCH_MS = 90_000;
 const KIRO_STOP_QUIT_GRACE_MS = 5_000;
 const KIRO_STOP_SIGTERM_GRACE_MS = 2_000;
 /**
@@ -1065,6 +1070,16 @@ export class Daemon extends EventEmitter {
      * would silently no-op exactly when we most need to stop feeding the CLI.
      */
     pausePending = false;
+    /**
+     * #1044: pause on the idle edge only if the auth error is still on screen.
+     * Set when nothing could vouch for the hit — the backend has no token-free
+     * auth check (muse), or it could not answer. A real expired session prints
+     * its failure at the end of the turn it broke, so it is still at the bottom
+     * when that turn ends; a phrase the pattern caught mid-turn has scrolled on
+     * by then. Recovery cannot decide this: an expired session is back at its
+     * prompt after every failed turn, which is what "recovered" means.
+     */
+    pauseReconfirmAuth = false;
     /**
      * An auth failure was detected and nothing has resolved it yet.
      *
@@ -1075,6 +1090,15 @@ export class Daemon extends EventEmitter {
     authFailureUnresolved = false;
     /** One login-screen auth report per spawn — the screen persists across polls. */
     loginScreenReported = false;
+    /**
+     * While the lifecycle is still starting this daemon, `pty_error` has no
+     * listener and `InstanceLifecycle.daemons` has no entry — so an incident the
+     * startup scan raises (sign-in screen, fatal modal) would be dropped on the
+     * floor while its dedupe flag stays set forever. Held here, then re-emitted by
+     * releaseStartupIncidents() once the handlers and the registry are in place.
+     */
+    holdingStartupIncidents = false;
+    heldStartupIncidents = [];
     /** A fatal startup screen (see StartupDialog.fatal) was already reported this run. */
     fatalStartupReported = false;
     /**
@@ -1145,6 +1169,8 @@ export class Daemon extends EventEmitter {
     lastProgressBroadcastAt = 0;
     // PTY error pattern monitoring
     errorMonitorTimer = null;
+    quitWatchTimer = null;
+    quitWatchToken = null;
     interactivePromptDetector = new InteractivePromptDetector();
     blockingProcessDetector = new BlockingProcessDetector();
     /** Same 5-min gate the error monitor uses, so a dead MCP server alerts once. */
@@ -1290,6 +1316,24 @@ export class Daemon extends EventEmitter {
         this.lastErrorCount.set(key, count);
         this.seededErrorCountBaselines.set(key, count);
     }
+    /** Start buffering `pty_error` until releaseStartupIncidents(); see holdingStartupIncidents. */
+    holdStartupIncidents() {
+        this.holdingStartupIncidents = true;
+    }
+    /** Stop buffering and deliver, in order, every `pty_error` raised while the lifecycle was starting us. */
+    releaseStartupIncidents() {
+        this.holdingStartupIncidents = false;
+        const held = this.heldStartupIncidents.splice(0);
+        for (const args of held)
+            super.emit("pty_error", ...args);
+    }
+    emit(event, ...args) {
+        if (event === "pty_error" && this.holdingStartupIncidents) {
+            this.heldStartupIncidents.push(args);
+            return true;
+        }
+        return super.emit(event, ...args);
+    }
     /**
      * Undo the auth suspicion a pattern match armed, once the backend's token-free
      * probe has said the credentials are fine.
@@ -1446,6 +1490,11 @@ export class Daemon extends EventEmitter {
     finishDurableDelivery(delivery, outcome, evidence) {
         if (!delivery)
             return;
+        // Delivered or uncertain means the pane write started: the work reached
+        // the CLI (or may have), so it is activity (Phase 2a). Only a failure is
+        // not — a pre-write failure never gets here; it is retried before begin.
+        if (outcome !== "failed")
+            this.noteWorkActivity();
         this.deliveryOutbox?.complete(delivery.deliveryId, this.bootId, delivery.attemptNo, outcome, evidence);
     }
     finishDurableSubmission(delivery, verdict) {
@@ -1464,6 +1513,56 @@ export class Daemon extends EventEmitter {
             return;
         this.retryDurableDeliveryBeforeBegin(delivery, reason);
     }
+    /** Restart the idle clock for work that reached the CLI, and persist it like a user inbound. */
+    noteWorkActivity(now = Date.now()) {
+        this.autoPauseController.recordActivity(now);
+        try {
+            writeLastInboundAt(this.instanceDir, now);
+        }
+        catch (err) {
+            this.logger.warn({ err: err.message }, "Failed to persist last activity timestamp");
+        }
+    }
+    /**
+     * A daemon started to wake, restart or explicitly start an instance begins
+     * its idle window now (Phase 2a). Only the fleet's own boot keeps the
+     * persisted last-inbound seed, so a long-idle fleet still pauses after a
+     * restart; a woken instance used to inherit a week-old timestamp and pause
+     * one second after it became ready.
+     */
+    seedActivityNow(now = Date.now()) {
+        this.autoPauseController.recordActivity(now);
+    }
+    /**
+     * Phase 2b (design §1.4): why this awake instance is not taking input, in
+     * words for a notice, or null when nothing known holds it. Read from the
+     * daemon's own state only; it never inspects the pane itself.
+     */
+    notAcceptingReason() {
+        if (this.pauseWakeState !== "active")
+            return `it is ${this.pauseWakeState}`;
+        if (this.inputBlockedDialogKey !== null)
+            return `a dialog is holding its input (${this.inputBlockedDialogKey})`;
+        if (this.authFailureUnresolved)
+            return "its login is failing";
+        if (this.instanceState === "stuck")
+            return "the CLI looks stuck";
+        return null;
+    }
+    workLeaseCheck = null;
+    /** Lifecycle-provided: whether the fleet holds a work lease on this instance. */
+    setWorkLeaseCheck(check) {
+        this.workLeaseCheck = check ?? null;
+    }
+    workLeaseActive() {
+        try {
+            return this.workLeaseCheck?.() === true;
+        }
+        catch (err) {
+            this.logger.debug({ err }, "Work lease check failed; treating as no lease");
+            return false;
+        }
+    }
     async start() {
         mkdirSync(this.instanceDir, { recursive: true });
         writeFileSync(join(this.instanceDir, "daemon.pid"), String(process.pid));
@@ -2468,6 +2567,13 @@ export class Daemon extends EventEmitter {
                     return;
                 const pane = await this.tmux.capturePane();
                 const inputBlockedDialog = this.updateInputBlockedState(pane, dialogs);
+                // A sign-in screen that appears AFTER the startup scan ended (first run:
+                // theme → login can take longer than its budget) is the same auth
+                // incident the scan reports. Structural predicate only — the loose
+                // patterns are for a fresh startup pane, not a long-lived transcript.
+                const loginFlow = LOGIN_FLOWS[this.config.backend ?? "claude-code"];
+                if (loginFlow?.loginScreenActive && this.reportLoginScreen(pane))
+                    return;
                 const interactivePrompt = this.interactivePromptDetector.observe(pane, Date.now(), this.instanceStateLastOutputAt);
                 if (interactivePrompt) {
                     this.logger.warn(interactivePrompt, "Interactive terminal prompt is waiting for human input");
@@ -2995,33 +3101,128 @@ export class Daemon extends EventEmitter {
             this.logger.warn(record, `${cliLabel} exited on its own (exit code ${exitCode ?? "unknown"}) — last output recorded`);
         }
     }
-    /** Send the backend-specific graceful quit command/key sequence. */
-    async sendQuitSequence(reason = "unspecified") {
+    /**
+     * Send the backend-specific graceful quit command/key sequence.
+     *
+     * Observability (#1030): every send is logged with reason + caller, the
+     * outcome is logged, and a non-stop quit arms a bounded relaunch watch —
+     * sol's silent wedge (quit landed, codex exited to its resume screen, no
+     * relaunch, daemon still running) was invisible precisely because none of
+     * this existed.
+     */
+    async sendQuitSequence(reason = "unspecified", caller = "unknown", opts) {
         if (!this.tmux || !this.backend)
             return false;
+        const backend = this.backend.binaryName;
         const quitCmd = this.backend.getQuitCommand();
         const quitKey = this.backend.getQuitKey?.();
+        const quitVia = quitCmd ? `quit command ${quitCmd}` : quitKey ? `quit key ${quitKey}` : "no quit sequence";
+        this.logger.info({ backend, quit: quitCmd ?? quitKey ?? null, reason, caller }, `Sending backend quit sequence (${reason} via ${caller})`);
         if (quitCmd || quitKey)
-            this.noteStopRequest(quitCmd ? `quit command ${quitCmd}` : `quit key ${quitKey}`, reason);
+            this.noteStopRequest(quitVia, reason);
+        let sent = false;
+        let step = "sendKeys";
         if (quitCmd) {
-            if (!await this.tmux.sendKeys(quitCmd))
-                return false;
-            // Delay before Enter to prevent tmux server races when instances stop in
-            // parallel (same pattern as pasteText).
-            await new Promise(r => setTimeout(r, 150));
-            return this.tmux.sendSpecialKey("Enter");
+            if (await this.tmux.sendKeys(quitCmd)) {
+                // Delay before Enter to prevent tmux server races when instances stop in
+                // parallel (same pattern as pasteText).
+                await new Promise(r => setTimeout(r, 150));
+                step = "enter";
+                sent = await this.tmux.sendSpecialKey("Enter");
+            }
+        }
+        else if (quitKey) {
+            step = "quit key";
+            sent = true;
+            const presses = Math.max(1, Math.floor(this.backend.getQuitKeyPresses?.() ?? 1));
+            for (let i = 0; i < presses; i++) {
+                if (!await this.tmux.sendSpecialKey(quitKey)) {
+                    sent = false;
+                    break;
+                }
+                if (i + 1 < presses)
+                    await new Promise(r => setTimeout(r, 250));
+            }
         }
-        if (!quitKey)
+        else {
+            this.logger.warn({ backend, reason, caller }, "Backend has no quit command or key — nothing sent");
             return false;
-        const presses = Math.max(1, Math.floor(this.backend.getQuitKeyPresses?.() ?? 1));
-        for (let i = 0; i < presses; i++) {
-            const sent = await this.tmux.sendSpecialKey(quitKey);
-            if (!sent)
-                return false;
-            if (i + 1 < presses)
-                await new Promise(r => setTimeout(r, 250));
         }
-        return true;
+        if (sent) {
+            this.logger.info({ backend, quit: quitCmd ?? quitKey, reason, caller }, "Backend quit sequence sent");
+            if (opts?.watchRelaunch)
+                this.armQuitRelaunchWatch(reason, caller, opts.watchMs);
+        }
+        else {
+            this.logger.warn({ backend, quit: quitCmd ?? quitKey, reason, caller, step }, "Backend quit sequence failed to send");
+        }
+        return sent;
+    }
+    /**
+     * Codex names its own resume command on its exit screen — the wedge
+     * signature. Backend-generic shell, codex-specific match (#1030 ask 4):
+     * other backends' panes simply never match.
+     */
+    isCodexResumeScreen(pane) {
+        return /codex\s+resume\b/i.test(pane);
+    }
+    clearQuitRelaunchWatch() {
+        if (this.quitWatchTimer) {
+            clearTimeout(this.quitWatchTimer);
+            this.quitWatchTimer = null;
+        }
+        // Invalidates any already-started checkQuitRelaunch whose pane reads are
+        // still pending: it must bail on return, not warn from stale results.
+        this.quitWatchToken = null;
+    }
+    /**
+     * After a non-stop quit, warn if the backend exited but nothing relaunched
+     * within the bound while this daemon still considers itself active.
+     * Skipped when paused/pausing/waking/stopped or a relaunch (new spawn
+     * generation) already happened — only the wedged state warns.
+     */
+    armQuitRelaunchWatch(reason, caller, delayMs = QUIT_RELAUNCH_WATCH_MS) {
+        this.clearQuitRelaunchWatch();
+        const generation = this.spawnGeneration;
+        const quitAt = Date.now();
+        const token = {};
+        this.quitWatchToken = token;
+        this.quitWatchTimer = setTimeout(() => {
+            this.quitWatchTimer = null;
+            void this.checkQuitRelaunch({ reason, caller, generation, quitAt, token }).catch(() => { });
+        }, delayMs);
+    }
+    async checkQuitRelaunch(opts) {
+        if (this.pauseWakeState !== "active")
+            return;
+        if (opts.generation !== this.spawnGeneration)
+            return;
+        if (!this.tmux)
+            return;
+        // TOCTOU guard: the token is captured at arm time, so a beginSpawn/stop
+        // that lands while the pane reads below are pending invalidates this
+        // check. Re-validate after every await before using the results.
+        const token = opts.token ?? this.quitWatchToken;
+        let pane;
+        let alive = true;
+        try {
+            const status = await this.tmux.getPaneStatus();
+            alive = !!status?.alive;
+            pane = await this.tmux.capturePane().catch(() => undefined);
+        }
+        catch {
+            return;
+        }
+        if (this.pauseWakeState !== "active")
+            return;
+        if (opts.generation !== this.spawnGeneration)
+            return;
+        if (token !== this.quitWatchToken)
+            return;
+        const resumeScreen = !!pane && this.isCodexResumeScreen(pane);
+        if (alive && !resumeScreen)
+            return;
+        this.logger.warn({ backend: this.backend?.binaryName, reason: opts.reason, caller: opts.caller, quitAt: opts.quitAt, paneAlive: alive, resumeScreen }, "codex-exited-without-relaunch: backend quit but no relaunch followed while this daemon is still active");
     }
     /**
      * Kiro drops `/quit` while a turn is running. Stop that turn first, then use
@@ -3045,6 +3246,7 @@ export class Daemon extends EventEmitter {
     }
     async stop() {
         this.logger.info("Stopping daemon instance");
+        this.clearQuitRelaunchWatch();
         this.fenceDeliveryWritesForStop();
         this.turnReplyGuard.reset();
         // Invalidate any bounded pre-Enter wait from the process generation being
@@ -3071,7 +3273,7 @@ export class Daemon extends EventEmitter {
             let killed = false;
             const windowId = this.tmux.getWindowId();
             const kiroReady = windowId ? await this.drainBusyKiroForStop(windowId) : true;
-            const quitSent = kiroReady && await this.sendQuitSequence("graceful stop");
+            const quitSent = kiroReady && await this.sendQuitSequence("graceful stop", "stop");
             const quitGraceMs = this.backend?.binaryName === "kiro-cli"
                 ? KIRO_STOP_QUIT_GRACE_MS : 3_000;
             const sigtermGraceMs = this.backend?.binaryName === "kiro-cli"
@@ -3190,12 +3392,35 @@ export class Daemon extends EventEmitter {
      * failures: pause() alone no-ops while the CLI is busy/stuck, which is the
      * usual state when the error surfaces. Cleared by a successful pause or wake.
      */
-    requestPauseWhenIdle() {
+    requestPauseWhenIdle(opts = {}) {
         if (this.pauseWakeState === "paused")
             return;
+        const reconfirm = opts.reconfirmAuth === true;
+        // An auth request wins over a pending non-auth one: the marker must not
+        // later read as auto-wakeable when a login failure is among the causes.
+        const reason = opts.reason ?? "auth";
+        this.pendingPauseReason = this.pausePending && this.pendingPauseReason === "auth" ? "auth" : reason;
+        // A pause already pending unconditionally stays unconditional.
+        this.pauseReconfirmAuth = this.pausePending ? this.pauseReconfirmAuth && reconfirm : reconfirm;
         this.pausePending = true;
-    }
-    async pause() {
+        // A conditional request is the only pause path it has (#1044): decide on
+        // the current pane now instead of at the next state sweep, so a pane that
+        // is already idle is not left open to a delivery in the meantime.
+        if (reconfirm)
+            void this.captureAndEvaluateInstanceState("deferred_pause");
+    }
+    /** The backend's own auth-error pattern, within the pane's last rows. */
+    authErrorAtPaneBottom(pane) {
+        const rows = pane.replace(/\r/g, "").split("\n").filter(r => r.trim() !== "");
+        const bottom = rows.slice(-Daemon.AUTH_RECONFIRM_ROWS).join("\n");
+        return (this.backend?.getErrorPatterns?.() ?? [])
+            .filter(ep => ep.type === "auth_error")
+            .some(ep => new RegExp(ep.pattern.source, ep.pattern.flags.replace("g", "")).test(bottom));
+    }
+    static AUTH_RECONFIRM_ROWS = 15;
+    /** Why the deferred pause was requested; what its marker records (Phase 2a). */
+    pendingPauseReason = "auth";
+    async pause(reason = "operator") {
         if (this.pauseWakeState === "paused")
             return;
         if (this.pauseWakeState === "pausing" || this.pauseWakeState === "waking") {
@@ -3223,7 +3448,7 @@ export class Daemon extends EventEmitter {
         const transition = (async () => {
             try {
                 this.saveSessionId();
-                await this.sendQuitSequence(pauseReason);
+                await this.sendQuitSequence(pauseReason, "pause", { watchRelaunch: true });
                 let exited = false;
                 for (let i = 0; i < 15; i++) {
                     await new Promise(r => setTimeout(r, 200));
@@ -3248,7 +3473,7 @@ export class Daemon extends EventEmitter {
                 }
                 this.pauseWakeState = "paused";
                 this.autoPauseController.markPaused();
-                writePausedMarker(this.instanceDir, this.lastPausedAt ?? Date.now());
+                writePausedMarker(this.instanceDir, this.lastPausedAt ?? Date.now(), reason);
                 this.logger.info({ pausedAt: this.lastPausedAt }, "Instance auto-paused");
                 this.ipcServer?.broadcast({
                     type: "instance_state", instanceName: this.name, state: "paused", pausedAt: this.lastPausedAt,
@@ -3277,6 +3502,7 @@ export class Daemon extends EventEmitter {
         // An explicit wake (e.g. after the user re-logs in) cancels a deferred
         // auth pause — otherwise the instance would pause again the moment it idles.
         this.pausePending = false;
+        this.pauseReconfirmAuth = false;
         this.authFailureUnresolved = false;
         this.loginScreenReported = false;
         this.fatalStartupReported = false;
@@ -3404,8 +3630,20 @@ export class Daemon extends EventEmitter {
         // live, kept its warm slot, and re-raised a hang alert on every scan.
         if (this.pausePending && (snapshot.state === "idle" || snapshot.state === "stuck") && this.pasteQueueDepth === 0) {
             this.pausePending = false;
+            const reconfirm = this.pauseReconfirmAuth;
+            this.pauseReconfirmAuth = false;
+            // An unreadable pane keeps the pause: when in doubt, stop feeding a CLI
+            // that may be unable to answer (42512a46).
+            if (reconfirm && pane !== undefined && !this.authErrorAtPaneBottom(pane)) {
+                this.authFailureUnresolved = false;
+                this.logger.warn({ state: snapshot.state }, "Deferred auth pause dropped: the auth-error text is no longer at the bottom of the pane at the end of the turn, "
+                    + "and this backend has no auth check to confirm it (likely conversation text)");
+                return;
+            }
             this.pauseAllowStuck = snapshot.state === "stuck";
-            this.emit("auto_pause_requested", { name: this.name, idleSince: snapshot.stateChangedAt });
+            const reason = this.pendingPauseReason;
+            this.pendingPauseReason = "auth";
+            this.emit("auto_pause_requested", { name: this.name, idleSince: snapshot.stateChangedAt, reason });
             return;
         }
         // A restart deferred by a dead MCP server: the turn it must not interrupt is
@@ -3414,9 +3652,14 @@ export class Daemon extends EventEmitter {
             this.fireMcpRestartRequest("idle_edge");
             return;
         }
-        if (!this.pauseRequested && this.pasteQueueDepth === 0 && this.autoPauseController.observe(snapshot.state)) {
+        // A work lease (Phase 2a) holds only this idle-timeout pause: a target the
+        // fleet just woke for queued work, or one with a submission in flight, must
+        // not be put back to sleep before that work reaches it. The auth-deferred
+        // pause above and operator pauses never consult it.
+        if (!this.pauseRequested && this.pasteQueueDepth === 0 && this.autoPauseController.observe(snapshot.state)
+            && !this.workLeaseActive()) {
             this.pauseRequested = true;
-            this.emit("auto_pause_requested", { name: this.name, idleSince: snapshot.stateChangedAt });
+            this.emit("auto_pause_requested", { name: this.name, idleSince: snapshot.stateChangedAt, reason: "idle" });
         }
     }
     /**
@@ -7121,6 +7364,8 @@ export class Daemon extends EventEmitter {
      * into exactly the window this exists to close.
      */
     beginSpawn() {
+        // A relaunch settles any outstanding quit watch: the CLI came back.
+        this.clearQuitRelaunchWatch();
         // A restarted CLI has no trustworthy turn edge for the process it replaced.
         // v1 deliberately does not persist obligations across restarts: missing one
         // warning is safer than treating the replacement's startup idle as the old
@@ -7871,6 +8116,29 @@ export class Daemon extends EventEmitter {
             this.firstDeliveryDelay.recordReady();
         return ready;
     }
+    /**
+     * Report the CLI's logged-out startup screen as an auth incident. Returns true
+     * when this call reported it (the scan is then over: only re-login ends the
+     * state); false when the screen is absent or was already reported this spawn.
+     * `loginScreenActive`, when the backend flow has one, replaces the bare
+     * pattern test so a transcript that merely mentions the screen cannot trip it.
+     */
+    reportLoginScreen(pane) {
+        const flow = LOGIN_FLOWS[this.config.backend ?? "claude-code"];
+        const onScreen = flow?.loginScreenActive ? flow.loginScreenActive(pane) : flow?.loginScreenPattern?.test(pane) === true;
+        if (!onScreen || this.loginScreenReported)
+            return false;
+        this.loginScreenReported = true;
+        this.authFailureUnresolved = true;
+        this.logger.warn({ backend: this.config.backend }, "CLI is waiting at its sign-in screen — reporting auth error");
+        this.emit("pty_error", {
+            name: this.name,
+            type: "auth_error",
+            action: "pause",
+            message: "CLI is waiting at its sign-in screen — credentials are missing or expired",
+        });
+        return true;
+    }
     /**
      * Poll the pane within a wall-clock budget, dismiss the confirmation dialogs
      * the backend knows, and return true once the CLI shows a ready prompt with
@@ -8046,6 +8314,11 @@ export class Daemon extends EventEmitter {
                 }
                 if (this.dialogParkedSince !== 0 && !this.deliveryBlockingDialogs().some(d => Daemon.dialogMatches(d, pane)))
                     this.trackDialogParked(null);
+                // A sign-in screen that carries the ready pattern's own glyph (Claude's
+                // `❯ 1. …` login menu) must be recognised BEFORE the ready check, or the
+                // ready check swallows it and the incident is never reported (#1074).
+                if (LOGIN_FLOWS[this.config.backend ?? "claude-code"]?.loginScreenBeforeReady && this.reportLoginScreen(pane))
+                    return true;
                 // CLI is ready (pattern defined by each backend). Require it on two
                 // consecutive polls: the first ready frame is also the moment a late
                 // dialog is about to be painted over it.
@@ -8076,19 +8349,8 @@ export class Daemon extends EventEmitter {
                 // an auth error (the lifecycle double-checks with the token-free probe)
                 // instead of letting it decay into crash/hang/MCP-died noise, and stop
                 // the retry loop — only re-login (see /login) ends this state.
-                const loginScreen = LOGIN_FLOWS[this.config.backend ?? "claude-code"]?.loginScreenPattern;
-                if (loginScreen?.test(pane) && !this.loginScreenReported) {
-                    this.loginScreenReported = true;
-                    this.authFailureUnresolved = true;
-                    this.logger.warn({ backend: this.config.backend }, "CLI is waiting at its sign-in screen — reporting auth error");
-                    this.emit("pty_error", {
-                        name: this.name,
-                        type: "auth_error",
-                        action: "pause",
-                        message: "CLI is waiting at its sign-in screen — credentials are missing or expired",
-                    });
+                if (this.reportLoginScreen(pane))
                     return true;
-                }
                 // Fatal: command not found (must match full phrase to avoid false positives
                 // like Kiro's "agent X not found, using default")
                 if (/command not found|: not found$/m.test(pane))