@songsid/agend 2.1.9 → 2.1.10-beta.2

package/dist/daemon.js

@@ -1,6 +1,7 @@
 import { join, dirname, basename, resolve } from "node:path";
 import { mkdirSync, writeFileSync, readFileSync, readdirSync, existsSync, unlinkSync, rmSync, appendFileSync, statSync, renameSync } from "node:fs";
 import { fileURLToPath } from "node:url";
+import { ensureInstanceDir } from "./private-dir.js";
 import { createHash, randomBytes, randomUUID } from "node:crypto";
 import { EventEmitter } from "node:events";
 import { rotateLogIfNeeded } from "./logger.js";
@@ -18,7 +19,7 @@ import { ContextGuardian } from "./context-guardian.js";
 import { IpcServer } from "./channel/ipc-bridge.js";
 import { daemonBudgetMs } from "./channel/ipc-timeouts.js";
 import { MessageBus } from "./channel/message-bus.js";
-import { shellQuote } from "./backend/types.js";
+import { shellQuote, UnsupportedCliError } from "./backend/types.js";
 import { getTmuxSession } from "./config.js";
 import { routeToolCall } from "./channel/tool-router.js";
 import { HangDetector } from "./hang-detector.js";
@@ -97,6 +98,13 @@ export const DEFAULT_STATE_SAFETY_SWEEP_MS = 60_000;
 export const PERIODIC_REDRAW_PROBE_MS = 25;
 /** Do not turn a streaming Codex turn into one capture-pane per burst. */
 export const STRUCTURED_IDLE_PROBE_MS = 500;
+/**
+ * An instance publishing more than this many execution-state edges inside the
+ * window is flapping; its edges stop being logged one by one (#1116). One
+ * flapping muse wrote 89,532 of the fleet's 91,000 state-change lines.
+ */
+export const STATE_EDGE_LOG_BURST = 10;
+export const STATE_EDGE_LOG_WINDOW_MS = 60_000;
 /** How long an unrecognised-but-empty composer must stay unchanged before it counts as idle (#978). */
 export const UNKNOWN_LAYOUT_STABLE_MS = 10_000;
 /** A foreground server/port-forward should hand control back or be acknowledged. */
@@ -161,7 +169,7 @@ export function readLastInboundAt(instanceDir, now = Date.now()) {
 }
 /** Atomically persist the last real channel inbound timestamp. */
 export function writeLastInboundAt(instanceDir, timestamp) {
-    mkdirSync(instanceDir, { recursive: true });
+    ensureInstanceDir(instanceDir);
     const target = join(instanceDir, LAST_INBOUND_FILE);
     const temp = `${target}.${process.pid}.tmp`;
     writeFileSync(temp, String(timestamp));
@@ -1020,6 +1028,15 @@ export class Daemon extends EventEmitter {
     /** Generation/signature fence for safety dialogs while their screen is still visible. */
     autoResolvedDialogKey = null;
     autoResolvedDialogGeneration = 0;
+    /**
+     * (#1103) Proof that THIS spawn's Codex went onto Luna Reserve: AgEnD saw (and
+     * answered) the usage-limit menu that codex shows when it switches itself to
+     * the reserve. `count` is how many usage-limit lines the pane held at that
+     * moment; only occurrences up to it are the text of the incident we already
+     * handled. A later, larger count is a NEW hit (the reserve ran out too).
+     * Keyed by spawn generation, so a respawn starts without any proof.
+     */
+    codexReserveAck = null;
     backgroundSessionRecoveryAttempted = false;
     /** Whether the last spawn started a fresh session (not resumed). */
     isNewSession = false;
@@ -1037,6 +1054,10 @@ export class Daemon extends EventEmitter {
     /** Consecutive structural idle captures for a noisy, periodically-redrawing
      * pane.  Two are required before a working -> idle transition. */
     instanceStatePeriodicIdleConfirmations = 0;
+    /** Recent execution-state edge times, for the flap log limiter (#1116). */
+    stateEdgeTimes = [];
+    stateEdgesNotLogged = 0;
+    stateEdgeSummaryAt = 0;
     /** #978: last unrecognised-but-empty-composer capture and since when it has been unchanged. */
     unknownLayoutPaneKey = null;
     unknownLayoutStableSince = 0;
@@ -1178,6 +1199,17 @@ export class Daemon extends EventEmitter {
     static MCP_DEATH_COOLDOWN_MS = 5 * 60_000;
     lastMcpDeathNotifiedAt = 0;
     mcpDeathNotifiedForPid = null;
+    /**
+     * How long after the CLI finished starting its fleet MCP server may take to
+     * show up before "it never connected" is an incident. Servers connect within
+     * seconds of the prompt (kiro loads MCP after it, claude/codex around it);
+     * 90s leaves room for a slow npx/node cold start on a loaded host.
+     */
+    static FLEET_MCP_STARTUP_GRACE_MS = 90_000;
+    /** spawnGeneration in which a server for this instance was last seen serving. */
+    mcpServedGeneration = -1;
+    /** spawnGeneration whose "fleet MCP never connected" incident was reported. */
+    mcpNeverConnectedReportedGeneration = -1;
     /**
      * Ceiling on how long a dead MCP server waits for an idle window before the
      * auto-restart fires anyway. A toolless instance cannot reply or report, so
@@ -1264,6 +1296,27 @@ export class Daemon extends EventEmitter {
             return false;
         }
     }
+    /** The Codex "usage limit reached" pause pattern (the only one the reserve proof applies to). */
+    static isCodexUsageLimitPause(ep) {
+        return ep.type === "quota" && ep.action === "pause" && ep.message.startsWith("Codex usage limit reached");
+    }
+    static countPatternMatches(pattern, text) {
+        const flags = pattern.flags.includes("g") ? pattern.flags : pattern.flags + "g";
+        return (text.match(new RegExp(pattern.source, flags)) || []).length;
+    }
+    /**
+     * Record that the Luna Reserve menu is (or was just) on screen, for the
+     * usage-limit decision in evaluateErrorPatterns. Called wherever a dialog is
+     * matched, before any key is sent.
+     */
+    noteCodexReserveDialog(dialog, pane) {
+        if (dialog.autoResolutionKey !== "codex-usage-limit-luna-reserve" || this.backend?.binaryName !== "codex")
+            return;
+        const ep = this.backend.getErrorPatterns?.().find(candidate => Daemon.isCodexUsageLimitPause(candidate));
+        const count = ep ? Daemon.countPatternMatches(ep.pattern, pane) : 0;
+        this.codexReserveAck = { generation: this.spawnGeneration, count };
+        this.logger.info({ backend: "codex", evidence: "reserve-menu-seen", usageLimitLines: count }, "codex usage-limit: the Luna Reserve menu was seen — usage-limit text up to this count is stale");
+    }
     /** Re-check the current pane after an async quota probe before pausing it. */
     async isCodexLivePane() {
         if (this.backend?.binaryName !== "codex" || !this.tmux)
@@ -1540,13 +1593,13 @@ export class Daemon extends EventEmitter {
      */
     notAcceptingReason() {
         if (this.pauseWakeState !== "active")
-            return `it is ${this.pauseWakeState}`;
+            return t("delivery.hold_state", this.pauseWakeState);
         if (this.inputBlockedDialogKey !== null)
-            return `a dialog is holding its input (${this.inputBlockedDialogKey})`;
+            return t("delivery.hold_dialog", this.inputBlockedDialogKey);
         if (this.authFailureUnresolved)
-            return "its login is failing";
+            return t("delivery.hold_auth");
         if (this.instanceState === "stuck")
-            return "the CLI looks stuck";
+            return t("delivery.hold_stuck");
         return null;
     }
     workLeaseCheck = null;
@@ -1564,7 +1617,7 @@ export class Daemon extends EventEmitter {
         }
     }
     async start() {
-        mkdirSync(this.instanceDir, { recursive: true });
+        ensureInstanceDir(this.instanceDir);
         writeFileSync(join(this.instanceDir, "daemon.pid"), String(process.pid));
         this.logger.info(`Starting ${this.name}`);
         // P1: Read crash state from previous run — skip resume if last run was a crash loop
@@ -1843,6 +1896,7 @@ export class Daemon extends EventEmitter {
             return;
         const status = mcpServerState(this.instanceDir);
         if (status.state === "alive") {
+            this.mcpServedGeneration = this.spawnGeneration;
             if (this.mcpDeathNotifiedForPid != null)
                 this.emitMcpRecovered("pid", status.pid);
             this.mcpDeathNotifiedForPid = null; // the CLI respawned it — re-arm
@@ -1870,8 +1924,13 @@ export class Daemon extends EventEmitter {
             this.noteMcpProofOfLife("connection", livePid);
             return;
         }
-        if (status.state === "unknown")
-            return; // never started, or nothing left to watch — as before
+        if (status.state === "unknown") {
+            // Never started, or nothing left to watch. The CLI may legitimately not
+            // have spawned it yet, but past the startup grace a CLI that never
+            // brought up AgEnD's server is running without fleet tools.
+            this.checkFleetMcpNeverConnected();
+            return;
+        }
         // Dead: report once per pid, and at most once per cooldown window.
         if (this.mcpDeathNotifiedForPid === status.pid)
             return;
@@ -1906,6 +1965,42 @@ export class Daemon extends EventEmitter {
         if (autoRestart)
             this.armMcpRestartWhenIdle();
     }
+    /**
+     * The CLI has been up past the startup grace and no server for this instance
+     * has ever served it: AgEnD's fleet tools are missing, so the agent cannot
+     * reply or reach other instances. Reported once per spawn, through the same
+     * mcp_died path (auth check, notice, idle-gated revival restart under the
+     * lifecycle's cooldown) as a server that died.
+     *
+     * This is the check that lets the CLI tolerate the user's OTHER servers
+     * failing (#1111): kiro's --require-mcp-startup could not tell the fleet
+     * server from a third-party one, and failed the whole launch for either.
+     */
+    checkFleetMcpNeverConnected() {
+        if (this.config.agent_mode === "cli")
+            return; // no MCP server by design
+        if (this.lastSpawnAt === 0 || this.mcpServedGeneration === this.spawnGeneration)
+            return;
+        if (this.mcpNeverConnectedReportedGeneration === this.spawnGeneration)
+            return;
+        if (Date.now() - this.lastSpawnAt < Daemon.FLEET_MCP_STARTUP_GRACE_MS)
+            return;
+        if (this.liveMcpSockets().length > 0) {
+            this.mcpServedGeneration = this.spawnGeneration;
+            return;
+        }
+        this.mcpNeverConnectedReportedGeneration = this.spawnGeneration;
+        // pid 0 stands for "no server ever had a pid": a late-connecting server
+        // still finds an alarm to retract (noteMcpProofOfLife / the alive branch).
+        this.mcpDeathNotifiedForPid = 0;
+        const authSuspected = this.authFailureUnresolved;
+        const autoRestart = this.config.mcp_auto_restart !== false && !authSuspected;
+        this.lastMcpDeathNotifiedAt = Date.now();
+        this.logger.error({ autoRestart, authSuspected, graceMs: Daemon.FLEET_MCP_STARTUP_GRACE_MS }, "Fleet MCP server never connected after the CLI started — instance has no agend tools");
+        this.emit("mcp_died", { name: this.name, pid: 0, autoRestart, authSuspected, neverConnected: true });
+        if (autoRestart)
+            this.armMcpRestartWhenIdle();
+    }
     /**
      * Whether a dead MCP pid on this backend is more likely a CLI-driven
      * replacement than a real loss. Codex only, and deliberately narrow: it is
@@ -2124,6 +2219,7 @@ export class Daemon extends EventEmitter {
      * if an alarm had already gone out, retracts it.
      */
     noteMcpProofOfLife(source, pid) {
+        this.mcpServedGeneration = this.spawnGeneration;
         const hadAlarm = this.mcpDeathNotifiedForPid != null;
         const hadPending = this.mcpRestartPending || this.mcpDeathDeferredForPid != null;
         if (!hadAlarm && !hadPending)
@@ -2139,6 +2235,9 @@ export class Daemon extends EventEmitter {
             this.repairMcpPidSlot(pid, source);
     }
     emitMcpRecovered(source, pid) {
+        // The retracted alarm was "never connected", not a death: say so.
+        if (this.mcpDeathNotifiedForPid === 0)
+            source = "late_connect";
         this.logger.info({ source, pid }, "MCP server recovered — retracting the earlier death report");
         this.emit("mcp_recovered", { name: this.name, source, pid });
     }
@@ -2462,6 +2561,10 @@ export class Daemon extends EventEmitter {
                             this.stormWindow?.markRecovered(this.name);
                     }
                     catch (err) {
+                        if (this.stopOnUnsupportedCliRespawn(err)) {
+                            this.healthCheckTimer = null;
+                            return;
+                        }
                         if (!this.handOffBackendUnreachableRespawn(err)) {
                             this.logger.error({ err }, `Failed to respawn ${cliLabel} window`);
                         }
@@ -2495,6 +2598,21 @@ export class Daemon extends EventEmitter {
      * the pane monitor to capture it would re-create a false idle state.
      */
     /**
+     * The CLI was replaced under a running instance by one that cannot run it as
+     * configured — kiro-cli auto-updating to a release without the instance's
+     * UI/engine (#1109). Every respawn would refuse the same way, so stop
+     * supervising and say why instead of failing every 30 seconds.
+     */
+    stopOnUnsupportedCliRespawn(err) {
+        if (!(err instanceof UnsupportedCliError))
+            return false;
+        this.healthCheckPaused = true;
+        this.logger.error({ reason: err.message }, "Not respawning: the installed CLI cannot run this instance as configured");
+        this.emitSupervisionEnded(err.message, "Install a CLI version that supports this instance's configuration, then start it again.");
+        return true;
+    }
+    /**
+     * Announce that this instance is no longer being supervised.  /**
      * Announce that this instance is no longer being supervised.
      *
      * Four health-check exits set `healthCheckPaused = true` and returned without
@@ -2608,6 +2726,7 @@ export class Daemon extends EventEmitter {
                 for (const dialog of dialogs) {
                     if (!Daemon.dialogMatches(dialog, pane))
                         continue;
+                    this.noteCodexReserveDialog(dialog, pane);
                     if (dialog.blocksDelivery || dialog.holdOnly)
                         blockingSeen = dialog;
                     if (dialog.holdOnly)
@@ -2795,15 +2914,29 @@ export class Daemon extends EventEmitter {
             const count = countMatches(ep.pattern);
             const seen = this.lastErrorCount.get(key) ?? 0;
             // The generic Codex usage-limit line remains in scrollback after the
-            // user selects Luna Reserve. A live composer + Context footer is positive
-            // evidence that the pane recovered; baseline the stale occurrence so it
-            // cannot reach the destructive quota pause path.
-            if (codexLivePane && ep.type === "quota" && ep.action === "pause"
-                && ep.message.startsWith("Codex usage limit reached")) {
-                if (count > 0)
+            // user selects Luna Reserve. What proves that is NOT a live composer —
+            // codex paints one after a plain hit too, and the old live-composer guard
+            // therefore swallowed every real hit (#1103) — but AgEnD having seen the
+            // Luna Reserve menu in this spawn. Occurrences up to that point are the
+            // incident we already handled; anything beyond is a new hit and goes on to
+            // the lifecycle, whose usage probe decides (available / reserve / exhausted).
+            if (Daemon.isCodexUsageLimitPause(ep)) {
+                const ack = this.codexReserveAck;
+                if (ack && ack.generation === this.spawnGeneration && count <= ack.count) {
+                    // The handled incident's lines can scroll out of the capture buffer as
+                    // the reserve session talks on. Lower the proof (and the occurrence
+                    // baseline) with them: otherwise `count <= ack.count` would keep
+                    // swallowing a NEW hit — the reserve running out — until it exceeded
+                    // the menu-time count, and it would never reach the lifecycle probe.
+                    if (count < ack.count)
+                        ack.count = count;
                     this.lastErrorCount.set(key, count);
-                this.logger.debug("Codex usage-limit text is stale — live reserve pane is running");
-                continue;
+                    this.logger.debug({ decision: "ignore", evidence: "reserve-menu-seen", count, acknowledged: ack.count }, "codex usage-limit: stale text of the handled Luna Reserve incident");
+                    continue;
+                }
+                if (count > seen) {
+                    this.logger.info({ decision: "candidate", evidence: ack && ack.generation === this.spawnGeneration ? "new-hit-after-reserve" : "no-reserve-proof", count, livePane: codexLivePane }, "codex usage-limit: new occurrence — handing to the quota probe");
+                }
             }
             if (count <= seen) {
                 // Occurrences scrolled out of the capture buffer → lower the baseline
@@ -3575,6 +3708,39 @@ export class Daemon extends EventEmitter {
                 this.pauseWakeTransition = null;
         }
     }
+    /**
+     * One line per edge, until the instance is flapping. The edges themselves are
+     * still published to every observer; only the log is limited, so the next
+     * flap cannot bury the fleet log the way #1116 did. The first suppressed edge
+     * warns once (that is the signal worth reading), and a summary follows every
+     * window while it lasts and at the first edge after it calms.
+     */
+    logStateEdge(previous, snapshot) {
+        const now = Date.now();
+        this.stateEdgeTimes.push(now);
+        while (this.stateEdgeTimes.length && this.stateEdgeTimes[0] <= now - STATE_EDGE_LOG_WINDOW_MS)
+            this.stateEdgeTimes.shift();
+        const edges = this.stateEdgeTimes.length;
+        const fields = { previousState: previous, state: snapshot.state, unchangedForMs: snapshot.unchangedForMs };
+        if (edges <= STATE_EDGE_LOG_BURST) {
+            if (this.stateEdgesNotLogged > 0) {
+                this.logger.info({ notLogged: this.stateEdgesNotLogged }, "Instance execution state edges were not logged while it was flapping");
+                this.stateEdgesNotLogged = 0;
+            }
+            this.logger.info(fields, "Instance execution state changed");
+            return;
+        }
+        if (this.stateEdgesNotLogged === 0) {
+            this.stateEdgeSummaryAt = now;
+            this.logger.warn({ edgesInWindow: edges, windowMs: STATE_EDGE_LOG_WINDOW_MS, backend: this.backend?.binaryName }, "Instance execution state is flapping — per-edge logging suppressed; the pane is repainting without a stable idle/working reading");
+        }
+        else if (now - this.stateEdgeSummaryAt >= STATE_EDGE_LOG_WINDOW_MS) {
+            this.stateEdgeSummaryAt = now;
+            this.logger.warn({ notLogged: this.stateEdgesNotLogged, edgesInWindow: edges }, "Instance execution state is still flapping");
+        }
+        this.stateEdgesNotLogged++;
+        this.logger.debug(fields, "Instance execution state changed (flapping, not logged at info)");
+    }
     applyInstanceStateSnapshot(snapshot, pane) {
         const previous = this.instanceState;
         // A live safety menu owns stdin.  It may still contain Claude's persistent
@@ -3610,11 +3776,7 @@ export class Daemon extends EventEmitter {
                 this.maybeProxyReplyOnTurnEnd(pane);
         }
         if (snapshot.state !== previous) {
-            this.logger.info({
-                previousState: previous,
-                state: snapshot.state,
-                unchangedForMs: snapshot.unchangedForMs,
-            }, "Instance execution state changed");
+            this.logStateEdge(previous, snapshot);
             this.emit("instance_state", { name: this.name, ...snapshot });
             this.ipcServer?.broadcast({ type: "instance_state", instanceName: this.name, ...snapshot });
             if (snapshot.state === "stuck" && pane && this.instanceStateReadyPattern) {
@@ -4112,6 +4274,19 @@ export class Daemon extends EventEmitter {
                 if (reason === "delivery_idle_gate")
                     this.resetFooterFallback();
                 if (reason === "output_probe") {
+                    // A TUI that repaints without changing a cell (muse parks its cursor
+                    // ~18 times a second while idle; Codex's star field) never leaves the
+                    // probe a quiet moment, so roughly one probe in five is overtaken. An
+                    // idle instance whose stale capture still shows the structural idle
+                    // layout has nothing to be called working on: the rule that cosmetic
+                    // output must not manufacture an idle -> working edge holds here too.
+                    // Without this every overtaken probe published the edge, and the next
+                    // clean probe (one confirmation was still banked) published it back —
+                    // a pair every ~1.6s for as long as the repaint lasted (#1116). Real
+                    // work cannot hide behind this: it changes the layout, the next
+                    // output arms another probe, and the debounce capture still fires.
+                    if (this.instanceState === "idle" && this.backend?.isPeriodicRedrawIdlePane?.(pane) === true)
+                        return;
                     // A second burst overtook the probe. Do not bless its stale capture as
                     // idle: continuous output is the safer working signal, while the next
                     // quiet probe/debounce can still prove that it was only repainting.
@@ -5476,10 +5651,18 @@ export class Daemon extends EventEmitter {
             return { state: "unknown" };
         }
         for (const dialog of dialogs) {
-            if (Daemon.dialogMatches(dialog, pane)) {
-                this.trackDialogParked(dialog);
-                return { state: "dialog", dialog };
-            }
+            if (!Daemon.dialogMatches(dialog, pane))
+                continue;
+            // Same one-shot fence as the scanners: a safety choice already sent in
+            // this spawn is not what the screen is waiting on any more — the entry
+            // after it (typically a hold) describes it.
+            if (dialog.verifyAfterKeys && dialog.autoResolutionKey
+                && this.autoResolvedDialogGeneration === this.spawnGeneration
+                && this.autoResolvedDialogKey === dialog.autoResolutionKey
+                && dialogs.some(other => other !== dialog && Daemon.dialogMatches(other, pane)))
+                continue;
+            this.trackDialogParked(dialog);
+            return { state: "dialog", dialog };
         }
         this.trackDialogParked(null);
         return { state: "clear" };
@@ -8188,7 +8371,7 @@ export class Daemon extends EventEmitter {
                 // Transient tmux trouble is not evidence about the CLI. Returning false
                 // here would clear the session; retry within the budget instead.
                 captureFailures++;
-                this.logger.warn({ err, captureFailures }, "capture-pane failed during the startup dialog scan — retrying");
+                this.logger.debug({ err, captureFailures }, "capture-pane failed during the startup dialog scan — retrying");
                 // Only a POSITIVE dead pane ends the scan; an unanswerable liveness
                 // query is the same transient trouble and must not clear the session.
                 if ((await this.paneLiveness()) === "dead")
@@ -8220,6 +8403,7 @@ export class Daemon extends EventEmitter {
                                     && this.autoResolvedDialogKey === dialog.autoResolutionKey)))
                             continue;
                         lastDialog = dialog;
+                        this.noteCodexReserveDialog(dialog, pane);
                         cleanReadyPolls = 0;
                         // Start the parked clock for every delivery-blocking dialog, exact
                         // ones included: if the auto-dismiss keeps failing here, the human