@songsid/agend 2.1.10-beta.1 → 2.1.10-beta.2

package/dist/daemon.d.ts

@@ -47,6 +47,13 @@ export declare const DEFAULT_STATE_SAFETY_SWEEP_MS = 60000;
 export declare const PERIODIC_REDRAW_PROBE_MS = 25;
 /** Do not turn a streaming Codex turn into one capture-pane per burst. */
 export declare const STRUCTURED_IDLE_PROBE_MS = 500;
+/**
+ * An instance publishing more than this many execution-state edges inside the
+ * window is flapping; its edges stop being logged one by one (#1116). One
+ * flapping muse wrote 89,532 of the fleet's 91,000 state-change lines.
+ */
+export declare const STATE_EDGE_LOG_BURST = 10;
+export declare const STATE_EDGE_LOG_WINDOW_MS = 60000;
 /** How long an unrecognised-but-empty composer must stay unchanged before it counts as idle (#978). */
 export declare const UNKNOWN_LAYOUT_STABLE_MS = 10000;
 /** A foreground server/port-forward should hand control back or be acknowledged. */
@@ -472,6 +479,15 @@ export declare class Daemon extends EventEmitter {
     /** Generation/signature fence for safety dialogs while their screen is still visible. */
     private autoResolvedDialogKey;
     private autoResolvedDialogGeneration;
+    /**
+     * (#1103) Proof that THIS spawn's Codex went onto Luna Reserve: AgEnD saw (and
+     * answered) the usage-limit menu that codex shows when it switches itself to
+     * the reserve. `count` is how many usage-limit lines the pane held at that
+     * moment; only occurrences up to it are the text of the incident we already
+     * handled. A later, larger count is a NEW hit (the reserve ran out too).
+     * Keyed by spawn generation, so a respawn starts without any proof.
+     */
+    private codexReserveAck;
     private backgroundSessionRecoveryAttempted;
     /** Whether the last spawn started a fresh session (not resumed). */
     isNewSession: boolean;
@@ -488,6 +504,10 @@ export declare class Daemon extends EventEmitter {
     /** Consecutive structural idle captures for a noisy, periodically-redrawing
      * pane.  Two are required before a working -> idle transition. */
     private instanceStatePeriodicIdleConfirmations;
+    /** Recent execution-state edge times, for the flap log limiter (#1116). */
+    private stateEdgeTimes;
+    private stateEdgesNotLogged;
+    private stateEdgeSummaryAt;
     /** #978: last unrecognised-but-empty-composer capture and since when it has been unchanged. */
     private unknownLayoutPaneKey;
     private unknownLayoutStableSince;
@@ -693,6 +713,15 @@ export declare class Daemon extends EventEmitter {
      * notice is scrollback, not a reason to pause and send /quit.
      */
     private isCodexLivePaneSnapshot;
+    /** The Codex "usage limit reached" pause pattern (the only one the reserve proof applies to). */
+    private static isCodexUsageLimitPause;
+    private static countPatternMatches;
+    /**
+     * Record that the Luna Reserve menu is (or was just) on screen, for the
+     * usage-limit decision in evaluateErrorPatterns. Called wherever a dialog is
+     * matched, before any key is sent.
+     */
+    private noteCodexReserveDialog;
     /** Re-check the current pane after an async quota probe before pausing it. */
     isCodexLivePane(): Promise<boolean>;
     /** Whether this instance is in a crash loop (3+ consecutive crashes). */
@@ -893,6 +922,14 @@ export declare class Daemon extends EventEmitter {
      * the pane monitor to capture it would re-create a false idle state.
      */
     /**
+     * The CLI was replaced under a running instance by one that cannot run it as
+     * configured — kiro-cli auto-updating to a release without the instance's
+     * UI/engine (#1109). Every respawn would refuse the same way, so stop
+     * supervising and say why instead of failing every 30 seconds.
+     */
+    private stopOnUnsupportedCliRespawn;
+    /**
+     * Announce that this instance is no longer being supervised.  /**
      * Announce that this instance is no longer being supervised.
      *
      * Four health-check exits set `healthCheckPaused = true` and returned without
@@ -1040,6 +1077,14 @@ export declare class Daemon extends EventEmitter {
     pause(reason?: PauseReason): Promise<void>;
     /** Respawn the CLI in the preserved window and block until its prompt is ready. */
     wake(timeoutMs?: number): Promise<void>;
+    /**
+     * One line per edge, until the instance is flapping. The edges themselves are
+     * still published to every observer; only the log is limited, so the next
+     * flap cannot bury the fleet log the way #1116 did. The first suppressed edge
+     * warns once (that is the signal worth reading), and a summary follows every
+     * window while it lasts and at the first edge after it calms.
+     */
+    private logStateEdge;
     private applyInstanceStateSnapshot;
     /**
      * An inbound message is in the CLI — the turn it starts owns the proxy-reply

package/dist/daemon.js

@@ -1,6 +1,7 @@
 import { join, dirname, basename, resolve } from "node:path";
 import { mkdirSync, writeFileSync, readFileSync, readdirSync, existsSync, unlinkSync, rmSync, appendFileSync, statSync, renameSync } from "node:fs";
 import { fileURLToPath } from "node:url";
+import { ensureInstanceDir } from "./private-dir.js";
 import { createHash, randomBytes, randomUUID } from "node:crypto";
 import { EventEmitter } from "node:events";
 import { rotateLogIfNeeded } from "./logger.js";
@@ -18,7 +19,7 @@ import { ContextGuardian } from "./context-guardian.js";
 import { IpcServer } from "./channel/ipc-bridge.js";
 import { daemonBudgetMs } from "./channel/ipc-timeouts.js";
 import { MessageBus } from "./channel/message-bus.js";
-import { shellQuote } from "./backend/types.js";
+import { shellQuote, UnsupportedCliError } from "./backend/types.js";
 import { getTmuxSession } from "./config.js";
 import { routeToolCall } from "./channel/tool-router.js";
 import { HangDetector } from "./hang-detector.js";
@@ -97,6 +98,13 @@ export const DEFAULT_STATE_SAFETY_SWEEP_MS = 60_000;
 export const PERIODIC_REDRAW_PROBE_MS = 25;
 /** Do not turn a streaming Codex turn into one capture-pane per burst. */
 export const STRUCTURED_IDLE_PROBE_MS = 500;
+/**
+ * An instance publishing more than this many execution-state edges inside the
+ * window is flapping; its edges stop being logged one by one (#1116). One
+ * flapping muse wrote 89,532 of the fleet's 91,000 state-change lines.
+ */
+export const STATE_EDGE_LOG_BURST = 10;
+export const STATE_EDGE_LOG_WINDOW_MS = 60_000;
 /** How long an unrecognised-but-empty composer must stay unchanged before it counts as idle (#978). */
 export const UNKNOWN_LAYOUT_STABLE_MS = 10_000;
 /** A foreground server/port-forward should hand control back or be acknowledged. */
@@ -161,7 +169,7 @@ export function readLastInboundAt(instanceDir, now = Date.now()) {
 }
 /** Atomically persist the last real channel inbound timestamp. */
 export function writeLastInboundAt(instanceDir, timestamp) {
-    mkdirSync(instanceDir, { recursive: true });
+    ensureInstanceDir(instanceDir);
     const target = join(instanceDir, LAST_INBOUND_FILE);
     const temp = `${target}.${process.pid}.tmp`;
     writeFileSync(temp, String(timestamp));
@@ -1020,6 +1028,15 @@ export class Daemon extends EventEmitter {
     /** Generation/signature fence for safety dialogs while their screen is still visible. */
     autoResolvedDialogKey = null;
     autoResolvedDialogGeneration = 0;
+    /**
+     * (#1103) Proof that THIS spawn's Codex went onto Luna Reserve: AgEnD saw (and
+     * answered) the usage-limit menu that codex shows when it switches itself to
+     * the reserve. `count` is how many usage-limit lines the pane held at that
+     * moment; only occurrences up to it are the text of the incident we already
+     * handled. A later, larger count is a NEW hit (the reserve ran out too).
+     * Keyed by spawn generation, so a respawn starts without any proof.
+     */
+    codexReserveAck = null;
     backgroundSessionRecoveryAttempted = false;
     /** Whether the last spawn started a fresh session (not resumed). */
     isNewSession = false;
@@ -1037,6 +1054,10 @@ export class Daemon extends EventEmitter {
     /** Consecutive structural idle captures for a noisy, periodically-redrawing
      * pane.  Two are required before a working -> idle transition. */
     instanceStatePeriodicIdleConfirmations = 0;
+    /** Recent execution-state edge times, for the flap log limiter (#1116). */
+    stateEdgeTimes = [];
+    stateEdgesNotLogged = 0;
+    stateEdgeSummaryAt = 0;
     /** #978: last unrecognised-but-empty-composer capture and since when it has been unchanged. */
     unknownLayoutPaneKey = null;
     unknownLayoutStableSince = 0;
@@ -1275,6 +1296,27 @@ export class Daemon extends EventEmitter {
             return false;
         }
     }
+    /** The Codex "usage limit reached" pause pattern (the only one the reserve proof applies to). */
+    static isCodexUsageLimitPause(ep) {
+        return ep.type === "quota" && ep.action === "pause" && ep.message.startsWith("Codex usage limit reached");
+    }
+    static countPatternMatches(pattern, text) {
+        const flags = pattern.flags.includes("g") ? pattern.flags : pattern.flags + "g";
+        return (text.match(new RegExp(pattern.source, flags)) || []).length;
+    }
+    /**
+     * Record that the Luna Reserve menu is (or was just) on screen, for the
+     * usage-limit decision in evaluateErrorPatterns. Called wherever a dialog is
+     * matched, before any key is sent.
+     */
+    noteCodexReserveDialog(dialog, pane) {
+        if (dialog.autoResolutionKey !== "codex-usage-limit-luna-reserve" || this.backend?.binaryName !== "codex")
+            return;
+        const ep = this.backend.getErrorPatterns?.().find(candidate => Daemon.isCodexUsageLimitPause(candidate));
+        const count = ep ? Daemon.countPatternMatches(ep.pattern, pane) : 0;
+        this.codexReserveAck = { generation: this.spawnGeneration, count };
+        this.logger.info({ backend: "codex", evidence: "reserve-menu-seen", usageLimitLines: count }, "codex usage-limit: the Luna Reserve menu was seen — usage-limit text up to this count is stale");
+    }
     /** Re-check the current pane after an async quota probe before pausing it. */
     async isCodexLivePane() {
         if (this.backend?.binaryName !== "codex" || !this.tmux)
@@ -1575,7 +1617,7 @@ export class Daemon extends EventEmitter {
         }
     }
     async start() {
-        mkdirSync(this.instanceDir, { recursive: true });
+        ensureInstanceDir(this.instanceDir);
         writeFileSync(join(this.instanceDir, "daemon.pid"), String(process.pid));
         this.logger.info(`Starting ${this.name}`);
         // P1: Read crash state from previous run — skip resume if last run was a crash loop
@@ -2519,6 +2561,10 @@ export class Daemon extends EventEmitter {
                             this.stormWindow?.markRecovered(this.name);
                     }
                     catch (err) {
+                        if (this.stopOnUnsupportedCliRespawn(err)) {
+                            this.healthCheckTimer = null;
+                            return;
+                        }
                         if (!this.handOffBackendUnreachableRespawn(err)) {
                             this.logger.error({ err }, `Failed to respawn ${cliLabel} window`);
                         }
@@ -2552,6 +2598,21 @@ export class Daemon extends EventEmitter {
      * the pane monitor to capture it would re-create a false idle state.
      */
     /**
+     * The CLI was replaced under a running instance by one that cannot run it as
+     * configured — kiro-cli auto-updating to a release without the instance's
+     * UI/engine (#1109). Every respawn would refuse the same way, so stop
+     * supervising and say why instead of failing every 30 seconds.
+     */
+    stopOnUnsupportedCliRespawn(err) {
+        if (!(err instanceof UnsupportedCliError))
+            return false;
+        this.healthCheckPaused = true;
+        this.logger.error({ reason: err.message }, "Not respawning: the installed CLI cannot run this instance as configured");
+        this.emitSupervisionEnded(err.message, "Install a CLI version that supports this instance's configuration, then start it again.");
+        return true;
+    }
+    /**
+     * Announce that this instance is no longer being supervised.  /**
      * Announce that this instance is no longer being supervised.
      *
      * Four health-check exits set `healthCheckPaused = true` and returned without
@@ -2665,6 +2726,7 @@ export class Daemon extends EventEmitter {
                 for (const dialog of dialogs) {
                     if (!Daemon.dialogMatches(dialog, pane))
                         continue;
+                    this.noteCodexReserveDialog(dialog, pane);
                     if (dialog.blocksDelivery || dialog.holdOnly)
                         blockingSeen = dialog;
                     if (dialog.holdOnly)
@@ -2852,15 +2914,29 @@ export class Daemon extends EventEmitter {
             const count = countMatches(ep.pattern);
             const seen = this.lastErrorCount.get(key) ?? 0;
             // The generic Codex usage-limit line remains in scrollback after the
-            // user selects Luna Reserve. A live composer + Context footer is positive
-            // evidence that the pane recovered; baseline the stale occurrence so it
-            // cannot reach the destructive quota pause path.
-            if (codexLivePane && ep.type === "quota" && ep.action === "pause"
-                && ep.message.startsWith("Codex usage limit reached")) {
-                if (count > 0)
+            // user selects Luna Reserve. What proves that is NOT a live composer —
+            // codex paints one after a plain hit too, and the old live-composer guard
+            // therefore swallowed every real hit (#1103) — but AgEnD having seen the
+            // Luna Reserve menu in this spawn. Occurrences up to that point are the
+            // incident we already handled; anything beyond is a new hit and goes on to
+            // the lifecycle, whose usage probe decides (available / reserve / exhausted).
+            if (Daemon.isCodexUsageLimitPause(ep)) {
+                const ack = this.codexReserveAck;
+                if (ack && ack.generation === this.spawnGeneration && count <= ack.count) {
+                    // The handled incident's lines can scroll out of the capture buffer as
+                    // the reserve session talks on. Lower the proof (and the occurrence
+                    // baseline) with them: otherwise `count <= ack.count` would keep
+                    // swallowing a NEW hit — the reserve running out — until it exceeded
+                    // the menu-time count, and it would never reach the lifecycle probe.
+                    if (count < ack.count)
+                        ack.count = count;
                     this.lastErrorCount.set(key, count);
-                this.logger.debug("Codex usage-limit text is stale — live reserve pane is running");
-                continue;
+                    this.logger.debug({ decision: "ignore", evidence: "reserve-menu-seen", count, acknowledged: ack.count }, "codex usage-limit: stale text of the handled Luna Reserve incident");
+                    continue;
+                }
+                if (count > seen) {
+                    this.logger.info({ decision: "candidate", evidence: ack && ack.generation === this.spawnGeneration ? "new-hit-after-reserve" : "no-reserve-proof", count, livePane: codexLivePane }, "codex usage-limit: new occurrence — handing to the quota probe");
+                }
             }
             if (count <= seen) {
                 // Occurrences scrolled out of the capture buffer → lower the baseline
@@ -3632,6 +3708,39 @@ export class Daemon extends EventEmitter {
                 this.pauseWakeTransition = null;
         }
     }
+    /**
+     * One line per edge, until the instance is flapping. The edges themselves are
+     * still published to every observer; only the log is limited, so the next
+     * flap cannot bury the fleet log the way #1116 did. The first suppressed edge
+     * warns once (that is the signal worth reading), and a summary follows every
+     * window while it lasts and at the first edge after it calms.
+     */
+    logStateEdge(previous, snapshot) {
+        const now = Date.now();
+        this.stateEdgeTimes.push(now);
+        while (this.stateEdgeTimes.length && this.stateEdgeTimes[0] <= now - STATE_EDGE_LOG_WINDOW_MS)
+            this.stateEdgeTimes.shift();
+        const edges = this.stateEdgeTimes.length;
+        const fields = { previousState: previous, state: snapshot.state, unchangedForMs: snapshot.unchangedForMs };
+        if (edges <= STATE_EDGE_LOG_BURST) {
+            if (this.stateEdgesNotLogged > 0) {
+                this.logger.info({ notLogged: this.stateEdgesNotLogged }, "Instance execution state edges were not logged while it was flapping");
+                this.stateEdgesNotLogged = 0;
+            }
+            this.logger.info(fields, "Instance execution state changed");
+            return;
+        }
+        if (this.stateEdgesNotLogged === 0) {
+            this.stateEdgeSummaryAt = now;
+            this.logger.warn({ edgesInWindow: edges, windowMs: STATE_EDGE_LOG_WINDOW_MS, backend: this.backend?.binaryName }, "Instance execution state is flapping — per-edge logging suppressed; the pane is repainting without a stable idle/working reading");
+        }
+        else if (now - this.stateEdgeSummaryAt >= STATE_EDGE_LOG_WINDOW_MS) {
+            this.stateEdgeSummaryAt = now;
+            this.logger.warn({ notLogged: this.stateEdgesNotLogged, edgesInWindow: edges }, "Instance execution state is still flapping");
+        }
+        this.stateEdgesNotLogged++;
+        this.logger.debug(fields, "Instance execution state changed (flapping, not logged at info)");
+    }
     applyInstanceStateSnapshot(snapshot, pane) {
         const previous = this.instanceState;
         // A live safety menu owns stdin.  It may still contain Claude's persistent
@@ -3667,11 +3776,7 @@ export class Daemon extends EventEmitter {
                 this.maybeProxyReplyOnTurnEnd(pane);
         }
         if (snapshot.state !== previous) {
-            this.logger.info({
-                previousState: previous,
-                state: snapshot.state,
-                unchangedForMs: snapshot.unchangedForMs,
-            }, "Instance execution state changed");
+            this.logStateEdge(previous, snapshot);
             this.emit("instance_state", { name: this.name, ...snapshot });
             this.ipcServer?.broadcast({ type: "instance_state", instanceName: this.name, ...snapshot });
             if (snapshot.state === "stuck" && pane && this.instanceStateReadyPattern) {
@@ -4169,6 +4274,19 @@ export class Daemon extends EventEmitter {
                 if (reason === "delivery_idle_gate")
                     this.resetFooterFallback();
                 if (reason === "output_probe") {
+                    // A TUI that repaints without changing a cell (muse parks its cursor
+                    // ~18 times a second while idle; Codex's star field) never leaves the
+                    // probe a quiet moment, so roughly one probe in five is overtaken. An
+                    // idle instance whose stale capture still shows the structural idle
+                    // layout has nothing to be called working on: the rule that cosmetic
+                    // output must not manufacture an idle -> working edge holds here too.
+                    // Without this every overtaken probe published the edge, and the next
+                    // clean probe (one confirmation was still banked) published it back —
+                    // a pair every ~1.6s for as long as the repaint lasted (#1116). Real
+                    // work cannot hide behind this: it changes the layout, the next
+                    // output arms another probe, and the debounce capture still fires.
+                    if (this.instanceState === "idle" && this.backend?.isPeriodicRedrawIdlePane?.(pane) === true)
+                        return;
                     // A second burst overtook the probe. Do not bless its stale capture as
                     // idle: continuous output is the safer working signal, while the next
                     // quiet probe/debounce can still prove that it was only repainting.
@@ -5533,10 +5651,18 @@ export class Daemon extends EventEmitter {
             return { state: "unknown" };
         }
         for (const dialog of dialogs) {
-            if (Daemon.dialogMatches(dialog, pane)) {
-                this.trackDialogParked(dialog);
-                return { state: "dialog", dialog };
-            }
+            if (!Daemon.dialogMatches(dialog, pane))
+                continue;
+            // Same one-shot fence as the scanners: a safety choice already sent in
+            // this spawn is not what the screen is waiting on any more — the entry
+            // after it (typically a hold) describes it.
+            if (dialog.verifyAfterKeys && dialog.autoResolutionKey
+                && this.autoResolvedDialogGeneration === this.spawnGeneration
+                && this.autoResolvedDialogKey === dialog.autoResolutionKey
+                && dialogs.some(other => other !== dialog && Daemon.dialogMatches(other, pane)))
+                continue;
+            this.trackDialogParked(dialog);
+            return { state: "dialog", dialog };
         }
         this.trackDialogParked(null);
         return { state: "clear" };
@@ -8245,7 +8371,7 @@ export class Daemon extends EventEmitter {
                 // Transient tmux trouble is not evidence about the CLI. Returning false
                 // here would clear the session; retry within the budget instead.
                 captureFailures++;
-                this.logger.warn({ err, captureFailures }, "capture-pane failed during the startup dialog scan — retrying");
+                this.logger.debug({ err, captureFailures }, "capture-pane failed during the startup dialog scan — retrying");
                 // Only a POSITIVE dead pane ends the scan; an unanswerable liveness
                 // query is the same transient trouble and must not clear the session.
                 if ((await this.paneLiveness()) === "dead")
@@ -8277,6 +8403,7 @@ export class Daemon extends EventEmitter {
                                     && this.autoResolvedDialogKey === dialog.autoResolutionKey)))
                             continue;
                         lastDialog = dialog;
+                        this.noteCodexReserveDialog(dialog, pane);
                         cleanReadyPolls = 0;
                         // Start the parked clock for every delivery-blocking dialog, exact
                         // ones included: if the auto-dismiss keeps failing here, the human