@songsid/agend 2.3.0-beta.9 → 2.3.0

package/dist/instance-lifecycle.js

@@ -7,7 +7,7 @@ import { execFileSync } from "node:child_process";
 import { join, basename, dirname, resolve, sep as pathSep } from "node:path";
 import { access } from "node:fs/promises";
 import { getAgendHome, ensureWorkspaceGit } from "./paths.js";
-import { DEFAULT_INSTANCE_CONFIG, getTmuxSession } from "./config.js";
+import { DEFAULT_INSTANCE_CONFIG, getEffectiveInstanceDefaults, getTmuxSession } from "./config.js";
 import { readStatuslineModel, uniqueInstanceName } from "./topic-commands.js";
 import { isModelCompatible } from "./backend/types.js";
 import { safeHandler } from "./safe-async.js";
@@ -290,6 +290,9 @@ export class InstanceLifecycle {
      * Stored on the lifecycle (not the daemon) so the count survives daemon restarts.
      */
     capacityBackoffAttempts = new Map();
+    /** #2282: transient429 recovery has its own budget; capacity errors cannot spend it. */
+    rateLimitNudgeAttempts = new Map();
+    static RATE_LIMIT_NUDGE_DELAYS_MS = [30_000, 60_000, 120_000];
     /** Active backoff restart timers, keyed by instance name. One per instance max. */
     capacityBackoffTimers = new Map();
     /** Old Codex capacity rows to baseline in the replacement daemon (#949). */
@@ -696,8 +699,10 @@ export class InstanceLifecycle {
             const incident = this.mcpIncident(name);
             const generation = ++incident.gen;
             incident.deathNoticeSent = false;
-            this.ctx.eventLog?.insert(name, "mcp_died", { pid: data.pid });
-            this.ctx.logger.error({ name, pid: data.pid }, "MCP server died — instance cannot use agend tools");
+            this.ctx.eventLog?.insert(name, "mcp_died", { pid: data.pid, ...(data.neverConnected ? { neverConnected: true } : {}) });
+            this.ctx.logger.error({ name, pid: data.pid, neverConnected: data.neverConnected === true }, data.neverConnected
+                ? "MCP server never connected — instance cannot use agend tools"
+                : "MCP server died — instance cannot use agend tools");
             this.ctx.webhookEmit("mcp_died", name, { pid: data.pid });
             // Auth outranks MCP: when the CLI lost authentication, "MCP server died"
             // is a symptom and the fix is /login, not a restart. The daemon flags a
@@ -804,13 +809,16 @@ export class InstanceLifecycle {
             // The daemon object dies with the restart it asks for, so the loop guard
             // lives here: if the previous auto-restart was under the cooldown, the new
             // MCP server evidently died right back (broken install, OOM pressure) and
-            // another attempt would just cycle the instance. Leave it to the operator —
-            // the mcp_died notification for this death has already gone out.
+            // another immediate attempt would just cycle the instance. Keep a bounded
+            // reconsideration on the daemon: a one-shot suppression strands this pid
+            // forever because later health ticks dedupe the already reported death.
             const last = this.mcpAutoRestartAt.get(name) ?? 0;
             const now = Date.now();
             if (now - last < InstanceLifecycle.MCP_AUTO_RESTART_COOLDOWN_MS) {
-                this.ctx.eventLog?.insert(name, "mcp_auto_restart_suppressed", { trigger: data.trigger });
-                this.ctx.logger.error({ name, trigger: data.trigger }, "MCP auto-restart suppressed — the previous attempt is still inside the cooldown, so the restart is not fixing it");
+                const retryAfterMs = InstanceLifecycle.MCP_AUTO_RESTART_COOLDOWN_MS - (now - last);
+                this.ctx.eventLog?.insert(name, "mcp_auto_restart_suppressed", { trigger: data.trigger, retryAfterMs });
+                this.ctx.logger.warn({ name, trigger: data.trigger, retryAfterMs }, "MCP auto-restart deferred — will reconsider after the cooldown with liveness and idle checks");
+                daemon.deferMcpRestart?.(retryAfterMs);
                 return;
             }
             this.mcpAutoRestartAt.set(name, now);
@@ -1005,7 +1013,7 @@ export class InstanceLifecycle {
                 this.ctx.restartSingleInstance(name, { freshStart: true }).catch(err => this.ctx.logger.error({ err, name }, "pty_error restart failed"));
             }
             else if (data.action === "nudge_continue") {
-                this.scheduleContinueNudge(name, daemon, data.pattern);
+                this.scheduleContinueNudge(name, daemon, data.pattern, data.type === "rate_limit");
             }
             else if (data.action === "backoff_restart") {
                 // Transient model capacity error (#905). The CLI already returned to its
@@ -1121,34 +1129,84 @@ export class InstanceLifecycle {
      * going after a minute — at most CAPACITY_BACKOFF_MAX times per 30 minutes, then pause like the old ladder did.
      * The daemon owns the nudge itself (its fences, its pane lock); this owns only whether one is allowed.
      */
-    scheduleContinueNudge(name, daemon, pattern) {
+    scheduleContinueNudge(name, daemon, pattern, rateLimit = false) {
+        // A retired daemon must never pause its replacement after exhausting an
+        // old budget. Arming itself also checks the captured spawn and pane.
+        if (rateLimit && this.daemons.get(name) !== daemon)
+            return;
+        const budget = rateLimit ? this.rateLimitNudgeAttempts : this.capacityBackoffAttempts;
         const now = Date.now();
-        const prev = this.capacityBackoffAttempts.get(name);
+        const prev = budget.get(name);
         // A wall clock that stepped back must not make an old window look current forever.
         const elapsed = prev ? now - prev.lastAt : -1;
         const attempts = prev && elapsed >= 0 && elapsed < InstanceLifecycle.CAPACITY_BACKOFF_WINDOW_MS ? prev.count : 0;
         const notificationTarget = this.ptyErrorNotificationTarget(name);
+        const delayMs = rateLimit
+            ? InstanceLifecycle.RATE_LIMIT_NUDGE_DELAYS_MS[Math.min(attempts, InstanceLifecycle.CAPACITY_BACKOFF_MAX - 1)]
+            : InstanceLifecycle.CAPACITY_NUDGE_DELAY_MS;
+        // Validate the fresh/live episode BEFORE a budget-exhaustion pause too.
+        // Arming only records a due time; on the fourth incident it is cancelled
+        // synchronously below, before any monitor tick can send a fourth nudge.
+        if (rateLimit) {
+            if (!(pattern instanceof RegExp) || !daemon.armCapacityNudge?.(pattern, delayMs)) {
+                if (notificationTarget)
+                    this.notifyIncident(notificationTarget, "pty_error", t("inst.codex_rate_retry_stopped"));
+                return;
+            }
+            // Preserve next-spawn failover bookkeeping only for a vouched fresh
+            // failure. It does not itself recover this already-failed turn.
+            this.ctx.checkModelFailover(name, 100);
+        }
         if (attempts >= InstanceLifecycle.CAPACITY_BACKOFF_MAX) {
+            if (rateLimit) {
+                void this.pauseAfterRateLimitNudges(name, daemon, attempts)
+                    .catch(err => this.ctx.logger.warn({ err, name }, "rate-limit-exhausted pause failed"));
+                return;
+            }
             this.ctx.logger.warn({ name, attempts }, "Model still at capacity after max nudges — pausing");
             daemon.cancelCapacityNudge?.("retries exhausted");
             if (notificationTarget)
-                this.notifyIncident(notificationTarget, "pty_error", t("inst.codex_capacity_backoff_exhausted", name, String(attempts)));
+                this.notifyIncident(notificationTarget, "pty_error", t(rateLimit ? "inst.codex_rate_retry_exhausted" : "inst.codex_capacity_backoff_exhausted", name, String(attempts)));
             void this.pause(name, "error")
                 .catch(err => this.ctx.logger.warn({ err, name }, "capacity-exhausted pause failed"))
-                .finally(() => { if (!this.isPaused(name))
-                this.daemons.get(name)?.requestPauseWhenIdle({ reason: "error" }); });
+                .finally(() => {
+                if (!this.isPaused(name) && (!rateLimit || this.daemons.get(name) === daemon)) {
+                    (rateLimit ? daemon : this.daemons.get(name))?.requestPauseWhenIdle({ reason: "error" });
+                }
+            });
             return;
         }
-        if (!(pattern instanceof RegExp) || !daemon.armCapacityNudge?.(pattern, InstanceLifecycle.CAPACITY_NUDGE_DELAY_MS)) {
+        if (!rateLimit && (!(pattern instanceof RegExp) || !daemon.armCapacityNudge?.(pattern, delayMs))) {
             // Nothing to arm (the screen already moved on): the user still hears about the capacity error, no attempt is used.
             if (notificationTarget)
-                this.notifyIncident(notificationTarget, "pty_error", t("inst.codex_model_capacity"));
+                this.notifyIncident(notificationTarget, "pty_error", t(rateLimit ? "inst.codex_rate_retry_stopped" : "inst.codex_model_capacity"));
             return;
         }
-        this.capacityBackoffAttempts.set(name, { count: attempts + 1, lastAt: now });
+        budget.set(name, { count: attempts + 1, lastAt: now });
         this.ctx.logger.info({ name, attempt: attempts + 1 }, "Model at capacity — will tell it to keep going");
         if (notificationTarget) {
-            this.notifyIncident(notificationTarget, "pty_error", t("inst.codex_capacity_nudge_scheduled", name, String(attempts + 1), String(InstanceLifecycle.CAPACITY_BACKOFF_MAX), String(Math.round(InstanceLifecycle.CAPACITY_NUDGE_DELAY_MS / 1000))));
+            this.notifyIncident(notificationTarget, "pty_error", t(rateLimit ? "inst.codex_rate_retry_scheduled" : "inst.codex_capacity_nudge_scheduled", name, String(attempts + 1), String(InstanceLifecycle.CAPACITY_BACKOFF_MAX), String(Math.round(delayMs / 1000))));
+        }
+    }
+    /** An exhausted error must not pause a draft, new delivery or replacement launch. */
+    async pauseAfterRateLimitNudges(name, daemon, attempts) {
+        const admission = await daemon.prepareCapacityNudgePause?.();
+        if (!admission || this.daemons.get(name) !== daemon || !admission.current())
+            return;
+        this.ctx.logger.warn({ name, attempts }, "HTTP 429 recovery exhausted — pausing the unchanged failed turn");
+        const target = this.ptyErrorNotificationTarget(name);
+        if (target)
+            this.notifyIncident(target, "pty_error", t("inst.codex_rate_retry_exhausted", name, String(attempts)));
+        try {
+            await this.pause(name, "error", admission);
+        }
+        finally {
+            // pause advances its own monitor fence, so use the captured launch and
+            // episode identity plus a fresh pane read, rather than that fence alone.
+            if (this.daemons.get(name) === daemon && !this.isPaused(name) && await admission.revalidate()
+                && this.daemons.get(name) === daemon && !this.isPaused(name) && admission.current()) {
+                daemon.requestPauseWhenIdle({ reason: "error" });
+            }
         }
     }
     epochOf(name) {
@@ -1382,7 +1440,7 @@ export class InstanceLifecycle {
     getLastPausedAt(name) {
         return this.daemons.get(name)?.lastPausedAt ?? readPausedAt(this.ctx.getInstanceDir(name));
     }
-    async pause(name, reason = "operator") {
+    async pause(name, reason = "operator", admission) {
         // Final backstop shared by slash, MCP, Settings, warm-cap, and any future
         // caller. General is the coordinator and must remain resident to route work.
         if (isGeneralInstance(this.ctx.fleetConfig, name)) {
@@ -1407,7 +1465,10 @@ export class InstanceLifecycle {
         }
         this.ctx.stopStatuslineWatcher(name);
         try {
-            await daemon.pause(reason);
+            if (admission)
+                await daemon.pause(reason, admission);
+            else
+                await daemon.pause(reason);
         }
         finally {
             // A rejected/no-op pause leaves the instance active and must not strand
@@ -1686,7 +1747,7 @@ export class InstanceLifecycle {
             }
             else {
                 try {
-                    const { execFile: execFileCb } = await import("node:child_process");
+                    const { execFile: execFileCb } = await import("./child-process.js");
                     const { promisify } = await import("node:util");
                     check();
                     const execFileAsync = promisify(execFileCb);
@@ -1715,7 +1776,7 @@ export class InstanceLifecycle {
             }
             // Prune stale worktree records (e.g. if directory was manually deleted)
             try {
-                const { execFile: execFileCb } = await import("node:child_process");
+                const { execFile: execFileCb } = await import("./child-process.js");
                 const { promisify } = await import("node:util");
                 check();
                 const execFileAsync = promisify(execFileCb);
@@ -1845,7 +1906,7 @@ export class InstanceLifecycle {
                 }
             };
             if (acquiredWorktree) {
-                const { execFile } = await import("node:child_process");
+                const { execFile } = await import("./child-process.js");
                 const { promisify } = await import("node:util");
                 if (!ownsPath())
                     return;
@@ -1863,7 +1924,7 @@ export class InstanceLifecycle {
         }
         if (branch) {
             try {
-                const { execFile: execFileCb } = await import("node:child_process");
+                const { execFile: execFileCb } = await import("./child-process.js");
                 const { promisify } = await import("node:util");
                 const nativeExec = promisify(execFileCb);
                 const execFileAsync = async (file, args, options, acquired) => {
@@ -1991,14 +2052,14 @@ export class InstanceLifecycle {
             if (!directory) {
                 workDir = join(getAgendHome(), "workspaces", newInstanceName);
                 const existed = existsSync(workDir);
-                mkdirSync(workDir, { recursive: true });
+                mkdirSync(workDir, { recursive: true, mode: 0o700 });
                 if (!existed)
                     acquiredDirectory = statSync(workDir);
                 ensureWorkspaceGit(workDir);
             }
             const instanceConfig = {
                 ...DEFAULT_INSTANCE_CONFIG,
-                ...this.ctx.fleetConfig.defaults,
+                ...getEffectiveInstanceDefaults(this.ctx.fleetConfig.defaults ?? {}),
                 working_directory: workDir,
                 topic_id: createdTopicId,
                 ...(description ? { description } : {}),