@songsid/agend 2.1.8 → 2.1.9

package/dist/instance-lifecycle.js

@@ -9,7 +9,7 @@ import { isModelCompatible } from "./backend/types.js";
 import { safeHandler } from "./safe-async.js";
 import { t } from "./locale.js";
 import { EMBEDDED_CREDENTIAL_REMOTE_REPAIR_FAILED, EMBEDDED_CREDENTIAL_REMOTE_WARNING, secureGitHubRemotesForWorktree, } from "./remote-credential-check.js";
-import { clearPausedMarker, hasPausedMarker, readPausedAt, writePausedMarker } from "./pause-marker.js";
+import { clearPausedMarker, hasPausedMarker, readPausedAt, readPauseReason, writePausedMarker } from "./pause-marker.js";
 import { reportProviderRateLimit } from "./usage/provider-alerts.js";
 import { isFleetStartCommandLine } from "./fleet-lock.js";
 import { GENERAL_PAUSE_ERROR, isGeneralInstance } from "./general-instance.js";
@@ -83,6 +83,19 @@ export function getUnsafeInstanceDaemonPidReason(pid, dataDir) {
     }
     return null;
 }
+/** Thrown by a start whose generation was invalidated (stop/restart) before it could publish. */
+export class SupersededStartError extends Error {
+    constructor(name) {
+        super(`Start of '${name}' was superseded by a stop or restart before it registered`);
+        this.name = "SupersededStartError";
+    }
+}
+/**
+ * How long a target just woken for queued work is held from idle auto-pause
+ * before its first queued delivery is claimed (Phase 2 design §1.6): the
+ * cross-instance idle gate (60s) plus a 60s grace.
+ */
+export const PRECLAIM_LEASE_MS = 120_000;
 /**
  * Working directories of the fleet's other instances on `backendName` (#984):
  * Codex consults them only to decide whether `resume --last` could take a
@@ -194,6 +207,21 @@ export class InstanceLifecycle {
     ctx;
     /** Active daemon processes: instanceName → Daemon */
     daemons = new Map();
+    /**
+     * Phase 2a: one transition (start / stop / wake / restart) at a time per
+     * instance. A call that passes the current transition's handle runs inline;
+     * anything else queues.
+     */
+    transitionTails = new Map();
+    activeTransition = new Map();
+    /** Bumped synchronously by stop/restart: a start begun under an older epoch must not publish. */
+    lifecycleEpochs = new Map();
+    /** Single wake flight per instance: concurrent wakes share it. */
+    wakeFlights = new Map();
+    /** Starts that begin their idle window now instead of at the persisted last inbound. */
+    activitySeedNow = new Set();
+    /** When each instance was last woken or started for work — the pre-claim lease clock. */
+    wokeAt = new Map();
     /** backend → last auth-error alert time, so one expiry sends one alert. */
     lastAuthAlertAt = new Map();
     /**
@@ -792,8 +820,9 @@ export class InstanceLifecycle {
             // Pattern-matched auth errors get a second opinion from the real CLI
             // before any pause/alert: an agent quoting "401 Unauthorized" in prose
             // must not pause the fleet. A working credential ends the incident here.
+            let authVerdict;
             if (data.type === "auth_error") {
-                const verdict = await this.verifyAuthError(name);
+                const verdict = authVerdict = await this.verifyAuthError(name);
                 if (verdict === "valid") {
                     this.ctx.logger.info({ name, backend: this.backendOf(name) }, "auth-error pattern ignored — token-free auth check passed (likely conversation text)");
                     // Dropping the incident is not enough: the daemon armed its auth
@@ -887,10 +916,10 @@ export class InstanceLifecycle {
                     const notificationTarget = this.ptyErrorNotificationTarget(name);
                     if (notificationTarget)
                         this.notifyIncident(notificationTarget, "pty_error", t("inst.codex_capacity_backoff_exhausted", name, String(attempts)));
-                    void this.pause(name)
+                    void this.pause(name, "error")
                         .catch(err => this.ctx.logger.warn({ err, name }, "capacity-exhausted pause failed"))
                         .finally(() => { if (!this.isPaused(name))
-                        this.daemons.get(name)?.requestPauseWhenIdle(); });
+                        this.daemons.get(name)?.requestPauseWhenIdle({ reason: "error" }); });
                 }
                 else {
                     const delays = InstanceLifecycle.CAPACITY_BACKOFF_DELAYS_MS;
@@ -933,14 +962,96 @@ export class InstanceLifecycle {
                 // credit and lost work. pause() alone no-ops while the pane is busy (the
                 // usual state when auth fails), so mark it to pause as soon as it idles.
                 // Queued messages survive: delivery wakes a paused instance.
-                void this.pause(name)
-                    .catch(err => this.ctx.logger.warn({ err, name }, "auth-error pause failed"))
-                    .finally(() => { if (!this.isPaused(name))
-                    daemon.requestPauseWhenIdle(); });
+                if (data.type === "auth_error" && authVerdict === "unknown") {
+                    // Nothing vouched for an auth hit the CLI could not be asked about
+                    // (#1044): pause only if it is still on screen when the turn ends.
+                    // Never the direct pause here — on an already-idle pane it would
+                    // /quit before anything looked, and the check would never run. The
+                    // daemon evaluates at once, so an idle pane is decided now.
+                    daemon.requestPauseWhenIdle({ reconfirmAuth: true });
+                }
+                else {
+                    // The marker must say auth (2b never auto-wakes a login failure); any
+                    // other pause-action error is an error pause, not an operator one.
+                    const reason = data.type === "auth_error" ? "auth" : "error";
+                    void this.pause(name, reason)
+                        .catch(err => this.ctx.logger.warn({ err, name }, "auth-error pause failed"))
+                        .finally(() => { if (!this.isPaused(name))
+                        daemon.requestPauseWhenIdle({ reason }); });
+                }
             }
         }, this.ctx.logger, `daemon.pty_error[${name}]`));
     }
-    async start(name, config, topicMode, runtimeIdentity) {
+    epochOf(name) {
+        return this.lifecycleEpochs.get(name) ?? 0;
+    }
+    /** Invalidate every start/wake already running for `name` (they will not publish). */
+    invalidate(name) {
+        this.lifecycleEpochs.set(name, this.epochOf(name) + 1);
+    }
+    /** Whether `transition` is `name`'s transition and is the one running now. */
+    holds(name, transition) {
+        return transition !== undefined && transition.name === name && this.activeTransition.get(name) === transition.token;
+    }
+    /** Run `fn` as `name`'s next transition, or inline when `parent` is the transition running now. */
+    async runTransition(name, fn, parent) {
+        if (this.holds(name, parent))
+            return fn(parent);
+        const previous = this.transitionTails.get(name) ?? Promise.resolve();
+        const handle = { name, token: Symbol(name) };
+        const run = previous.then(() => {
+            this.activeTransition.set(name, handle.token);
+            return fn(handle);
+        }).finally(() => {
+            if (this.activeTransition.get(name) === handle.token)
+                this.activeTransition.delete(name);
+        });
+        const tail = run.then(() => { }, () => { });
+        this.transitionTails.set(name, tail);
+        try {
+            return await run;
+        }
+        finally {
+            if (this.transitionTails.get(name) === tail)
+                this.transitionTails.delete(name);
+        }
+    }
+    /** The next start of `name` is a wake/restart/explicit start: begin its idle window now. */
+    markActivitySeedNow(name) {
+        this.activitySeedNow.add(name);
+    }
+    /** Drop a seed request that no start consumed (the start returned early or failed before it). */
+    clearActivitySeedNow(name) {
+        this.activitySeedNow.delete(name);
+    }
+    /** Record that `name` was just woken or (re)started, opening its pre-claim lease window. */
+    noteWokeForWork(name, now = Date.now()) {
+        this.wokeAt.set(name, now);
+    }
+    /**
+     * Phase 2a work lease (design §1.6). Held while a durable submission to the
+     * target is in flight (unbounded: no timer releases a writer), or for
+     * PRECLAIM_LEASE_MS after a wake/start while queued work is still waiting to
+     * be claimed. It holds only the idle-timeout pause and warm-cap eviction —
+     * never an auth-deferred or operator pause.
+     */
+    hasWorkLease(name, now = Date.now()) {
+        const outbox = this.ctx.deliveryOutbox;
+        if (!outbox)
+            return false;
+        if (outbox.countForTarget(name, ["delivering", "submission_started"]) > 0)
+            return true;
+        const woke = this.wokeAt.get(name);
+        if (woke === undefined || now - woke >= PRECLAIM_LEASE_MS)
+            return false;
+        return outbox.countForTarget(name, ["queued", "retry_wait"]) > 0;
+    }
+    async start(name, config, topicMode, runtimeIdentity, transition) {
+        return this.runTransition(name, () => this.startInTransition(name, config, topicMode, runtimeIdentity), transition);
+    }
+    async startInTransition(name, config, topicMode, runtimeIdentity) {
+        const epoch = this.epochOf(name);
+        const seedNow = this.activitySeedNow.delete(name);
         if (this.daemons.has(name)) {
             this.ctx.logger.info({ name }, "Instance already running, skipping");
             return;
@@ -979,6 +1090,9 @@ export class InstanceLifecycle {
             model: config.model ?? "default",
         }, this.ctx.spawnGate, this.ctx.stormWindow, this.ctx.backendOutage);
         daemon.setDeliveryOutboxPort(this.ctx.deliveryOutbox ?? undefined);
+        daemon.setWorkLeaseCheck(() => this.hasWorkLease(name));
+        if (seedNow)
+            daemon.seedActivityNow();
         // Read at each spawn so a fleet.yaml reload is honoured (#984).
         daemon.setPeerWorkingDirectories(() => peerWorkingDirectories(this.ctx.fleetConfig, name, backendName));
         daemon.setStatusEmojiAvoidList(() => this.ctx.statusEmojiAvoidList?.(name));
@@ -1026,12 +1140,30 @@ export class InstanceLifecycle {
                 throw new Error(message);
             }
         }
+        if (this.epochOf(name) !== epoch) {
+            // A stop/restart arrived while this start was reconciling: do not spawn.
+            await daemon.abortStartup().catch(() => { });
+            throw new SupersededStartError(name);
+        }
+        // The startup scan can raise an incident (sign-in screen, fatal modal)
+        // before the handlers below exist and before this daemon is registered;
+        // hold it and deliver it once both are true.
+        daemon.holdStartupIncidents?.();
         await InstanceLifecycle.startOrDispose(daemon, name, this.ctx.logger);
+        // Publication fence (Phase 2a): the spawn cannot be cancelled, so a stop or
+        // restart requested meanwhile has been waiting behind this transition. A
+        // superseded start must not register, signal ready, or keep its window.
+        if (this.epochOf(name) !== epoch) {
+            await daemon.abortStartup().catch(err => this.ctx.logger.warn({ err, name }, "Failed to dispose a superseded start"));
+            throw new SupersededStartError(name);
+        }
         this.capacityBackoffBaselines.delete(name);
         this.daemons.set(name, daemon);
+        if (seedNow)
+            this.noteWokeForWork(name);
         this.ctx.onDaemonReady?.(name, daemon.bootId);
-        daemon.on("auto_pause_requested", safeHandler(async () => {
-            await this.pause(name);
+        daemon.on("auto_pause_requested", safeHandler(async (data) => {
+            await this.pause(name, data?.reason ?? "idle");
         }, this.ctx.logger, `autoPause[${name}]`));
         daemon.on("auto_paused", (data) => {
             this.ctx.eventLog?.insert(name, "instance_paused", { reason: "idle", paused_at: data.pausedAt });
@@ -1057,6 +1189,9 @@ export class InstanceLifecycle {
         this.attachDeliveryStatusHandlers(name, daemon);
         this.ctx.setTopicIcon(name, "green");
         this.ctx.touchActivity(name);
+        // Last: handlers attached and daemon registered, so a held startup incident
+        // (pause, notify) finds both.
+        daemon.releaseStartupIncidents?.();
     }
     isPaused(name) {
         return this.daemons.get(name)?.isPaused ?? hasPausedMarker(this.ctx.getInstanceDir(name));
@@ -1064,7 +1199,7 @@ export class InstanceLifecycle {
     getLastPausedAt(name) {
         return this.daemons.get(name)?.lastPausedAt ?? readPausedAt(this.ctx.getInstanceDir(name));
     }
-    async pause(name) {
+    async pause(name, reason = "operator") {
         // Final backstop shared by slash, MCP, Settings, warm-cap, and any future
         // caller. General is the coordinator and must remain resident to route work.
         if (isGeneralInstance(this.ctx.fleetConfig, name)) {
@@ -1089,7 +1224,7 @@ export class InstanceLifecycle {
         }
         this.ctx.stopStatuslineWatcher(name);
         try {
-            await daemon.pause();
+            await daemon.pause(reason);
         }
         finally {
             // A rejected/no-op pause leaves the instance active and must not strand
@@ -1098,25 +1233,65 @@ export class InstanceLifecycle {
                 this.ctx.startStatuslineWatcher(name);
         }
     }
-    async wake(name, timeoutMs = 30_000) {
+    /** When `name` was last woken or (re)started for work, if recorded. */
+    wokeAtFor(name) {
+        return this.wokeAt.get(name);
+    }
+    async wake(name, timeoutMs = 30_000, transition, opts = {}) {
+        const woke = await this.wakeInner(name, timeoutMs, transition);
+        // Only a successful operator/user wake lifts the coordinator's park.
+        if (opts.source !== "coordinator")
+            this.ctx.onExternalWake?.(name);
+        return woke;
+    }
+    async wakeInner(name, timeoutMs, transition) {
+        if (this.holds(name, transition))
+            return this.wakeInTransition(name, timeoutMs, transition);
+        // An explicit wake that meets a restart joins it: the restart of a paused
+        // instance is itself a wake (Phase 2a), and a second flight would race it.
+        const restarting = this.ctx.restartInFlight?.(name);
+        if (restarting) {
+            await restarting;
+            return;
+        }
+        const inFlight = this.wakeFlights.get(name);
+        if (inFlight)
+            return inFlight;
+        const flight = this.runTransition(name, handle => this.wakeInTransition(name, timeoutMs, handle))
+            .finally(() => { if (this.wakeFlights.get(name) === flight)
+            this.wakeFlights.delete(name); });
+        this.wakeFlights.set(name, flight);
+        return flight;
+    }
+    async wakeInTransition(name, timeoutMs, transition) {
+        const epoch = this.epochOf(name);
         const daemon = this.daemons.get(name);
         if (daemon) {
             await daemon.wake(timeoutMs);
+            // A stop/restart requested while the resident CLI was respawning is
+            // queued behind this wake and will tear it down next: do not open a
+            // pre-claim window or restart its watchers on the way out.
+            if (this.epochOf(name) !== epoch)
+                return;
         }
         else {
             const instanceDir = this.ctx.getInstanceDir(name);
             if (!hasPausedMarker(instanceDir))
                 throw new Error(`Cannot wake stopped instance '${name}'`);
             const pausedAt = readPausedAt(instanceDir) ?? Date.now();
+            const reason = readPauseReason(instanceDir);
             clearPausedMarker(instanceDir);
+            this.markActivitySeedNow(name);
             try {
-                await this.ctx.startPersistedPausedInstance(name);
+                await this.ctx.startPersistedPausedInstance(name, transition);
             }
             catch (err) {
-                writePausedMarker(instanceDir, pausedAt);
+                this.clearActivitySeedNow(name);
+                writePausedMarker(instanceDir, pausedAt, reason);
                 throw err;
             }
         }
+        this.noteWokeForWork(name);
         this.ctx.startStatuslineWatcher(name);
     }
     /**
@@ -1141,7 +1316,16 @@ export class InstanceLifecycle {
         await this.stop(name);
         return true;
     }
-    async stop(name) {
+    async stop(name, transition) {
+        // Synchronously, before queueing: a start still spawning must see this and
+        // not publish when it finishes (Phase 2a publication fence).
+        if (!this.holds(name, transition))
+            this.invalidate(name);
+        return this.runTransition(name, () => this.stopInTransition(name), transition);
+    }
+    async stopInTransition(name) {
+        this.invalidate(name);
+        this.wokeAt.delete(name);
         this.ctx.setTopicIcon(name, "remove");
         // B1: cancel any pending capacity backoff timer so a stopped/removed
         // instance is not restarted behind the user's back.