@cotal-ai/manager 0.55.0 → 0.56.1

package/dist/manager.js

@@ -4,7 +4,7 @@ import { hostname } from "node:os";
 import { credsAuthenticator } from "@nats-io/transport-node";
 import { existsSync, lstatSync, readFileSync, realpathSync, rmSync, statSync } from "node:fs";
 import { isAbsolute, join, dirname, resolve } from "node:path";
-import { CotalEndpoint, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, assertLifecycleToken, divergentSecretStoreNotice, parseDaemonStoreAnswer, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, mintCreds, mintGeneration, mintAcceptedToken, openIssuedStore, acceptedBucket, issuedBucket, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, epRequestSubject, epCallerReplyFilter, parseEpSubject, controlServiceSubject, eventChannelPrincipal, } from "@cotal-ai/core";
+import { CotalEndpoint, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, assertLifecycleToken, divergentSecretStoreNotice, parseDaemonStoreAnswer, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, credsRenewalDelayMs, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, mintCreds, mintGeneration, mintAcceptedToken, openIssuedStore, acceptedBucket, issuedBucket, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, epRequestSubject, epCallerReplyFilter, parseEpSubject, controlServiceSubject, eventChannelPrincipal, } from "@cotal-ai/core";
 import { agentAuthState, agentCredsDir, agentLifecycleSecretFilePaths, agentSecretFilePaths, agentSecretKeyForFile, authDir, connectorInstallHint, DEFAULT_CONNECTOR, defaultAgentType, DELIVERY_CREDS_KIND, extensionConnectors, findCotalRoot, getSpaceAuth, hasUserAuthState, loadExtensionsManifest, loadManagerInstanceIdentity, loadMeshes, localTrustOfSpace, manifestExtensionNames, materializeFromManifest, materializeSecretToFile, MEMBERSHIP_RW_CREDS_KIND, mergeLaunchOptions, remintDaemonCreds, resolveOnPath, createManagerInstanceIdentity, spaceMaterialKey, SYSTEM_CREDS_FILES, userAuthStateDir, workspaceSecretStore, writeRenewalRecord } from "@cotal-ai/workspace";
 import { createRuntime, isCustodialRuntime, requireRuntimeAdopt, requireRuntimeReap, RuntimeReapUnproven, } from "./runtime/index.js";
 import { AttachEndpoint } from "./attach-endpoint.js";
@@ -109,7 +109,7 @@ const STARTUP_RECONCILING = "startup static lifecycle reconciliation";
  *  broker/eviction blip; the wider later spacing avoids hammering a persistently unavailable
  *  authority. The finite ~36s window is intentionally not a liveness inference: exhausting it
  *  leaves the durable alias held and requires a later manager start. */
-const STATIC_RECONCILE_RETRY_DELAYS_MS = [1_000, 5_000, 30_000];
+export const STATIC_RECONCILE_RETRY_DELAYS_MS = [1_000, 5_000, 30_000];
 const STATIC_RECONCILE_MAX_ATTEMPTS = STATIC_RECONCILE_RETRY_DELAYS_MS.length + 1;
 /** The STABLE retirement opId for one lifecycle (#29 piece 3): deterministic from the uid, so a
  *  despawn retry, a same-name-spawn nudge, and the auth service's boot resume all drive the SAME
@@ -514,6 +514,11 @@ export class Manager {
     leaseStopping = false;
     /** The class-2 renewal owner's half-TTL schedule (D5 slice 5); armed only on auth meshes. */
     credRenewTimer;
+    /** #2073: the `endpoint-serve` credential's OWN renewal, armed from its `[renewAt, exp)` window
+     *  (`credsRenewalDelayMs`) rather than left to `credRenewTimer`'s boot-phased TTL/4 tick, which can
+     *  miss the window by ordering. Re-armed at registration and after every successful re-mint;
+     *  cleared in `stop()` beside `credRenewTimer`. */
+    endpointServeRenewalTimer;
     /** #1634: does THIS manager own the delivery daemon's credential renewal? True only while the
      *  daemon reloads from this manager's store AND this manager holds the per-space renewal lease.
      *  Starts false: a manager that has not proved both must not remint. */
@@ -639,17 +644,64 @@ export class Manager {
      * write it to. The operator reading the refusal is on the manager's mesh, never on the record's. A
      * boolean parameter is how that happened, so the mode is read from `this` and a third door cannot
      * pass the wrong one.
+     *
+     * The selector below reads the three-valued mesh mode (`open | static | user`), not a boolean
+     * derived from it (#567). `this.userMode` is `false` for BOTH static and open, so a boolean
+     * selector collapses those two and an open mesh was handed the static route (`cotal mint
+     * --provision`), which `mint` refuses outright on an open mesh. A boolean also has no third
+     * state to invert, so a mutation grid over `return this.userMode ? … : …` proves the one arm it
+     * has is load-bearing but is structurally blind to a missing arm: there is no mutant that turns
+     * "two arms" into "three arms". Switching on `meshMode` gives a fourth mesh mode nowhere to fall
+     * silently, since `switch` has no default arm to catch it.
      */
+    get meshMode() {
+        if (this.userMode)
+            return "user";
+        if (this.auth !== undefined)
+            return "static";
+        return "open";
+    }
+    /** The CHAT stream frontier at a preservation cut (issue #545), read through whichever local
+     *  authority this mesh mode holds: open mode's own endpoint already has `$JS.API.STREAM.INFO`
+     *  on the CHAT stream ({@link supervisorPermissions} grants it there); static/user mode's own
+     *  supervisor credential does NOT (it holds that grant only on the manager/presence/delivery
+     *  buckets), so those modes borrow the same ephemeral `provisioner` profile the manager already
+     *  mints per spawn, which DOES hold it under space auth. A remote-authority manager (`this.auth`
+     *  undefined in static/user mode) has no local space authority to mint that provisioner with, so
+     *  it prints one line and returns no floor — the inventory then carries none and resumed seats
+     *  behave exactly as before this fix, not a silent partial degrade. */
+    async chatFrontierForPreservation() {
+        switch (this.meshMode) {
+            case "open":
+                return this.ep.chatFrontier();
+            case "static":
+            case "user":
+                if (!this.auth) {
+                    console.error("! preservation: this manager holds no local space authority to read the chat frontier; retained seats replay their channel windows on resume");
+                    return undefined;
+                }
+                return this.withProvisioner((prov) => prov.chatFrontier());
+        }
+    }
     readerRemedy(owner, channel) {
-        return this.userMode
-            ? `\`cotal actor grant <reader> --owner ${owner} --scope '' --allow-subscribe '${channel}' ` +
-                `--allow-publish ''\`, with every field spelled out: \`actor grant\` is an upsert of the ` +
-                `WHOLE row and an omitted flag means the WIDE default (\`>\` read, \`>\` post, \`spawn,role:default\` ` +
-                `scope), not "leave it alone".`
-            : `\`cotal mint <reader> --profile agent --allow-subscribe ${channel} --provision\`, where there ` +
-                `is no actor ledger for \`actor grant\` to write to. The AGENT profile, not the observer ` +
-                `one: \`mint\` reads --allow-subscribe only for that profile, so an observer mint is ` +
-                `refused outright and writes no creds file.`;
+        switch (this.meshMode) {
+            case "user":
+                return (`\`cotal actor grant <reader> --owner ${owner} --scope '' --allow-subscribe '${channel}' ` +
+                    `--allow-publish ''\`, with every field spelled out: \`actor grant\` is an upsert of the ` +
+                    `WHOLE row and an omitted flag means the WIDE default (\`>\` read, \`>\` post, \`spawn,role:default\` ` +
+                    `scope), not "leave it alone".`);
+            case "static":
+                return (`\`cotal mint <reader> --profile agent --allow-subscribe ${channel} --provision\`, where there ` +
+                    `is no actor ledger for \`actor grant\` to write to. The AGENT profile, not the observer ` +
+                    `one: \`mint\` reads --allow-subscribe only for that profile, so an observer mint is ` +
+                    `refused outright and writes no creds file.`);
+            case "open":
+                return (`there is nothing to grant on an open mesh: it has no credentials and no ACLs, so any peer ` +
+                    `may read \`${channel}\` by connecting bare and listing that channel (\`cotal join\` or an ` +
+                    `endpoint whose channels include it); this refusal stands because the own-channel rule ` +
+                    `applies to every mesh, and a spawn is not the place to hand out a read on another agent's ` +
+                    `tool inputs and outputs.`);
+        }
     }
     start() {
         if (this.startTask)
@@ -1055,6 +1107,50 @@ export class Manager {
         }
         await this.keepDaemonRenewalLease();
     }
+    /** #2073: arm the `endpoint-serve` credential's own renewal from its `[renewAt, exp)` window
+     *  instead of `credRenewTimer`'s boot-phased TTL/4 tick, which can miss that window depending on
+     *  when the timer happened to start relative to the credential's `iat`. Called after the initial
+     *  mint (`registerManagerService`) and after every successful re-mint in the serve branch below,
+     *  so the schedule always tracks the CURRENT credential. `credsRenewalDelayMs` is the same 75%
+     *  convention the endpoint's delivery renewal and the membership feed use (`@cotal-ai/core`); no
+     *  second copy of that arithmetic lives here. */
+    scheduleServeRenewal(creds) {
+        if (this.endpointServeRenewalTimer)
+            clearTimeout(this.endpointServeRenewalTimer);
+        const delay = Math.max(1_000, credsRenewalDelayMs(creds));
+        const renewAt = new Date(Date.now() + delay).toISOString();
+        this.endpointServeRenewalTimer = setTimeout(() => { void this.renewDaemonCreds(); }, delay);
+        this.endpointServeRenewalTimer.unref?.();
+        console.error(`manager endpoint-serve renewal scheduled at ${renewAt}`);
+    }
+    /** #2073: push a freshly re-minted credential onto its LIVE connection instead of waiting for a
+     *  reconnect the expiry itself would force two dials too late. `nc.reconnect()` re-evaluates the
+     *  authenticator, which reads the mutable `creds` holder on every attempt, so the new CONNECT
+     *  presents the fresh credential without re-registration. In-flight requests on the connection
+     *  fail once, on the reconnect. Best-effort, matching `CotalEndpoint.swapConnectionOntoFreshCreds`:
+     *  a connection that is already closed (e.g. mid re-dial after a fault) cannot reconnect, and that
+     *  failure is not this call's to report. */
+    async pushRenewedCreds(nc) {
+        await nc.reconnect().catch(() => { });
+    }
+    /** #2073: the shared bounded-retry shape for a closed standing connection: wait out each of
+     *  {@link STATIC_RECONCILE_RETRY_DELAYS_MS}, trying `attempt` after each. A stop landing while
+     *  this loop waits is not a fault this loop reports (the caller re-checks `leaseStopping`). */
+    async boundedReconnect(label, attempt) {
+        for (const delayMs of STATIC_RECONCILE_RETRY_DELAYS_MS) {
+            await new Promise((r) => setTimeout(r, delayMs));
+            if (this.leaseStopping)
+                return true;
+            try {
+                if (await attempt())
+                    return true;
+            }
+            catch (e) {
+                console.error(`! manager ${label} re-dial attempt failed: ${e.message}`);
+            }
+        }
+        return false;
+    }
     /** One class-2 renewal pass (D5 slice 5): re-sign `.cotal/delivery.creds` + `.cotal/membership-rw.creds`
      *  for their existing nkeys, then request the delivery daemon's EXPLICIT `reloadCreds` adoption on the
      *  delivery-admin rail and persist the audit record (`.cotal/renewal.<spaceKey>.json`) that `cotal doctor auth`
@@ -1165,6 +1261,12 @@ export class Manager {
                             serveIssuance: serveIssuanceGateKv(authKv, this.space, { endpoint: MANAGER_ENDPOINT, instanceId: this.managerInstanceId }),
                             endpointServe: s.grant,
                         }));
+                        // #2073: a superseded instance's re-mint fails the gate CAS above and lands in the
+                        // catch below, not here — this re-mint already re-checks the issuance gate through
+                        // the SAME executor a fresh registration would use, so there is no second epoch read
+                        // to add before re-binding after a re-dial.
+                        await this.pushRenewedCreds(s.nc);
+                        this.scheduleServeRenewal(s.creds);
                     }
                 }
                 catch (e) {
@@ -1183,6 +1285,7 @@ export class Manager {
                         const fresh = await this.withEndpointServeExecutor(({ authKv }) => this.mintAndStageGoalWriter(authKv));
                         this.goalWriterCreds = fresh;
                         gw.creds = fresh;
+                        await this.pushRenewedCreds(gw.nc);
                     }
                 }
                 catch (e) {
@@ -1204,6 +1307,7 @@ export class Manager {
                         const fresh = await this.withEndpointServeExecutor(({ authKv }) => this.mintAndStageSessionLedger(authKv));
                         this.sessionLedgerCreds = fresh;
                         sw.creds = fresh;
+                        await this.pushRenewedCreds(sw.nc);
                     }
                 }
                 catch (e) {
@@ -1353,11 +1457,17 @@ export class Manager {
     async runPreparation(attemptId, generation) {
         await this.awaitLifecycleDrain();
         this.assertPreservationGeneration(attemptId, generation);
+        // Read the chat frontier BEFORE any child stops (issue #545): a message posted between this
+        // read and the child's stop is delivered live now and again as `historical` on resume, a
+        // duplicate bounded to the cut window that the seat's own id dedup on `durable` channels
+        // absorbs, whereas a read taken after the stops would instead skip messages the seat never
+        // actually saw.
+        const backfillFloor = await this.chatFrontierForPreservation();
         const inventory = this.preservationInventory ?? {
             version: "cotal-manager-resume/v1",
             space: this.space,
             createdAt: new Date().toISOString(),
-            agents: [...this.agents.values()].map((a) => this.resumeEntry(a)),
+            agents: [...this.agents.values()].map((a) => this.resumeEntry(a, backfillFloor)),
         };
         const failures = [];
         for (const entry of inventory.agents) {
@@ -1573,7 +1683,7 @@ export class Manager {
             return "";
         }
     }
-    resumeEntry(a) {
+    resumeEntry(a, backfillFloor) {
         const principal = a.userOwner
             ? parsePrincipalKey(a.id)
             : { owner: DEV_OWNER, actor: a.id };
@@ -1636,6 +1746,7 @@ export class Manager {
             spawner: a.spawner,
             authorityParent: a.authorityParent,
             startedAt: new Date(a.startedAt).toISOString(),
+            ...(backfillFloor !== undefined ? { backfillFloor } : {}),
         };
     }
     /** Tear down every managed agent's footprint on an explicit destructive {@link stop}. A manager exit is
@@ -1739,6 +1850,8 @@ export class Manager {
             clearInterval(this.leaseTimer);
         if (this.credRenewTimer)
             clearInterval(this.credRenewTimer);
+        if (this.endpointServeRenewalTimer)
+            clearTimeout(this.endpointServeRenewalTimer);
         if (this.daemonRenewalLeaseTimer)
             clearInterval(this.daemonRenewalLeaseTimer);
         if (this.sessionKeyRenewTimer)
@@ -3540,9 +3653,14 @@ export class Manager {
      *  uncertain black hole. */
     liveRosterNames() {
         const live = new Set();
-        for (const p of this.ep.getRoster())
-            if (p.status !== "offline")
-                live.add(p.card.name);
+        for (const p of this.ep.getRoster()) {
+            if (p.status === "offline")
+                continue;
+            const predecessor = this.confirmedRetiredPredecessors.get(p.card.name);
+            if (predecessor !== undefined && predecessor.principal === p.card.id && predecessor.lifecycleUid === p.lifecycleUid)
+                continue;
+            live.add(p.card.name);
+        }
         return live;
     }
     /** THE single name-liveness predicate both the hard-pinned collision refuse (M6, P2 item 2) and
@@ -3561,7 +3679,9 @@ export class Manager {
      *  choice at allocation, never an authority check (the broker still enforces): a stale
      *  still-live-looking row only costs a numbered suffix, and a missed freshly-joined occupant is
      *  still refused downstream exactly as before. Offline rows do NOT occupy — a properly retired
-     *  name stays reusable. */
+     *  name stays reusable. A row this manager itself confirmed retired (its alias, principal, and
+     *  lifecycleUid all matching a {@link confirmedRetiredPredecessors} entry) does not occupy
+     *  either, even while its advisory presence record ages out. */
     uniqueName(base) {
         const live = this.liveRosterNames();
         return firstFreeName(base, (n) => this.nameInUse(n, live));
@@ -3682,33 +3802,40 @@ export class Manager {
             route,
         }, caller, hooks);
     }
-    /** Resolve a placed spawn directory on this manager's host. The manager serves any existing
-     * absolute directory it can resolve. It creates nothing and never substitutes the workspace root. */
-    resolveSpawnCwd(value) {
-        const asked = typeof value === "string" ? value : String(value ?? "");
-        const refuse = (reason) => {
-            throw new EpEnvelopeError("failed-precondition", `cwd does not resolve on this host: ${JSON.stringify(asked)} (${reason})`);
-        };
-        if (!isAbsolute(asked))
-            refuse("expected an absolute path");
-        let cwd = asked;
+    /** The realpath + directory checks a placed spawn directory must pass on this manager's host,
+     * shared by the served `resolve-cwd` command and the `spawn` admission check (#385 item 2): one
+     * function, so the two doors never drift. Creates nothing and never substitutes the workspace root. */
+    checkSpawnCwd(resolved) {
+        let cwd = resolved;
         try {
-            cwd = realpathSync(asked);
+            cwd = realpathSync(resolved);
         }
         catch (error) {
-            refuse(`path cannot be resolved: ${error.message}`);
+            return { refusal: `path cannot be resolved: ${error.message}` };
         }
         try {
             if (!statSync(cwd).isDirectory())
-                refuse("path is not a directory");
+                return { refusal: "path is not a directory" };
         }
         catch (error) {
-            if (error instanceof EpEnvelopeError)
-                throw error;
-            refuse(error.message);
+            return { refusal: error.message };
         }
         return { cwd, host: hostname() };
     }
+    /** Resolve a placed spawn directory on this manager's host. The manager serves any existing
+     * absolute directory it can resolve. It creates nothing and never substitutes the workspace root. */
+    resolveSpawnCwd(value) {
+        const asked = typeof value === "string" ? value : String(value ?? "");
+        const refuse = (reason) => {
+            throw new EpEnvelopeError("failed-precondition", `cwd does not resolve on this host: ${JSON.stringify(asked)} (${reason})`);
+        };
+        if (!isAbsolute(asked))
+            refuse("expected an absolute path");
+        const result = this.checkSpawnCwd(asked);
+        if ("refusal" in result)
+            return refuse(result.refusal);
+        return result;
+    }
     /** Resolve a connector by agent type. Library composition (installedExtensions off) → a registry
      *  hit, exactly as before (the composition root imported what it wants). The published binary gates
      *  on MANIFEST membership FIRST — so a live `cotal ext remove` is honored even though the registry
@@ -4060,6 +4187,20 @@ export class Manager {
             if (this.userMode)
                 return { ok: false, error: "supervise is a restart policy this host cannot enforce: a user-mode seat has no static slot to keep the incarnation owned across a process death" };
         }
+        // A placed spawn's cwd is checked on this host BEFORE anything is minted or reserved: an absolute
+        // path this manager cannot realpath, or one that is not a directory, is refused here with the
+        // asked path, the reason and this host, so a misplaced spawn in a multi-manager space is
+        // diagnosable at the moment it happens (#385 item 2) instead of failing inside the pty launch.
+        // The resolved realpath is threaded to the launch cwd below (`resolvedCwd`) so it is computed once.
+        let resolvedCwd;
+        if (typeof opts.cwd === "string" && opts.cwd !== "") {
+            const asked = opts.cwd;
+            const target = resolve(this.workspaceRoot, asked);
+            const checked = this.checkSpawnCwd(target);
+            if ("refusal" in checked)
+                return { ok: false, error: `spawn: cwd does not resolve on this host (${hostname()}): ${JSON.stringify(asked)} (${checked.refusal}); the seat was not launched` };
+            resolvedCwd = checked.cwd;
+        }
         // Resolve the launch profile: IDENTITY (free-form `name:`) + role + read/post ACL + capabilities
         // + model/variant. Either from a fully-resolved manifest launch object (`opts.resolved`, whose `config`
         // is a materialized transient persona — the file is NOT the access authority), or from the
@@ -4411,7 +4552,9 @@ export class Manager {
             // Per-agent cwd overrides the manager's shared workspace root, so agents can be rooted at
             // arbitrary folders/repos. A relative path resolves against the workspace root; omitted → the
             // agent shares the workspace root (the prior, unchanged behavior).
-            const cwd = opts.cwd ? resolve(this.workspaceRoot, opts.cwd) : this.workspaceRoot;
+            // The realpath the admission check above already resolved (#385 item 2) — never re-resolved here,
+            // so the admission refusal and the launch cwd can never disagree on the same request.
+            const cwd = resolvedCwd ?? this.workspaceRoot;
             const configSha256 = this.fileDigest(configPath);
             const manifestPath = opts.launchRef
                 ? join(this.workspaceRoot, ".cotal", "run", `${opts.launchRef.runId}.json`)
@@ -4490,7 +4633,7 @@ export class Manager {
                             configSha256,
                             manifestSha256,
                         }
-                        : { kind: "persona", ref, configPath, configSha256 },
+                        : { kind: "persona", ref: opts.config ? identityName : ref, configPath, configSha256 },
                     cwd,
                     model,
                     variant,
@@ -5007,6 +5150,7 @@ export class Manager {
                     // with no COTAL_LIFECYCLE_UID: static/user fail the connector auth gate and open self-mints a
                     // fresh uid that orphans the preserved durables and never matches the readiness fence.
                     lifecycleUid: entry.identity.lifecycleUid,
+                    backfillFloor: entry.backfillFloor,
                     acceptedToken: entry.identity.mode === "static" ? entry.identity.issued?.acceptedToken : undefined,
                     servers: this.servers,
                     configPath: entry.launch.source.configPath,
@@ -5571,6 +5715,104 @@ export class Manager {
         }
         console.error(`✓ boot self-heal: ${MANAGER_ENDPOINT}/${instanceId} registration gate reopened at generation ${report.reopenedAtGeneration} (processEpoch ${report.after.processEpoch}, registrationRevision ${report.after.registrationRevision}; freeze-holder gone, sweepComplete=true). Continuing the normal takeover.`);
     }
+    /** #2073: dial the manager's ONE control-plane connection and serve its typed command surface on
+     *  it, exactly the shape `registerManagerService` used inline before this lane. Shared by the
+     *  initial registration AND the closed-connection recovery path ({@link onServeConnectionClosed}),
+     *  so there is one place that dials and serves this endpoint. Attaches {@link
+     *  onServeConnectionClosed} as the close handler in both cases: a closed serve connection is
+     *  always a fault this manager acts on, never a bare log line. */
+    async dialServeConnection(state) {
+        const enc = new TextEncoder();
+        const nc = await this.dial({
+            // Open mesh: a bare serve connection (no credential exists; the broker enforces nothing).
+            ...(state.creds !== undefined ? { authenticator: (nonce) => credsAuthenticator(enc.encode(state.creds))(nonce) } : {}),
+            inboxPrefix: `_INBOX_${state.identity.id}`,
+            maxReconnectAttempts: -1,
+        });
+        nc.closed().then((err) => { void this.onServeConnectionClosed(err ?? undefined); });
+        let handle;
+        try {
+            // The 1b typed surface + the derived `describe`. The descriptor stays PUBLIC in static
+            // mode: the broker grant (who holds each command's request-publish row) is the
+            // load-bearing authority tier, and a static single-operator mesh leaks nothing by listing
+            // command names; the trusted per-caller `view(caller)` scoping joins the user-mode
+            // registration follow-up (where actorScope is the trusted source). Every ordinary handler
+            // runs the SHARED admission chokepoint ({@link serveGated}).
+            handle = serveEndpoint(nc, this.space, state.grant, this.managerServiceDefs(), { public: true }, {
+                // The FRESH target resolver (§13.3) for the targeted commands (`despawn`/`attach`): the
+                // manager's live managed set IS the current-mapping authority for its own agents (the
+                // durable slot rows mirror it). Static mode carries no mapping-revision dimension, so
+                // the revision is the constant 0 — a caller that pins a revision pins 0.
+                resolveTarget: (t) => {
+                    if (t.owner === DEV_OWNER) {
+                        for (const a of this.agents.values())
+                            if (!a.userOwner && a.id === t.actor)
+                                return { lifecycleUid: a.lifecycleUid, mappingRevision: 0 };
+                        return undefined;
+                    }
+                    const key = principalKey(t.owner, t.actor).key;
+                    for (const a of this.agents.values())
+                        if (a.userOwner && a.id === key)
+                            return { lifecycleUid: a.lifecycleUid, mappingRevision: 0 };
+                    return undefined;
+                },
+                ...(this.omitClassCommands() ? { omitClassCommands: this.omitClassCommands() } : {}),
+            });
+        }
+        catch (e) {
+            await nc.drain().catch(() => nc.close());
+            throw e;
+        }
+        return { nc, handle };
+    }
+    /** #2073: re-mint the SAME `endpoint-serve` identity through the SAME scoped one-shot executor and
+     *  §13.1 mint fence `renewDaemonCreds`'s serve branch uses — never a second copy of that mint. Used
+     *  by both the periodic renewal pass and the closed-connection recovery path below, so a
+     *  superseded instance's re-mint fails the SAME gate CAS in both places and lands in the exit
+     *  path rather than needing a second epoch check. */
+    async remintServeCreds(s) {
+        const authRef = this.auth;
+        return this.withEndpointServeExecutor(({ authKv }) => mintCreds(authRef, s.identity, "endpoint-serve", {
+            serveIssuance: serveIssuanceGateKv(authKv, this.space, { endpoint: MANAGER_ENDPOINT, instanceId: this.managerInstanceId }),
+            endpointServe: s.grant,
+        }));
+    }
+    /** #2073: a closed serve connection is a FAULT the manager acts on, not a log line: this is the
+     *  control plane a report showed staying dead for hours while the process kept its liveness
+     *  lease and answered nothing (#2073). `stop()` sets `leaseStopping` BEFORE it tears down this
+     *  connection, so a close landing from OUR OWN graceful teardown returns here at once. Any other
+     *  close re-dials with the current credential (a transport blip needs nothing new), re-mints
+     *  through the existing serve branch when the credential itself has expired, and re-serves on the
+     *  new connection — bounded by {@link STATIC_RECONCILE_RETRY_DELAYS_MS}. Exhausting every attempt
+     *  releases this process's lease and exits loud (without `withAgents`, so seats stay up, the path
+     *  `supervise`'s signal handler already uses) so a restart can serve: a manager that holds the
+     *  lease and answers nothing is the worst of the three states this issue named, and holding the
+     *  connection open under unbounded reconnects (the transport-drop case) never reaches this method
+     *  at all — `nc.closed()` does not resolve until the client itself gives up. */
+    async onServeConnectionClosed(err) {
+        if (this.leaseStopping)
+            return;
+        const s = this.serviceServe;
+        console.error(`! manager service endpoint connection closed: ${err?.message ?? "unknown reason"}`);
+        const recovered = await this.boundedReconnect("service endpoint", async () => {
+            if (!s)
+                return false;
+            if (inspectCredHealth(s.creds ?? "").state !== "healthy" && s.creds !== undefined) {
+                s.creds = await this.remintServeCreds(s);
+                this.scheduleServeRenewal(s.creds);
+            }
+            const { nc, handle } = await this.dialServeConnection(s);
+            s.nc = nc;
+            s.handle = handle;
+            console.error(`manager service endpoint re-dialed: ${MANAGER_ENDPOINT}/${this.managerInstanceId} (epoch ${s.grant.epoch})`);
+            return true;
+        });
+        if (this.leaseStopping || recovered)
+            return;
+        console.error(`! manager service endpoint could not be restored for ${this.space}/${MANAGER_ENDPOINT}/${this.managerInstanceId}: releasing the lease and exiting`);
+        await this.stop();
+        process.exit(1);
+    }
     /** P2 item 1: register the manager as an ordinary v0.4 `service` endpoint and serve its typed
      *  command surface on the ep rails - since 1d the manager's ONLY control door. On an AUTH mesh
      *  the whole credential path is the SAME one an ordinary endpoint traverses (the enforcement
@@ -5820,50 +6062,16 @@ export class Manager {
         // without re-registration. Reconnects stay unbounded: the serve rails are this instance's
         // registered surface for its whole incarnation.
         const state = { handle: undefined, nc: undefined, identity: serveIdentity, grant, creds };
-        const enc = new TextEncoder();
-        const nc = await this.dial({
-            // Open mesh: a bare serve connection (no credential exists; the broker enforces nothing).
-            ...(creds !== undefined ? { authenticator: (nonce) => credsAuthenticator(enc.encode(state.creds))(nonce) } : {}),
-            inboxPrefix: `_INBOX_${serveIdentity.id}`,
-            maxReconnectAttempts: -1,
-        });
-        nc.closed().then((err) => { if (err)
-            console.error(`! manager service endpoint connection closed: ${err.message}`); });
-        try {
-            // The 1b typed surface + the derived `describe`. The descriptor stays PUBLIC in static
-            // mode: the broker grant (who holds each command's request-publish row) is the
-            // load-bearing authority tier, and a static single-operator mesh leaks nothing by listing
-            // command names; the trusted per-caller `view(caller)` scoping joins the user-mode
-            // registration follow-up (where actorScope is the trusted source). Every ordinary handler
-            // runs the SHARED admission chokepoint ({@link serveGated}).
-            state.handle = serveEndpoint(nc, this.space, grant, this.managerServiceDefs(), { public: true }, {
-                // The FRESH target resolver (§13.3) for the targeted commands (`despawn`/`attach`): the
-                // manager's live managed set IS the current-mapping authority for its own agents (the
-                // durable slot rows mirror it). Static mode carries no mapping-revision dimension, so
-                // the revision is the constant 0 — a caller that pins a revision pins 0.
-                resolveTarget: (t) => {
-                    if (t.owner === DEV_OWNER) {
-                        for (const a of this.agents.values())
-                            if (!a.userOwner && a.id === t.actor)
-                                return { lifecycleUid: a.lifecycleUid, mappingRevision: 0 };
-                        return undefined;
-                    }
-                    const key = principalKey(t.owner, t.actor).key;
-                    for (const a of this.agents.values())
-                        if (a.userOwner && a.id === key)
-                            return { lifecycleUid: a.lifecycleUid, mappingRevision: 0 };
-                    return undefined;
-                },
-                ...(this.omitClassCommands() ? { omitClassCommands: this.omitClassCommands() } : {}),
-            });
-        }
-        catch (e) {
-            await nc.drain().catch(() => nc.close());
-            throw e;
-        }
+        const { nc, handle } = await this.dialServeConnection(state);
         state.nc = nc;
+        state.handle = handle;
         this.serviceServe = state;
         console.error(`manager service endpoint registered: ${MANAGER_ENDPOINT}/${iid} (epoch ${grant.epoch}, registrationRevision ${grant.registrationRevision})`);
+        // #2073: arm the serve credential's own renewal now, from ITS window, instead of leaving it to
+        // `credRenewTimer`'s boot-phased TTL/4 tick. Open mesh mints no credential (`creds` is
+        // undefined) and has nothing to schedule.
+        if (state.creds !== undefined)
+            this.scheduleServeRenewal(state.creds);
     }
     /** P2 item 2 must-5 (b): mint the standing `goal-writer` credential and STAGE it into this
      *  instance's §13.1 revocation family (`epcred.<e>.<iid>`), over the passed executor's `authKv`
@@ -5911,19 +6119,18 @@ export class Manager {
      *  renewal is adopted without reconnecting the whole endpoint); the ActionContext bonds its
      *  KV + JS + JSM to this one connection and space (SPEC 13.4), so a composition mixup cannot splice
      *  goal state across brokers. */
-    async startGoalWriter() {
-        const identity = (this.auth || this.remoteAuthority) ? this.goalWriterIdentity : newIdentity();
+    /** #2073: dial the goal-writer's ONE standing connection and rebuild everything derived from it
+     *  (the fence-resolving action context, the issuance-gate reader) — shared by the initial start
+     *  and the closed-connection recovery ({@link onGoalWriterConnectionClosed}), so a redial never
+     *  leaves the context bound to a dead socket. */
+    async dialGoalWriter(gw) {
         const enc = new TextEncoder();
-        // The mutable holder captured by the authenticator (mirrors the serve connection): a half-TTL
-        // renewal updates `gw.creds` and the next (re)connect presents the refreshed credential.
-        const gw = { nc: undefined, ctx: undefined, creds: (this.auth || this.remoteAuthority) ? this.goalWriterCreds : undefined, identity };
         const nc = await this.dial({
             ...((this.auth || this.remoteAuthority) ? { authenticator: (nonce) => credsAuthenticator(enc.encode(gw.creds))(nonce) } : {}),
-            inboxPrefix: `_INBOX_${identity.id}`,
+            inboxPrefix: `_INBOX_${gw.identity.id}`,
             maxReconnectAttempts: -1,
         });
-        nc.closed().then((err) => { if (err)
-            console.error(`! manager goal-writer connection closed: ${err.message}`); });
+        nc.closed().then((err) => { void this.onGoalWriterConnectionClosed(err ?? undefined); });
         gw.nc = nc;
         // The (i) fence resolver (SPEC 13.6 P2 item 3): resolve an executing instance's CURRENT gate
         // epoch. This manager reconciles only ITS OWN goals (security pin 4), so it resolves its own
@@ -5949,6 +6156,40 @@ export class Manager {
         // rather than closing it. An AUTH mesh gets durable closure from the §13.1 barrier (revoke +
         // cluster-verified eviction BEFORE the epoch advance); an open mesh gets none.
         gw.gate = serveIssuanceGateKv(await new Kvm(nc).open(epAuthBucket(this.space)), this.space, { endpoint: MANAGER_ENDPOINT, instanceId: this.managerInstanceId });
+    }
+    /** #2073: the goal-writer rail's closed-connection fault, the same bounded shape the serve rail
+     *  uses ({@link onServeConnectionClosed}) through the same {@link boundedReconnect} helper:
+     *  re-dial with the current credential (a transport blip needs nothing new), re-mint through the
+     *  existing goal-writer renewal branch when the credential itself has expired, then re-attach.
+     *  Unlike the serve rail this never exits the process on exhaustion (brief step 4: only the serve
+     *  rail is the control plane the report lost) — it logs and leaves the credential to the next
+     *  scheduled renewal pass. */
+    async onGoalWriterConnectionClosed(err) {
+        if (this.leaseStopping)
+            return;
+        console.error(`! manager goal-writer connection closed: ${err?.message ?? "unknown reason"}`);
+        const gw = this.goalWriter;
+        const recovered = await this.boundedReconnect("goal-writer", async () => {
+            if (!gw)
+                return false;
+            if (gw.creds !== undefined && inspectCredHealth(gw.creds).state !== "healthy") {
+                gw.creds = await this.withEndpointServeExecutor(({ authKv }) => this.mintAndStageGoalWriter(authKv));
+                this.goalWriterCreds = gw.creds;
+            }
+            await this.dialGoalWriter(gw);
+            console.error("manager goal-writer re-dialed");
+            return true;
+        });
+        if (this.leaseStopping || recovered)
+            return;
+        console.error("! manager goal-writer could not be re-dialed; leaving the credential to the next renewal pass");
+    }
+    async startGoalWriter() {
+        const identity = (this.auth || this.remoteAuthority) ? this.goalWriterIdentity : newIdentity();
+        // The mutable holder captured by the authenticator (mirrors the serve connection): a half-TTL
+        // renewal updates `gw.creds` and the next (re)connect presents the refreshed credential.
+        const gw = { nc: undefined, ctx: undefined, creds: (this.auth || this.remoteAuthority) ? this.goalWriterCreds : undefined, identity };
+        await this.dialGoalWriter(gw);
         this.goalWriter = gw;
         console.error(`manager goal-writer standing (endpoint ${MANAGER_ENDPOINT}, ${(this.auth || this.remoteAuthority) ? "scoped cred, §13.1 family-staged" : "open/bare"})`);
     }
@@ -6016,20 +6257,55 @@ export class Manager {
      *  holder never verifies the signature (it presents the grant back over the rail; the broker's
      *  per-session caller cred is the holder's real fence). The plane's ledger lives in the DEDICATED
      *  sessions bucket (createEndpointStreams provisioned it at registration). */
-    async startSessionPlane() {
-        const identity = (this.auth || this.remoteAuthority) ? this.sessionLedgerIdentity : newIdentity();
+    /** #2073: dial the session-ledger's ONE standing connection — shared by the initial start and the
+     *  closed-connection recovery ({@link onSessionLedgerConnectionClosed}), the same shape {@link
+     *  dialGoalWriter} uses for its rail. The KV/JetStream clients the session plane opened from the
+     *  connection at start stay bound to it for this incarnation's life; a redial refreshes the
+     *  standing authenticator connection the credential renewal presents onto (mirroring `sw.creds`),
+     *  which is what a closed-connection fault on this rail needs restored. */
+    async dialSessionLedgerConnection(sw, identity) {
         const enc = new TextEncoder();
-        // The mutable holder captured by the authenticator (mirrors the goal-writer): a half-TTL renewal
-        // updates `sw.creds` and the next (re)connect presents the refreshed credential.
-        const sw = { nc: undefined, creds: (this.auth || this.remoteAuthority) ? this.sessionLedgerCreds : undefined };
         const nc = await this.dial({
             ...((this.auth || this.remoteAuthority) ? { authenticator: (nonce) => credsAuthenticator(enc.encode(sw.creds))(nonce) } : {}),
             inboxPrefix: `_INBOX_${identity.id}`,
             maxReconnectAttempts: -1,
         });
-        nc.closed().then((err) => { if (err)
-            console.error(`! manager session-ledger connection closed: ${err.message}`); });
+        nc.closed().then((err) => { void this.onSessionLedgerConnectionClosed(err ?? undefined); });
         sw.nc = nc;
+    }
+    /** #2073: the session-ledger rail's closed-connection fault, the same bounded shape {@link
+     *  onGoalWriterConnectionClosed} uses: re-dial with the current credential, re-mint through the
+     *  existing session-ledger renewal branch when the credential itself has expired, then re-attach.
+     *  Exhaustion logs and leaves the credential to the next renewal pass; it never exits the process
+     *  (brief step 4: only the serve rail is the control plane the report lost). */
+    async onSessionLedgerConnectionClosed(err) {
+        if (this.leaseStopping)
+            return;
+        console.error(`! manager session-ledger connection closed: ${err?.message ?? "unknown reason"}`);
+        const sw = this.sessionLedgerConn;
+        const identity = this.sessionLedgerIdentity;
+        const recovered = await this.boundedReconnect("session-ledger", async () => {
+            if (!sw || !identity)
+                return false;
+            if (sw.creds !== undefined && inspectCredHealth(sw.creds).state !== "healthy") {
+                sw.creds = await this.withEndpointServeExecutor(({ authKv }) => this.mintAndStageSessionLedger(authKv));
+                this.sessionLedgerCreds = sw.creds;
+            }
+            await this.dialSessionLedgerConnection(sw, identity);
+            console.error("manager session-ledger re-dialed");
+            return true;
+        });
+        if (this.leaseStopping || recovered)
+            return;
+        console.error("! manager session-ledger could not be re-dialed; leaving the credential to the next renewal pass");
+    }
+    async startSessionPlane() {
+        const identity = (this.auth || this.remoteAuthority) ? this.sessionLedgerIdentity : newIdentity();
+        // The mutable holder captured by the authenticator (mirrors the goal-writer): a half-TTL renewal
+        // updates `sw.creds` and the next (re)connect presents the refreshed credential.
+        const sw = { nc: undefined, creds: (this.auth || this.remoteAuthority) ? this.sessionLedgerCreds : undefined };
+        await this.dialSessionLedgerConnection(sw, identity);
+        const nc = sw.nc;
         const serveEpoch = this.serviceServe?.grant.epoch;
         if (serveEpoch === undefined)
             throw new Error("the manager session plane needs the serve grant epoch; registerManagerService must run first");
@@ -7262,10 +7538,13 @@ export class Manager {
                     managerInstance: this.managerInstanceId, managerProcessUid: this.managerLifecycleUid,
                 }, { cleanup: reapThenCleanup, evict, log: (line) => console.error(`static retirement ${a.name}: ${line}`) });
             });
-            this.retiredPrincipals.add(principalKey(DEV_OWNER, a.id).key);
+            const principal = principalKey(DEV_OWNER, a.id).key;
+            this.retiredPrincipals.add(principal);
             const cur = this.retiring.get(a.name);
-            if (cur && cur.lifecycleUid === a.lifecycleUid)
+            if (cur && cur.lifecycleUid === a.lifecycleUid) {
                 this.retiring.delete(a.name); // ABA-guarded hold clear
+                this.confirmedRetiredPredecessors.set(a.name, { principal, lifecycleUid: a.lifecycleUid });
+            }
         }
         catch (e) {
             const h = this.retiring.get(a.name);