@cotal-ai/manager 0.57.0 → 0.58.0

package/dist/manager.js

@@ -4,7 +4,7 @@ import { hostname } from "node:os";
 import { credsAuthenticator } from "@nats-io/transport-node";
 import { existsSync, lstatSync, readFileSync, realpathSync, rmSync, statSync } from "node:fs";
 import { isAbsolute, join, dirname, resolve } from "node:path";
-import { CotalEndpoint, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, assertLifecycleToken, divergentSecretStoreNotice, parseDaemonStoreAnswer, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, credsRenewalDelayMs, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, mintCreds, mintGeneration, mintAcceptedToken, openIssuedStore, acceptedBucket, issuedBucket, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, epRequestSubject, epCallerReplyFilter, parseEpSubject, controlServiceSubject, eventChannelPrincipal, } from "@cotal-ai/core";
+import { CotalEndpoint, accountFromCreds, credsClaims, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, assertLifecycleToken, divergentSecretStoreNotice, parseDaemonStoreAnswer, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, DeprovisionError, isConcreteChannel, deprovisionTargetPrincipal, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, credsRenewalDelayMs, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, mintCreds, mintGeneration, mintAcceptedToken, openIssuedStore, acceptedBucket, issuedBucket, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, controlServiceSubject, eventChannelPrincipal, resolveService, invokeCommand, } from "@cotal-ai/core";
 import { agentAuthState, agentCredsDir, agentLifecycleSecretFilePaths, agentSecretFilePaths, agentSecretKeyForFile, authDir, connectorInstallHint, DEFAULT_CONNECTOR, defaultAgentType, DELIVERY_CREDS_KIND, extensionConnectors, findCotalRoot, getSpaceAuth, hasUserAuthState, loadExtensionsManifest, loadManagerInstanceIdentity, loadMeshes, localTrustOfSpace, manifestExtensionNames, materializeFromManifest, materializeSecretToFile, MEMBERSHIP_RW_CREDS_KIND, mergeLaunchOptions, remintDaemonCreds, resolveOnPath, createManagerInstanceIdentity, spaceMaterialKey, SYSTEM_CREDS_FILES, userAuthStateDir, workspaceSecretStore, writeRenewalRecord } from "@cotal-ai/workspace";
 import { createRuntime, isCustodialRuntime, requireRuntimeAdopt, requireRuntimeReap, RuntimeReapUnproven, } from "./runtime/index.js";
 import { AttachEndpoint } from "./attach-endpoint.js";
@@ -197,55 +197,6 @@ function parseTurnNote(raw) {
 function turnHoldToken(goalId) {
     return createHash("sha256").update(`${goalId}:turn-deadline`, "utf8").digest("base64url").slice(0, 43);
 }
-/** One ep request/reply round-trip on the caller's OWN reply-plane filter (§13.2). The responder
- *  derives the reply subject from the authenticated request, so there is no caller-selected reply
- *  target to honour; the caller binds the answer off the reply SUBJECT — endpoint and nonce, both
- *  broker-pinned by the responder's serve publish grant. A reply on another nonce belongs to a
- *  different request (the rail is shared) and is ignored rather than allowed to fail this one. */
-export async function epAwaitReply(nc, space, caller, nonce, requestId, requestSubject, body, timeoutMs) {
-    let sub;
-    let timer;
-    try {
-        const got = new Promise((resolve, reject) => {
-            sub = nc.subscribe(epCallerReplyFilter(space, caller), {
-                callback: (err, msg) => {
-                    if (err) {
-                        reject(new Error(`the retirement reply subscription failed: ${err.message}`));
-                        return;
-                    }
-                    const parsed = parseEpSubject(msg.subject);
-                    if (!parsed || parsed.plane !== "reply" || parsed.endpoint !== AUTH_ENDPOINT || parsed.nonce !== nonce)
-                        return;
-                    let body;
-                    try {
-                        body = JSON.parse(new TextDecoder().decode(msg.data));
-                    }
-                    catch {
-                        return;
-                    } // a malformed body on our nonce is not an answer; keep waiting
-                    // REQUIRE THE ID ECHO. Binding on (endpoint, nonce) alone would accept a malformed or
-                    // WRONG-ID `{ok:true}` and clear a retirement hold on it. Ignore rather than fail: the
-                    // rail is shared across concurrent requests and a responder may publish at any nonce, so
-                    // an unmatched reply is someone else's answer, never grounds to fail the honest one.
-                    if (body.id !== requestId)
-                        return;
-                    resolve(body);
-                },
-            });
-            timer = setTimeout(() => reject(new Error("timeout")), timeoutMs);
-            nc.publish(requestSubject, new TextEncoder().encode(body));
-        });
-        return await got;
-    }
-    finally {
-        if (timer)
-            clearTimeout(timer);
-        try {
-            sub?.unsubscribe();
-        }
-        catch { /* connection already down */ }
-    }
-}
 /**
  * The agent supervisor: a long-lived mesh node that owns agent process lifecycle.
  * It serves control requests on the "manager" service and spawns/kills agents
@@ -467,6 +418,8 @@ export class Manager {
      *  lifecycle authority: every re-drive re-reads broker KV and re-runs planStaticSlotResume. */
     staticReconcileItems = new Map();
     staticReconcileLastSweep;
+    staticReconcileLastSweepResources;
+    lastDeprovisionResources;
     staticReconcileSweepsInFlight = 0;
     staticReconcileDrainWaiters = [];
     staticReconcileStopping = false;
@@ -549,6 +502,11 @@ export class Manager {
     resumedAgentNames = new Set();
     remoteAuthority;
     remoteExecutorCreds;
+    remoteSupervisorCreds;
+    remoteBundleRenewal;
+    /** A refused or failed standing candidate is cleanup debt, never a silent loss: the last-good
+     * family stays live and this records what the host may have signed but the manager never adopted. */
+    remoteRenewalDebt;
     /** Every broker dial this manager makes, through ONE transport decision.
      *
      *  `this.servers` is whatever the mesh record held (`cotal supervise` reads it from there), so it
@@ -561,6 +519,10 @@ export class Manager {
         return dialerFor(servers)({ ...opts, servers });
     }
     constructor(opts) {
+        if (opts.pooled && (!opts.remoteAuthority || opts.runtime === undefined || opts.runtime === "auto" || opts.runtime === "pty"))
+            throw new Error("pooled control requires signerless remote authority and an explicit non-PTY runtime; local custodial execution is forbidden");
+        if (opts.pooled && !opts.remoteAuthority?.renewStandingBundle)
+            throw new Error("pooled control requires a closed host-issued all-duty renewal callback before construction");
         this.space = opts.space;
         this.servers = opts.servers;
         this.name = opts.name ?? "manager";
@@ -569,6 +531,10 @@ export class Manager {
         this.maxSessions = opts.maxSessions;
         this.remoteAuthority = opts.remoteAuthority;
         this.remoteExecutorCreds = opts.remoteAuthority?.executorCreds;
+        this.remoteSupervisorCreds = opts.remoteAuthority?.supervisorCreds;
+        if (opts.pooled && (!opts.remoteAuthority?.accountPublicKey ||
+            accountFromCreds(opts.remoteAuthority.supervisorCreds) !== opts.remoteAuthority.accountPublicKey))
+            throw new Error("pooled control requires a proved assigned account for its initial supervisor credential");
         if (opts.remoteAuthority)
             this.managerLifecycleUid = opts.remoteAuthority.lifecycleUid;
         this.secrets = opts.secretStore ?? workspaceSecretStore(this.workspaceRoot);
@@ -577,6 +543,8 @@ export class Manager {
             : { kind: "fs", root: resolve(this.workspaceRoot) };
         this.installedExtensions = opts.installedExtensions ?? false;
         this.runtime = createRuntime(opts.runtime ?? "auto", `cotal-${this.space}`);
+        if (opts.pooled && isCustodialRuntime(this.runtime))
+            throw new Error(`pooled control refuses custodial runtime "${this.runtime.kind}" before starting agents`);
         this.preserveStopTimeoutMs = opts.preserveStopTimeoutMs ?? PRESERVE_STOP_TIMEOUT_MS;
         this.endpointServeExecutorExpiresInSeconds = opts.endpointServeExecutorExpiresInSeconds;
         if (opts.resumeAttemptId && !/^[A-Za-z0-9_-]{1,128}$/.test(opts.resumeAttemptId))
@@ -792,7 +760,7 @@ export class Manager {
         if (this.remoteAuthority) {
             const remote = this.remoteAuthority;
             id = remote.identities.supervisor.id;
-            creds = async () => remote.supervisorCreds;
+            creds = async () => this.remoteSupervisorCreds;
         }
         else if (this.auth) {
             const identity = newIdentity();
@@ -895,8 +863,38 @@ export class Manager {
             this.credRenewTimer.unref?.();
         }
         else if (this.remoteAuthority) {
-            await this.renewRemoteExecutor();
-            this.credRenewTimer = setInterval(() => { void this.renewRemoteExecutor(); }, credRenewIntervalMs(5 * 60));
+            if (this.remoteAuthority.renewStandingBundle)
+                await this.renewRemoteStandingBundle();
+            else
+                await this.renewRemoteExecutor();
+            let standingTtl = STANDING_RENEWABLE_TTL_SEC;
+            if (this.remoteSupervisorCreds) {
+                try {
+                    const claims = credsClaims(this.remoteSupervisorCreds);
+                    if (typeof claims.exp === "number" && typeof claims.iat === "number" && claims.exp > claims.iat) {
+                        standingTtl = claims.exp - claims.iat;
+                    }
+                }
+                catch { }
+            }
+            const intervalMs = Math.min(credRenewIntervalMs(5 * 60), credRenewIntervalMs(standingTtl));
+            this.credRenewTimer = setInterval(async () => {
+                try {
+                    if (this.remoteAuthority?.renewStandingBundle)
+                        await this.renewRemoteStandingBundle();
+                    else
+                        await this.renewRemoteExecutor();
+                }
+                catch (e) {
+                    console.error(`! remote manager renewal pass failed: ${e.message}`);
+                }
+                try {
+                    await this.runHosting?.renew();
+                }
+                catch (e) {
+                    console.error(`! remote run-hosting renewal pass failed: ${e.message}`);
+                }
+            }, intervalMs);
             this.credRenewTimer.unref?.();
         }
         // stop() fences before it waits for an accepted startup reconciliation terminal. Once that
@@ -929,9 +927,26 @@ export class Manager {
         // incarnation takes back every run a dead predecessor was driving before it accepts new ones.
         // The serve surface is already live by here, so the family itself holds the gate: `runHost()`
         // refuses `run-start`/`run-resume` as `unavailable` until the host exists and `RunHosting`
-        // refuses them until its reconcile has returned. A user-auth mesh stands no host up at all
-        // (`runHost()` names why); a remote-authority manager holds no signer to mint with.
-        if (!this.remoteAuthority && !this.userMode) {
+        // refuses them until its reconcile has returned. A remote user-auth manager uses its four
+        // registered issuer callbacks instead of holding a signer; a local user mesh stands none up.
+        const remoteRuns = this.remoteAuthority?.runHosting;
+        if (this.remoteAuthority && remoteRuns) {
+            this.runHosting = new RunHosting({
+                space: this.space,
+                servers: this.servers,
+                endpoint: MANAGER_ENDPOINT,
+                instanceId: this.managerInstanceId,
+                holder: { id: this.remoteAuthority.identities.supervisor.id, lifecycleUid: this.managerLifecycleUid },
+                auth: undefined,
+                log: (line) => console.error(line),
+                admitRun: remoteRuns.admitRun,
+                issueAttempt: remoteRuns.issueAttempt,
+                issueOperator: remoteRuns.issueOperator,
+                renewRun: remoteRuns.renewRun,
+            });
+            await this.runHosting.reconcile();
+        }
+        else if (!this.remoteAuthority && !this.userMode) {
             this.runHosting = new RunHosting({
                 space: this.space,
                 servers: this.servers,
@@ -1339,6 +1354,94 @@ export class Manager {
                 throw e;
         }
     }
+    /** An active signerless manager renews its standing duties together. The issuer stages the closed
+     * credential family first; broker preflight proves every JWT and existing caller-held nkey
+     * before any mutable holder changes. A failed candidate leaves all current creds intact. */
+    async renewRemoteStandingBundle(force = false) {
+        if (this.remoteBundleRenewal)
+            return this.remoteBundleRenewal;
+        const renewal = this.renewRemoteStandingBundleOnce(force);
+        this.remoteBundleRenewal = renewal;
+        try {
+            await renewal;
+        }
+        finally {
+            if (this.remoteBundleRenewal === renewal)
+                this.remoteBundleRenewal = undefined;
+        }
+    }
+    /** A signerless manager's closed standing connection re-dials through the all-duty family only.
+     *  A healthy holder re-dials as is. An unhealthy one waits for the closed renewal and adoption,
+     *  and a refusal fails this attempt with last-good and debt kept. Nothing is minted locally. */
+    async renewRemoteBeforeRedial(label, read) {
+        if (inspectCredHealth(read() ?? "").state === "healthy")
+            return;
+        await this.renewRemoteStandingBundle(true);
+        if (inspectCredHealth(read() ?? "").state !== "healthy")
+            throw new Error(`the ${label} credential is not healthy after all-duty renewal; last-good is kept and nothing is minted locally`);
+    }
+    async renewRemoteStandingBundleOnce(force) {
+        const remote = this.remoteAuthority;
+        if (!remote?.renewStandingBundle || !this.serviceServe)
+            return;
+        const current = [this.remoteSupervisorCreds, this.remoteExecutorCreds, this.serviceServe.creds,
+            this.goalWriterCreds, this.sessionLedgerCreds];
+        if (!force && current.every((cred) => cred && inspectCredHealth(cred).state === "healthy"))
+            return;
+        const processEpoch = this.serviceServe.grant.epoch;
+        let candidate;
+        try {
+            candidate = await remote.renewStandingBundle(processEpoch);
+            const names = ["supervisor", "executor", "serve", "goalWriter", "sessionLedger"];
+            if (Object.keys(candidate).sort().join(",") !== [...names].sort().join(","))
+                throw new Error("host returned an incomplete or widened standing credential family");
+            for (const name of names) {
+                if (typeof candidate[name] !== "string" || inspectCredHealth(candidate[name]).state === "unreadable")
+                    throw new Error(`host returned an unreadable ${name} credential`);
+                if (!remote.accountPublicKey || accountFromCreds(candidate[name]) !== remote.accountPublicKey)
+                    throw new Error(`host returned a ${name} credential for another account`);
+                if (credsClaims(candidate[name]).sub !== remote.identities[name].id)
+                    throw new Error(`host returned a ${name} credential for a different nkey than the held identity`);
+                const staged = candidate[name];
+                const nc = await this.dial({ authenticator: (nonce) => credsAuthenticator(new TextEncoder().encode(staged))(nonce), maxReconnectAttempts: 0 });
+                await nc.close();
+            }
+            if (this.leaseStopping || !this.serviceServe || this.serviceServe.grant.epoch !== processEpoch)
+                throw new Error("manager changed its active serve epoch while standing renewal was in flight");
+            this.remoteSupervisorCreds = candidate.supervisor;
+            this.remoteExecutorCreds = candidate.executor;
+            this.serviceServe.creds = candidate.serve;
+            this.goalWriterCreds = candidate.goalWriter;
+            this.sessionLedgerCreds = candidate.sessionLedger;
+            if (this.goalWriter)
+                this.goalWriter.creds = candidate.goalWriter;
+            if (this.sessionLedgerConn)
+                this.sessionLedgerConn.creds = candidate.sessionLedger;
+            // Adoption is committed above; a connection that cannot come back yet is a push failure the
+            // endpoint's own reconnect loop retries with the adopted credential, not unadopted debt.
+            this.remoteRenewalDebt = undefined;
+            await Promise.all([
+                this.ep.reconnect().catch((e) => console.error(`! remote manager supervisor reconnect after renewal: ${e.message}`)),
+                this.pushRenewedCreds(this.serviceServe.nc),
+                ...(this.goalWriter ? [this.pushRenewedCreds(this.goalWriter.nc)] : []),
+                ...(this.sessionLedgerConn ? [this.pushRenewedCreds(this.sessionLedgerConn.nc)] : []),
+            ]);
+        }
+        catch (e) {
+            const unadoptedSubjects = candidate && typeof candidate === "object"
+                ? Object.values(candidate).flatMap((cred) => { try {
+                    return typeof cred === "string" && credsClaims(cred).sub ? [credsClaims(cred).sub] : [];
+                }
+                catch {
+                    return [];
+                } })
+                : [];
+            this.remoteRenewalDebt = { at: Date.now(), processEpoch, reason: e.message, unadoptedSubjects };
+            console.error(`! remote manager all-duty renewal: ${e.message} - no unsigned or local replacement is available`);
+            if (force)
+                throw e;
+        }
+    }
     /** Warn, on every renewal pass, when a $SYS credential is at or past its renewal point.
      *
      *  The manager is the renewal owner for every credential it CAN re-sign, and these two are the ones
@@ -2357,9 +2460,8 @@ export class Manager {
      *  admin tier allowed operator cross-owner redefine; (2) launch is owner-equality-only, above.
      *  Both are least-privilege reductions, never widenings. */
     /** The run host, or one of three refusals. A remote-authority manager holds no space signer, so
-     *  it cannot mint the per-run driver credential SPEC 14.6 requires, and hosting on any other
-     *  identity would be the fallback this tree does not take: `unimplemented`, for good. A
-     *  user-auth mesh is `unimplemented` too, for a different reason it names: a hosted run's seats
+     *  it cannot mint the per-run driver credential SPEC 14.6 requires without all four closed
+     *  host callbacks. A local user-auth mesh is `unimplemented`: a hosted run's seats
      *  are spawned, turned and despawned by a caller derived from the run id under the static
      *  owner, which the user-mode spawn door refuses (no `u_` owner), so a program would fail at
      *  its first seat; no path to `--local` is offered there since a user bearer holds no run rows
@@ -2369,9 +2471,9 @@ export class Manager {
     runHost() {
         if (this.runHosting)
             return this.runHosting;
-        if (this.remoteAuthority)
-            throw new EpEnvelopeError("unimplemented", "this manager does not host workflow runs: a remote-authority manager mints no run-driver credentials (SPEC 14.6); drive the run from a terminal with `cotal run start --local --file <program>`");
-        if (this.userMode)
+        if (this.remoteAuthority && !this.remoteAuthority.runHosting)
+            throw new EpEnvelopeError("unimplemented", "this manager does not host workflow runs: its issuing host supplied no closed run callbacks, and a remote-authority manager mints no run-driver credentials (SPEC 14.6); drive the run from a terminal with `cotal run start --local --file <program>`");
+        if (this.userMode && !this.remoteAuthority?.runHosting)
             throw new EpEnvelopeError("unimplemented", `user-auth space "${this.space}" hosts no workflow runs yet: a hosted run's seats would be spawned under the static owner, which a user mesh refuses; run programs on a static-auth mesh`);
         throw new EpEnvelopeError("unavailable", "the manager is still booting its workflow-run host; retry shortly (SPEC 14.3)");
     }
@@ -3101,14 +3203,14 @@ export class Manager {
         if (this.userMode) {
             const p = parsePrincipalKey(a.id);
             if (p)
-                this.retiring.set(a.name, { opId: retireOpId(a.lifecycleUid), lifecycleUid: a.lifecycleUid, owner: p.owner, actor: p.actor, agentId: a.id, userOwner: a.userOwner, secretPaths: a.secretPaths, startedAt: Date.now() });
+                this.retiring.set(a.name, { opId: retireOpId(a.lifecycleUid), lifecycleUid: a.lifecycleUid, owner: p.owner, actor: p.actor, agentId: a.id, userOwner: a.userOwner, secretPaths: a.secretPaths, launch: { allowSubscribe: a.launch.allowSubscribe }, startedAt: Date.now() });
         }
         else if (this.auth) {
             // Unit B: a STATIC lifecycle now also holds its name pending its own terminal (the F1
             // static retirement the detached deprovision below drives) — the alias frees only when the
             // gate+head terminal completes, exactly the user-mode discipline. The wire principal is the
             // incarnation-unique nkey (F5-bind); owner is the dev owner.
-            this.retiring.set(a.name, { opId: retireOpId(a.lifecycleUid), lifecycleUid: a.lifecycleUid, owner: DEV_OWNER, actor: a.id, agentId: a.id, secretPaths: a.secretPaths, runtime: a.handle?.reference, startedAt: Date.now() });
+            this.retiring.set(a.name, { opId: retireOpId(a.lifecycleUid), lifecycleUid: a.lifecycleUid, owner: DEV_OWNER, actor: a.id, agentId: a.id, secretPaths: a.secretPaths, launch: { allowSubscribe: a.launch.allowSubscribe }, runtime: a.handle?.reference, startedAt: Date.now() });
         }
         // Auth mode: tear down the departed agent's minted broker footprint + creds file (#159 B2). The
         // process is already gone, so this must never block the slot free or throw into the caller — it runs
@@ -3232,7 +3334,21 @@ export class Manager {
                 console.error(`revoke agent grant ${a.name}: ${e.message}`);
             }
         }
-        await this.deprovisionBroker(a);
+        try {
+            const res = await this.deprovisionBroker(a);
+            const h = this.retiring.get(a.name);
+            if (h && h.lifecycleUid === a.lifecycleUid)
+                h.lastResources = res;
+        }
+        catch (e) {
+            const h = this.retiring.get(a.name);
+            if (h && h.lifecycleUid === a.lifecycleUid) {
+                h.lastError = e.message;
+                if (e instanceof DeprovisionError)
+                    h.lastResources = e.accounting;
+            }
+            throw e;
+        }
         // #29 piece 3: after the footprint teardown, ask the AUTH plane to RETIRE the lifecycle over
         // the auth endpoint rail. The rail re-checks the SERVE-ISSUANCE GATE at serve time (not the
         // space-manager lease - that check was replaced in 02794b2f) and refuses unless the registration
@@ -3324,46 +3440,44 @@ export class Manager {
                 : await mintCreds(this.auth, requestIdentity, "retirement-requester", {
                     retirementRequester: { ...caller, target: retirementTarget },
                 });
-            const nc = await this.dial({ authenticator: credsAuthenticator(new TextEncoder().encode(creds)), maxReconnectAttempts: 0 });
+            // The requester grant subscribes `_INBOX_<id>.>` only (provision.ts): the endpoint path's
+            // JetStream API requests (the contract-store read inside resolveService) must reply under it, not under the client default `_INBOX.`.
+            const nc = await this.dial({ authenticator: credsAuthenticator(new TextEncoder().encode(creds)), inboxPrefix: `_INBOX_${requestIdentity.id}`, maxReconnectAttempts: 0 });
             try {
-                // §13.2 nonce: >=128 bits of CSPRNG entropy, base64url (the `endpoint-invoke` idiom).
-                const nonce = randomBytes(24).toString("base64url");
-                // The caller-chosen request id the reply MUST echo (the minimal correctness guard from the
-                // not-yet-migrated endpoint envelope - see the residual in the auth listener).
-                const requestId = randomBytes(16).toString("base64url");
-                const subject = epRequestSubject(this.space, {
-                    route: { mode: "one" },
-                    endpoint: AUTH_ENDPOINT,
-                    command: EP_CMD_RETIRE_LIFECYCLE,
-                    // The TARGET rides the subject (authz mode `handle`, arity 3) — broker-enforced, and
-                    // pinned by the grant minted above, so it is not a body claim the caller can vary.
-                    target: { mode: "handle", tOwner: target.owner, tActor: target.actor, tUid: a.lifecycleUid },
-                    caller,
-                    nonce,
+                // The service resolves FRESH on every attempt (never cached on `this`): a plane restart
+                // mid-despawn then lands on the new epoch through an ordinary resolve, rather than a
+                // caller pinned to a now-superseded incarnation.
+                const service = await resolveService(nc, this.space, AUTH_ENDPOINT, caller, { deadlineMs: 10_000 });
+                // The TARGET rides the invoke's own target (authz mode `exact`, arity 3) — broker-enforced,
+                // and pinned by the grant minted above, so it is not a body claim the caller can vary.
+                const attributed = await invokeCommand(nc, this.space, service, EP_CMD_RETIRE_LIFECYCLE, {
+                    opId,
+                    serveEndpoint: MANAGER_ENDPOINT, serveInstanceId: this.managerInstanceId, serveEpoch,
+                }, {
+                    target: { mode: "exact", owner: target.owner, actor: target.actor, lifecycleUid: a.lifecycleUid },
+                    deadlineMs: 20_000,
                 });
-                // The responder derives its OWN reply subject from this request (caller triple + nonce,
-                // prefixed with the RESPONDER's instance identity), so a caller-supplied `reply` header
-                // would be ignored — that is the confused-deputy boundary being structural. The caller
-                // therefore reads its own reply-plane filter and binds the answer off the reply SUBJECT.
-                const m = await epAwaitReply(nc, this.space, caller, nonce, requestId, subject, 
-                // Declare THIS manager instance's serve identity. Since #350 these SELECT the gate row;
-                // they no longer authorize — the rail refuses unless the row they name belongs to this
-                // caller's own subject-derived principal. A superseded predecessor (same instanceId, OLD
-                // epoch after a restart) is still refused by the epoch comparison.
-                JSON.stringify({ id: requestId, op: "retireLifecycle", args: {
-                        opId,
-                        serveEndpoint: MANAGER_ENDPOINT, serveInstanceId: this.managerInstanceId, serveEpoch,
-                    } }), 20_000);
-                const r = m;
+                const r = attributed.reply;
                 if (r.ok) {
                     this.confirmRetirement(a);
                 }
+                else if (r.error?.code === "expired") {
+                    // A stale-epoch refusal: the responder answered, but at another epoch than this resolve
+                    // bound (a plane restart mid-despawn landed on the new epoch) - the retry copy, never a
+                    // hard despawn failure line, since a same-name spawn (or the auth service's own boot
+                    // resume) re-drives the same teardown.
+                    const copy = uncertain(r.error.message);
+                    if (held)
+                        held.lastError = copy;
+                    console.error(`despawn ${a.name}: ${copy}`);
+                }
                 else {
                     // The rail's refusal is already the operator copy (lease-loss/stale/foreign-op faces,
                     // full-no-op statements included) - surface it INTACT, never flattened.
+                    const message = r.error?.message ?? "the auth service refused the retirement without a reason";
                     if (held)
-                        held.lastError = r.error ?? "the auth service refused the retirement without a reason";
-                    console.error(`despawn ${a.name}: ${r.error ?? "the auth service refused the retirement without a reason"}`);
+                        held.lastError = message;
+                    console.error(`despawn ${a.name}: ${message}`);
                 }
             }
             finally {
@@ -3414,13 +3528,59 @@ export class Manager {
         // LIFECYCLE-PINNED (SPEC 13.1): both the credential's exact-name grants and the delete names
         // carry a.lifecycleUid, so a stale/replayed teardown for this retired incarnation is broker-denied
         // against a same-name successor's footprint (its names embed a different uid).
+        // Durable membership rows are lifecycle-keyed per concrete channel. Start with the launch's
+        // concrete read channels, then include wildcard-covered and unnamed channels from the inventory.
+        const memberChannels = new Set((a.launch?.allowSubscribe ?? []).filter(isConcreteChannel));
+        // Complete the set from the delivery daemon's lifecycle-exact inventory, which covers rows on
+        // wildcard-granted channels and teardowns that carry no launch spec. An inventory that cannot be
+        // read leaves those rows RETAINED and says so; it never widens the delete beyond exact keys.
+        const inventory = await this.lifecycleMembershipInventory(a);
+        for (const ch of inventory.channels)
+            memberChannels.add(ch);
+        if (!inventory.complete)
+            console.error(`deprovision ${a.name} (${a.id}): membership inventory incomplete (${inventory.reason}); rows on channels outside its launch policy stay RETAINED for uid ${a.lifecycleUid}`);
         const creds = await mintCreds(this.auth, newIdentity(), "deprovisioner", {
-            deprovisionTarget: { principal: a.id, lifecycleUid: a.lifecycleUid },
+            deprovisionTarget: { principal: a.id, lifecycleUid: a.lifecycleUid, memberChannels: [...memberChannels] },
         });
         // Bound the detached broker teardown so a wedged broker can't leave the deprovision promise pending
         // forever with no log — the timeout rejects into freeSlot's fail-loud `.catch` (paired with the
         // helper's own fail-fast connect). The durables/ACL row still fall to space teardown as a backstop.
-        await withTimeout(deprovisionAgent({ servers: this.servers ?? DEFAULT_SERVER, space: this.space, targetId: a.id, lifecycleUid: a.lifecycleUid, creds }), DEPROVISION_TIMEOUT_MS, `deprovision ${a.name} (${a.id}): broker teardown timed out`);
+        let accounting;
+        try {
+            accounting = await withTimeout(deprovisionAgent({
+                servers: this.servers ?? DEFAULT_SERVER,
+                space: this.space,
+                targetId: a.id,
+                lifecycleUid: a.lifecycleUid,
+                memberChannels: [...memberChannels],
+                creds,
+            }), DEPROVISION_TIMEOUT_MS, `deprovision ${a.name} (${a.id}): broker teardown timed out`);
+            this.lastDeprovisionResources = accounting;
+        }
+        catch (e) {
+            if (e instanceof DeprovisionError) {
+                this.lastDeprovisionResources = e.accounting;
+            }
+            throw e;
+        }
+        if (!inventory.complete)
+            throw new Error(`membership inventory incomplete (${inventory.reason}); rows on channels outside its launch policy stay RETAINED for uid ${a.lifecycleUid}`);
+        return accounting;
+    }
+    /** The retiring lifecycle's durable membership channels, read from the delivery daemon's
+     *  `lifecycleMemberships` admin verb (read-only, lifecycle-exact). Incomplete on any refusal. */
+    async lifecycleMembershipInventory(a) {
+        const t = deprovisionTargetPrincipal({ principal: a.id, lifecycleUid: a.lifecycleUid });
+        try {
+            const r = await this.ep.requestDeliveryAdmin("lifecycleMemberships", { principal: principalKey(t.owner, t.actor).key, lifecycleUid: t.lifecycleUid }, 5_000);
+            const data = r.data;
+            if (!r.ok || data?.complete !== true || !Array.isArray(data.channels) || !data.channels.every((c) => typeof c === "string" && isConcreteChannel(c)))
+                return { complete: false, channels: [], reason: r.ok ? "malformed inventory reply" : (r.error ?? "refused") };
+            return { complete: true, channels: data.channels };
+        }
+        catch (e) {
+            return { complete: false, channels: [], reason: e.message };
+        }
     }
     /** Reap a parent's children on its exit (P4b). Every descendant remains managed until the runtime's
      * authoritative wait proves exit; the wait participates in the lifecycle drain, so preservation can
@@ -4328,7 +4488,7 @@ export class Manager {
             return { ok: false, error: `the name "${identityName}" is still reconciling at manager startup; its prior lifecycle terminal owns this alias until it completes. Retry shortly.` };
         const held = this.retiring.get(identityName);
         if (held !== undefined) {
-            void this.deprovision({ id: held.agentId, name: identityName, lifecycleUid: held.lifecycleUid, userOwner: held.userOwner, secretPaths: held.secretPaths }).catch(() => { });
+            void this.deprovision({ id: held.agentId, name: identityName, lifecycleUid: held.lifecycleUid, userOwner: held.userOwner, secretPaths: held.secretPaths, launch: held.launch }).catch(() => { });
             const err = lifecycleBlocked("failed-precondition", `the name "${identityName}" is reserved pending retirement: its previous agent's despawn started that lifecycle's teardown (footprint + standing-authority revoke + auth-side retirement), and the name frees only when all of it completes${held.lastError !== undefined ? ` (last attempt: ${held.lastError})` : ""}. NEXT: wait a moment and retry this spawn (retrying re-drives the whole teardown), or pick another name.`, { blockedOp: "retirement", headState: "retiring", opId: held.opId, remedy: "retry" });
             return { ok: false, error: renderLifecycleBlocked(err.message, err), details: err.details };
         }
@@ -5623,8 +5783,14 @@ export class Manager {
      *  This is NEVER the manager's standing seed/supervisor connection. */
     async withEndpointServeExecutor(fn) {
         const identity = this.remoteAuthority?.identities.executor ?? newIdentity();
-        if (this.remoteAuthority && (!this.remoteExecutorCreds || inspectCredHealth(this.remoteExecutorCreds).state !== "healthy"))
-            await this.renewRemoteExecutor(true);
+        if (this.remoteAuthority && (!this.remoteExecutorCreds || inspectCredHealth(this.remoteExecutorCreds).state !== "healthy")) {
+            // Shutdown has drained and discarded the standing serve context. Renew only its scoped
+            // maintenance executor through the existing host operation; never resurrect standing duties.
+            if (this.leaseStopping)
+                await this.renewRemoteExecutor(true);
+            else
+                await (this.remoteAuthority.renewStandingBundle ? this.renewRemoteStandingBundle(true) : this.renewRemoteExecutor(true));
+        }
         const creds = this.remoteExecutorCreds ?? (this.auth
             ? await mintCreds(this.auth, identity, "endpoint-serve-executor", {
                 endpointServeExecutor: { endpoint: MANAGER_ENDPOINT, instanceId: this.managerInstanceId },
@@ -5755,7 +5921,6 @@ export class Manager {
             inboxPrefix: `_INBOX_${state.identity.id}`,
             maxReconnectAttempts: -1,
         });
-        nc.closed().then((err) => { void this.onServeConnectionClosed(err ?? undefined); });
         let handle;
         try {
             // The 1b typed surface + the derived `describe`. The descriptor stays PUBLIC in static
@@ -5770,6 +5935,13 @@ export class Manager {
                 // durable slot rows mirror it). Static mode carries no mapping-revision dimension, so
                 // the revision is the constant 0 — a caller that pins a revision pins 0.
                 resolveTarget: (t) => {
+                    if (this.remoteAuthority) {
+                        const key = principalKey(t.owner, t.actor).key;
+                        for (const a of this.agents.values())
+                            if (a.id === key)
+                                return { lifecycleUid: a.lifecycleUid, mappingRevision: 0 };
+                        return undefined;
+                    }
                     if (t.owner === DEV_OWNER) {
                         for (const a of this.agents.values())
                             if (!a.userOwner && a.id === t.actor)
@@ -5789,6 +5961,9 @@ export class Manager {
             await nc.drain().catch(() => nc.close());
             throw e;
         }
+        // Wired only once the connection serves: a connection this method drains itself after a
+        // refused serve is not a fault to recover, and the caller already sees the refusal.
+        nc.closed().then((err) => { void this.onServeConnectionClosed(err ?? undefined); });
         return { nc, handle };
     }
     /** #2073: re-mint the SAME `endpoint-serve` identity through the SAME scoped one-shot executor and
@@ -5823,7 +5998,9 @@ export class Manager {
         const recovered = await this.boundedReconnect("service endpoint", async () => {
             if (!s)
                 return false;
-            if (inspectCredHealth(s.creds ?? "").state !== "healthy" && s.creds !== undefined) {
+            if (this.remoteAuthority)
+                await this.renewRemoteBeforeRedial("serve", () => s.creds);
+            else if (inspectCredHealth(s.creds ?? "").state !== "healthy" && s.creds !== undefined) {
                 s.creds = await this.remintServeCreds(s);
                 this.scheduleServeRenewal(s.creds);
             }
@@ -5867,28 +6044,9 @@ export class Manager {
                 grant: remote.serveGrant,
                 creds: remote.serveCreds,
             };
-            const enc = new TextEncoder();
-            const nc = await this.dial({
-                authenticator: (nonce) => credsAuthenticator(enc.encode(state.creds))(nonce),
-                inboxPrefix: `_INBOX_${state.identity.id}`,
-                maxReconnectAttempts: -1,
-            });
-            try {
-                state.handle = serveEndpoint(nc, this.space, state.grant, this.managerServiceDefs(), { public: true }, {
-                    resolveTarget: (t) => {
-                        const key = principalKey(t.owner, t.actor).key;
-                        for (const a of this.agents.values())
-                            if (a.id === key)
-                                return { lifecycleUid: a.lifecycleUid, mappingRevision: 0 };
-                        return undefined;
-                    },
-                    ...(this.omitClassCommands() ? { omitClassCommands: this.omitClassCommands() } : {}),
-                });
-            }
-            catch (e) {
-                await nc.drain().catch(() => nc.close());
-                throw e;
-            }
+            // The shared dialer wires the closed-connection handler, which re-dials through the all-duty family.
+            const { nc, handle } = await this.dialServeConnection(state);
+            state.handle = handle;
             state.nc = nc;
             this.serviceServe = state;
             console.error(`remote manager service endpoint activated: ${MANAGER_ENDPOINT}/${this.managerInstanceId} (epoch ${state.grant.epoch})`);
@@ -6198,7 +6356,9 @@ export class Manager {
         const recovered = await this.boundedReconnect("goal-writer", async () => {
             if (!gw)
                 return false;
-            if (gw.creds !== undefined && inspectCredHealth(gw.creds).state !== "healthy") {
+            if (this.remoteAuthority)
+                await this.renewRemoteBeforeRedial("goal-writer", () => gw.creds);
+            else if (gw.creds !== undefined && inspectCredHealth(gw.creds).state !== "healthy") {
                 gw.creds = await this.withEndpointServeExecutor(({ authKv }) => this.mintAndStageGoalWriter(authKv));
                 this.goalWriterCreds = gw.creds;
             }
@@ -6313,7 +6473,9 @@ export class Manager {
         const recovered = await this.boundedReconnect("session-ledger", async () => {
             if (!sw || !identity)
                 return false;
-            if (sw.creds !== undefined && inspectCredHealth(sw.creds).state !== "healthy") {
+            if (this.remoteAuthority)
+                await this.renewRemoteBeforeRedial("session-ledger", () => sw.creds);
+            else if (sw.creds !== undefined && inspectCredHealth(sw.creds).state !== "healthy") {
                 sw.creds = await this.withEndpointServeExecutor(({ authKv }) => this.mintAndStageSessionLedger(authKv));
                 this.sessionLedgerCreds = sw.creds;
             }
@@ -7921,6 +8083,10 @@ export class Manager {
             finally {
                 await nc.drain().catch(() => nc.close());
             }
+            let retiredSlots = 0;
+            let preservedForeign = 0;
+            let preservedAdopted = 0;
+            let deferredSlots = 0;
             const terminalRows = [];
             for (const row of slotRows) {
                 if (row.phase === "retired") {
@@ -7928,6 +8094,7 @@ export class Manager {
                     // regardless of which instance owned it, so a sibling-retired incarnation's copied credential
                     // is refused at this control surface too. Ownership gates only the DESTRUCTIVE sweep below.
                     this.retiredPrincipals.add(principalKey(row.owner, row.actor).key);
+                    retiredSlots++;
                     continue;
                 }
                 // 3b-2 RECONCILE OWNERSHIP (multi-manager-per-space): a manager adjudicates ONLY the non-retired
@@ -7936,19 +8103,26 @@ export class Manager {
                 // hazard, now cross-instance). A legacy row (pre-3b-2, no owner recorded) predates multi-manager,
                 // so this manager is its legitimate single-manager-past successor and reconciles it. An orphaned
                 // sibling row is reclaimed only by an explicit operator CAS takeover (ruling 1), never here.
-                if (row.ownerInstanceId !== undefined && row.ownerInstanceId !== this.managerInstanceId)
+                if (row.ownerInstanceId !== undefined && row.ownerInstanceId !== this.managerInstanceId) {
+                    preservedForeign++;
                     continue;
+                }
                 // ADOPTION is genuine membership: a slot backed by a live managed agent THIS process owns
                 // at the SAME uid is never an orphan (empty at boot; exactly the adopted set at the
                 // post-adoption sweep — the fix for the F3 resume hole).
                 const live = this.agents.get(row.alias);
                 const adopted = live !== undefined && live.lifecycleUid === row.lifecycleUid;
+                if (adopted) {
+                    preservedAdopted++;
+                }
                 // Boot sweep with a resume PENDING: an active slot may yet be adopted (the resume path runs
                 // AFTER this boot sweep), so DEFER it — the post-adoption sweep terminalizes any the resume
                 // did not claim. provisioning/terminalizing NEVER defer (they are crashed operations, never
                 // an agent to adopt). At `postAdoption` (or a non-resume boot) nothing defers.
-                if (!postAdoption && row.phase === "active" && !adopted && this.resumeRequired)
+                if (!postAdoption && row.phase === "active" && !adopted && this.resumeRequired) {
+                    deferredSlots++;
                     continue;
+                }
                 if (planStaticSlotResume(row, adopted) !== "none")
                     terminalRows.push(row);
             }
@@ -7957,6 +8131,17 @@ export class Manager {
                 // alias, but the initial serial sweep must close every discovery-to-await gap up front.
                 for (const row of terminalRows)
                     this.reconcilingAliases.add(row.alias);
+                const sweepDeprovision = {
+                    examined: 0,
+                    deleted: 0,
+                    absent: 0,
+                    refused: 0,
+                    acknowledged: 0,
+                    disappeared: 0,
+                    consumers: { examined: 0, deleted: 0, absent: 0, refused: 0, acknowledged: 0, disappeared: 0 },
+                    acls: { examined: 0, deleted: 0, absent: 0, refused: 0, disappeared: 0 },
+                    members: { examined: 0, deleted: 0, absent: 0, refused: 0, disappeared: 0 },
+                };
                 for (const row of terminalRows) {
                     if (this.staticReconcileStopping)
                         break;
@@ -7976,7 +8161,33 @@ export class Manager {
                         };
                         this.staticReconcileItems.set(key, item);
                     }
+                    this.lastDeprovisionResources = undefined;
                     const succeeded = await this.attemptStaticReconcile(key, item, row);
+                    const acc = this.lastDeprovisionResources;
+                    if (acc) {
+                        sweepDeprovision.examined += acc.examined;
+                        sweepDeprovision.deleted = sweepDeprovision.deleted === null || acc.deleted === null ? null : sweepDeprovision.deleted + acc.deleted;
+                        sweepDeprovision.absent += acc.absent;
+                        sweepDeprovision.refused += acc.refused;
+                        sweepDeprovision.acknowledged += acc.acknowledged;
+                        sweepDeprovision.disappeared += acc.disappeared;
+                        sweepDeprovision.consumers.examined += acc.consumers.examined;
+                        sweepDeprovision.consumers.deleted = sweepDeprovision.consumers.deleted === null || acc.consumers.deleted === null ? null : sweepDeprovision.consumers.deleted + acc.consumers.deleted;
+                        sweepDeprovision.consumers.absent += acc.consumers.absent;
+                        sweepDeprovision.consumers.refused += acc.consumers.refused;
+                        sweepDeprovision.consumers.acknowledged += acc.consumers.acknowledged;
+                        sweepDeprovision.consumers.disappeared += acc.consumers.disappeared;
+                        sweepDeprovision.acls.examined += acc.acls.examined;
+                        sweepDeprovision.acls.deleted += acc.acls.deleted;
+                        sweepDeprovision.acls.absent += acc.acls.absent;
+                        sweepDeprovision.acls.refused += acc.acls.refused;
+                        sweepDeprovision.acls.disappeared += acc.acls.disappeared;
+                        sweepDeprovision.members.examined += acc.members.examined;
+                        sweepDeprovision.members.deleted += acc.members.deleted;
+                        sweepDeprovision.members.absent += acc.members.absent;
+                        sweepDeprovision.members.refused += acc.members.refused;
+                        sweepDeprovision.members.disappeared += acc.members.disappeared;
+                    }
                     if (succeeded)
                         sweep.succeeded++;
                     else
@@ -7984,6 +8195,18 @@ export class Manager {
                     this.reconcilingAliases.delete(row.alias);
                 }
                 sweep.completedAt = new Date().toISOString();
+                this.staticReconcileLastSweepResources = {
+                    slots: {
+                        examined: slotRows.length,
+                        retired: retiredSlots,
+                        preservedForeign,
+                        preservedAdopted,
+                        deferred: deferredSlots,
+                        terminalized: sweep.succeeded,
+                        failed: sweep.failed,
+                    },
+                    deprovision: sweepDeprovision,
+                };
                 for (const row of terminalRows) {
                     const key = this.staticReconcileKey(row);
                     const item = this.staticReconcileItems.get(key);