@cotal-ai/auth 0.71.0 → 0.72.1

package/dist/service.js

@@ -64,6 +64,7 @@ import { findCotalRoot, loadManagerInstanceIdentity, userAuthStateDir, workspace
 import { decodeJwt } from "jose";
 import { deriveOwnerForIdpSubject, platformControlOwner } from "./derive.js";
 import { makePlatformControlAuthority, makePlatformControlReadiness, requireManagerAuthorityHolder } from "./platform-control.js";
+import { OwnedConnections } from "./owned-connections.js";
 import { startAuthCallout } from "./callout.js";
 import { createIdpBridge, verifyIdpToken } from "./idp.js";
 import { PUBLIC_EXCHANGE_VIEWS, assertTransferWriterClaim } from "./token.js";
@@ -199,11 +200,13 @@ async function registerSelfAuthorizedInstance(nc, args, register) {
  * transit the auth callout — the callout cannot deadlock on its own reader.
  */
 export async function openAuthAuthorityPlane(opts) {
-    // Construction records each resource's close here as soon as it opens, so a throw anywhere up to
-    // the returned plane closes exactly what is open, in reverse order.
+    return await openAuthAuthorityPlaneWithCustody(opts);
+}
+/** The context's custody callback is private. Failed construction unwinds from one place. */
+async function openAuthAuthorityPlaneWithCustody(opts, onConnection) {
     const unwind = [];
     try {
-        return await buildAuthAuthorityPlane(opts, unwind);
+        return await buildAuthAuthorityPlane(opts, unwind, onConnection);
     }
     catch (e) {
         for (const close of unwind.reverse())
@@ -211,14 +214,14 @@ export async function openAuthAuthorityPlane(opts) {
         throw e;
     }
 }
-/** The body of {@link openAuthAuthorityPlane}, which owns the unwind of a failed construction. */
-async function buildAuthAuthorityPlane(opts, unwind) {
+/** The body of the plane opener, which owns the unwind of a failed construction. */
+async function buildAuthAuthorityPlane(opts, unwind, onConnection) {
     const { server, space, dataAccount, log } = opts;
     const standingTtl = opts.standingRenewableTtlSeconds ?? STANDING_RENEWABLE_TTL_SEC;
     if (!Number.isSafeInteger(standingTtl) || standingTtl < 5 || standingTtl > STANDING_RENEWABLE_TTL_SEC) {
         throw new Error(`standingRenewableTtlSeconds must be an integer between 5 and ${STANDING_RENEWABLE_TTL_SEC} seconds (got ${standingTtl})`);
     }
-    const writer = await openAuthorityClient({ server, space, dataAccount, label: `cotal:auth-mint:${space}`, grants: (id) => authorityWriterGrants(space, id), log });
+    const writer = await openAuthorityClient({ onConnection, server, space, dataAccount, label: `cotal:auth-mint:${space}`, grants: (id) => authorityWriterGrants(space, id), log });
     unwind.push(() => writer.close());
     await ensureAuthorityStores(await jetstreamManager(writer.nc), new Kvm(writer.nc), space);
     const registry = await openLifecycleRegistry(writer.nc, space);
@@ -239,7 +242,7 @@ async function buildAuthAuthorityPlane(opts, unwind) {
         const iid = authIdentity.instanceId;
         const artifacts = authClusterArtifacts();
         const values = [...authContractArtifactValues(), artifacts.document, artifacts.manifest];
-        const regClient = await openAuthorityClient({
+        const regClient = await openAuthorityClient({ onConnection,
             server, space, dataAccount, label: `cotal:auth-registration:${space}`,
             grants: (id) => registrationExecutorGrants(space, id, AUTH_SERVICE_ENDPOINT, iid),
             log,
@@ -257,13 +260,13 @@ async function buildAuthAuthorityPlane(opts, unwind) {
             await regClient.close();
         }
     }
-    const reader = await openSupervisedConnectReader({ server, space, dataAccount, log });
+    const reader = await openSupervisedConnectReader({ onConnection, server, space, dataAccount, log });
     unwind.push(() => reader.close());
     // The typed remote-manager issuer is a distinct self-minted connection. It adds only the
     // endpoint-manager gate/credential family to the root issuance surface and is never exposed as
     // a generic mint endpoint. Its JWTs are signed for caller-generated public nkeys; private seeds
     // stay on the participant machine.
-    const remoteIssuer = await openAuthorityClient({ server, space, dataAccount, label: `cotal:remote-manager-issuer:${space}`, grants: (id) => remoteManagerIssuerGrants(space, id), log });
+    const remoteIssuer = await openAuthorityClient({ onConnection, server, space, dataAccount, label: `cotal:remote-manager-issuer:${space}`, grants: (id) => remoteManagerIssuerGrants(space, id), log });
     unwind.push(() => remoteIssuer.close());
     // The BARRIER EXECUTOR: the third self-minted connection, with its own registry bind — the
     // mint writer stays the minimal issuance credential ("barriers are NOT this credential's
@@ -295,17 +298,17 @@ async function buildAuthAuthorityPlane(opts, unwind) {
     let fatalReason;
     let fireFatal;
     const fenced = new Promise((r) => { fireFatal = r; });
-    const barrier = await openAuthorityClient({ server, space, dataAccount, label: `cotal:auth-barrier:${space}`, grants: (id) => authorityBarrierGrants(space, id), log });
+    const barrier = await openAuthorityClient({ onConnection, server, space, dataAccount, label: `cotal:auth-barrier:${space}`, grants: (id) => authorityBarrierGrants(space, id), log });
     unwind.push(() => barrier.close());
     // Unwound after both scanners close: held → released is valid only once neither scanner can act.
     // A failed open throws its own error, so a release failing on the way out is only logged: the
     // row stays held and the next open reclaims it like a crash.
     unwind.push(async () => hold?.release().catch((r) => log(r.message)));
-    const ledgerCand = await openAuthLedgerScannerCandidate({ server, space, dataAccount, log });
+    const ledgerCand = await openAuthLedgerScannerCandidate({ onConnection, server, space, dataAccount, log });
     unwind.push(() => ledgerCand.close());
-    const recordsCand = await openRecordsScannerCandidate({ server, space, dataAccount, log });
+    const recordsCand = await openRecordsScannerCandidate({ onConnection, server, space, dataAccount, log });
     unwind.push(() => recordsCand.close());
-    const oracle = opts.probePlaneOracle ?? makeDeliveryAdminPlaneOracle({ space, server, dataAccount, log });
+    const oracle = opts.probePlaneOracle ?? makeDeliveryAdminPlaneOracle({ onConnection, space, server, dataAccount, log });
     hold = await acquirePlaneClaim({ nc: barrier.nc, space, ledger: ledgerCand.tuple, records: recordsCand.tuple, oracle, log });
     const scanner = ledgerCand.activate(hold.guard);
     const recordsScanner = recordsCand.activate(hold.guard);
@@ -328,7 +331,7 @@ async function buildAuthAuthorityPlane(opts, unwind) {
     unwind.push(async () => { closing = true; });
     opts.probePlaneDeath?.({ ledger: () => ledgerCand.close(), records: () => recordsCand.close() });
     const barrierReg = await openLifecycleRegistry(barrier.nc, space, scanner, recordsScanner);
-    const evictPrincipal = opts.probeEvictor ?? makeDeliveryAdminEvictor({ space, server, dataAccount, log });
+    const evictPrincipal = opts.probeEvictor ?? makeDeliveryAdminEvictor({ onConnection, space, server, dataAccount, log });
     // The RETIREMENT deps (#29 piece 4): the barrier's injected mechanics, assembled from reviewed
     // §13.9 profiles only. The obligation drain runs per endpoint on a SHORT-LIVED client minted
     // with that endpoint's admission-mediator profile (the daemon owns no standing mediator; the
@@ -351,11 +354,11 @@ async function buildAuthAuthorityPlane(opts, unwind) {
     //     effect gets the EffectCancelledFact union member. Never a forged success: a reader sees
     //     the effect did NOT run and which retirement cancelled it; a racing real completion wins
     //     by landing first.
-    const repairers = makeDrainRepairers({ server, space, dataAccount, log });
+    const repairers = makeDrainRepairers({ onConnection, server, space, dataAccount, log });
     const retirement = {
         evictPrincipal,
         drainTargetObligations: async (endpoint, targetUid, opId) => {
-            const drain = await openAuthorityClient({
+            const drain = await openAuthorityClient({ onConnection,
                 server, space, dataAccount,
                 label: `cotal:ep-drain:${space}:${endpoint}`,
                 grants: (id) => admissionMediatorGrants(space, endpoint, id),
@@ -372,7 +375,7 @@ async function buildAuthAuthorityPlane(opts, unwind) {
                 await drain.close();
             }
         },
-        ...makeRetirementCleaners({ server, space, dataAccount, log }),
+        ...makeRetirementCleaners({ onConnection, server, space, dataAccount, log }),
         now: Date.now,
     };
     // Boot crash-resume BEFORE the plane answers anything (SPEC 13.1): finish every takeover this
@@ -389,7 +392,7 @@ async function buildAuthAuthorityPlane(opts, unwind) {
     // stays with the plane's own registry + retirement deps (the drain rides the ONE sealed records
     // scanner exactly like the boot resume); the listener only authorizes (subject attribution +
     // the FRESH space-manager-lease holder check) and dispatches.
-    const authAdmin = await openAuthAdminListener({ server, space, dataAccount, reg: barrierReg, retirement, barrierFlight: retirementFlights, instanceId: authServeInstanceId, epoch: authServeGrant.epoch, grant: authServeGrant, log });
+    const authAdmin = await openAuthAdminListener({ onConnection, server, space, dataAccount, reg: barrierReg, retirement, barrierFlight: retirementFlights, instanceId: authServeInstanceId, epoch: authServeGrant.epoch, grant: authServeGrant, log });
     unwind.push(() => authAdmin.close());
     const fileArm = ledgerAuthorizeConnect(opts.dir);
     const recordsJsm = await jetstreamManager(remoteIssuer.nc);
@@ -495,7 +498,7 @@ async function buildAuthAuthorityPlane(opts, unwind) {
     // window, not the operator — the agent cannot "restart the auth service", so it gets a retryable
     // unavailability, while the operator's state-3 copy stays on the log and the exit line.
     const refuseIfFenced = () => {
-        if (fatalReason !== undefined)
+        if (fatalReason !== undefined || closing)
             throw new EpEnvelopeError("unavailable", `the auth service for space "${space}" is momentarily unavailable (it detected a fault and is restarting); retry shortly`);
     };
     // The plane holds only the data account's signing key. The operator and system halves stay blank
@@ -894,12 +897,12 @@ async function buildAuthAuthorityPlane(opts, unwind) {
                 scanner,
                 observeManagerGate,
             });
-            const evictHolders = makeDeliveryAdminHolderEvictor({ space, server, dataAccount, log });
+            const evictHolders = makeDeliveryAdminHolderEvictor({ onConnection, space, server, dataAccount, log });
             if (authorized.operation === "evict-family-principal") {
                 const evictions = await evictHolders(authorized.principals);
                 return completeRemoteManagerMaintenance(authorized, owner, { evictions });
             }
-            const principalOracle = makeDeliveryAdminPrincipalOracle({ space, server, dataAccount, log });
+            const principalOracle = makeDeliveryAdminPrincipalOracle({ onConnection, space, server, dataAccount, log });
             const executorIdentity = newIdentity();
             const executorCreds = await mintCreds(issuerAuth(), executorIdentity, "endpoint-serve-executor", {
                 endpointServeExecutor: { endpoint: "manager", instanceId: authorized.targetInstanceId },
@@ -910,6 +913,7 @@ async function buildAuthAuthorityPlane(opts, unwind) {
                 ...standaloneConnectOpts({ creds: executorCreds, tls: false }),
                 maxReconnectAttempts: 0,
             });
+            onConnection?.(maintenanceNc, `cotal:remote-manager-maintenance:${space}`);
             let report;
             try {
                 const maintenanceKvm = new Kvm(maintenanceNc);
@@ -999,6 +1003,7 @@ async function buildAuthAuthorityPlane(opts, unwind) {
                     const provisioner = new CotalEndpoint({
                         space,
                         servers: server,
+                        onConnection: (nc) => onConnection?.(nc, `cotal:provisioner:${space}`),
                         channels: [],
                         creds: await mintCreds(issuerAuth(), identity, "provisioner"),
                         card: { id: identity.id, name: "provisioner", role: "provisioner", kind: "endpoint" },
@@ -1126,6 +1131,7 @@ async function buildAuthAuthorityPlane(opts, unwind) {
                     }),
                     maxReconnectAttempts: 0,
                 });
+                onConnection?.(admitterNc, `cotal:run-admitter:${space}`);
                 try {
                     return await admitRemoteRun({
                         request, owner, space, accountPublicKey: dataAccount.pub, proofSecret: dataAccount.signingSeed, endpoint: "manager",
@@ -1180,6 +1186,7 @@ async function buildAuthAuthorityPlane(opts, unwind) {
                         }),
                         maxReconnectAttempts: 0,
                     });
+                    onConnection?.(readerNc, `cotal:run-attempt-reader:${space}`);
                     try {
                         const replay = await replayRunJournal(jetstream(readerNc), await jetstreamManager(readerNc), space, runId, takeoverId);
                         return replay.records.flatMap((stored) => stored.record.kind === "step" ? [stored.record.entry] : []);
@@ -1260,7 +1267,7 @@ async function buildAuthAuthorityPlane(opts, unwind) {
         },
         registerHostInstance: async (host, instanceId) => {
             refuseIfFenced();
-            const regClient = await openAuthorityClient({
+            const regClient = await openAuthorityClient({ onConnection,
                 server, space, dataAccount, label: `cotal:host-registration:${space}`,
                 grants: (id) => registrationExecutorGrants(space, id, host.endpoint, instanceId),
                 log,
@@ -1286,29 +1293,42 @@ async function buildAuthAuthorityPlane(opts, unwind) {
             // Clean-close order (SPEC 13.13): the rail stops answering first, then scan-capable
             // clients down, then `held → released` (never released while either scanner can still
             // act), then the barrier that wrote it. An unreachable release keeps the barrier alive
-            // for a later close retry; the scanner teardown is not run a second time.
+            // for a later close retry; successful scanner teardown is not run a second time.
             closing = true;
-            await (closeScanners ??= (async () => {
-                await authAdmin.close();
-                await reader.close();
-                await recordsScanner.close();
-                await scanner.close();
-            })());
-            let retryable = false;
+            const scannerFlight = closeScanners ??= (async () => {
+                await authAdmin?.close();
+                const results = await Promise.allSettled([reader.close(), recordsScanner.close(), scanner.close()]);
+                const errors = results.filter((r) => r.status === "rejected").map((r) => r.reason);
+                if (errors.length)
+                    throw new AggregateError(errors, "auth scan-capable clients failed to close");
+            })();
             try {
-                await hold.release();
+                await scannerFlight;
             }
             catch (e) {
-                retryable = planeClaimRefusal(e) === "release-unreachable";
+                if (closeScanners === scannerFlight)
+                    closeScanners = undefined;
                 throw e;
             }
-            finally {
-                if (!retryable) {
-                    await barrier.close();
-                    await remoteIssuer.close();
-                    await writer.close();
-                }
+            let releaseFailure;
+            try {
+                await hold.release();
+            }
+            catch (e) {
+                releaseFailure = e;
             }
+            // Keep the barrier for an unreachable release retry. All other release refusals terminate
+            // the remaining clients but retain the release failure, even if their closes fail too.
+            if (planeClaimRefusal(releaseFailure) !== "release-unreachable") {
+                const results = await Promise.allSettled([barrier.close(), remoteIssuer.close(), writer.close()]);
+                const errors = results.filter((r) => r.status === "rejected").map((r) => r.reason);
+                if (releaseFailure !== undefined && errors.length)
+                    throw new AggregateError([releaseFailure, ...errors], "auth claim release and connection close failed");
+                if (errors.length)
+                    throw new AggregateError(errors, "auth remaining clients failed to close");
+            }
+            if (releaseFailure !== undefined)
+                throw releaseFailure;
         },
     };
 }
@@ -1560,7 +1580,9 @@ async function startAuthContext(o) {
         throw new Error(`auth-service: can't reach the broker at ${server} with the callout creds - is the mesh up (with the callout account preloaded)?`);
     // ---- The authority plane (R1): stores ensured + registry + supervised reader, BEFORE the
     // callout exists — a connect must never be answered without the credential arm bound.
-    const plane = await openAuthAuthorityPlane({
+    const owned = new OwnedConnections();
+    const onConnection = (nc, label) => owned.track(nc, label);
+    const plane = await openAuthAuthorityPlaneWithCustody({
         server,
         space,
         dir,
@@ -1569,7 +1591,7 @@ async function startAuthContext(o) {
         log: (l) => console.error(l),
         localManager: o.localManager,
         ...(o.standingRenewableTtlSeconds !== undefined ? { standingRenewableTtlSeconds: o.standingRenewableTtlSeconds } : {}),
-    });
+    }, onConnection);
     let nc;
     let http;
     let publicHttp;
@@ -1592,6 +1614,7 @@ async function startAuthContext(o) {
             authenticator: credsAuthenticator(new TextEncoder().encode(callout.calloutCreds)),
             name: `cotal:auth-service:${space}`,
         });
+        onConnection(nc, `cotal:auth-service:${space}`);
         startAuthCallout(nc, {
             xkeySeed: callout.xkey.seed,
             authAccount: { pub: callout.account.pub, signingSeed: callout.account.signingSeed },
@@ -1708,20 +1731,44 @@ async function startAuthContext(o) {
             if (state === "ready")
                 state = "draining";
             const flight = (async () => {
-                await (closeAdmission ??= (async () => {
+                const admissionFlight = closeAdmission ??= (async () => {
                     await Promise.all([closeServer(http), closeServer(publicHttp)]);
                     await reader?.client.then((c) => c.close(), () => { });
-                })());
+                })();
+                let admissionFailure;
+                try {
+                    await admissionFlight;
+                }
+                catch (e) {
+                    admissionFailure = e;
+                    if (closeAdmission === admissionFlight)
+                        closeAdmission = undefined;
+                }
+                let planeFailure;
                 try {
                     await plane.close();
                 }
-                finally {
+                catch (e) {
+                    planeFailure = e;
+                }
+                let calloutFailure;
+                try {
                     await callNc.close();
                 }
+                catch (cause) {
+                    calloutFailure = new Error(`auth connection cotal:auth-service:${space} failed to close`, { cause });
+                }
+                const failures = [admissionFailure, planeFailure, calloutFailure].filter((e) => e !== undefined);
+                if (failures.length > 1)
+                    throw new AggregateError(failures, "auth context close failed");
+                if (failures.length)
+                    throw failures[0];
+                await owned.closeRemaining();
+                owned.complete();
             })();
             closePromise = flight;
-            void flight.catch((e) => {
-                if (planeClaimRefusal(e) === "release-unreachable" && closePromise === flight)
+            void flight.catch(() => {
+                if (closePromise === flight)
                     closePromise = undefined;
             });
             return flight;
@@ -1769,9 +1816,12 @@ async function startAuthContext(o) {
         const readStatus = (owner) => async (instanceId) => {
             refuseUnlessReady();
             if (reader?.instanceId !== instanceId) {
-                void reader?.client.then((c) => c.close(), () => { });
+                void reader?.client.then((c) => c.close(), () => { }).catch((e) => {
+                    console.error(`auth readiness reader close failed: ${e instanceof Error ? e.message : String(e)}`);
+                });
                 const caller = { owner, actor: `manager_ready_${instanceId}`, uid: mintLifecycleUid() };
                 const client = openAuthorityClient({
+                    onConnection,
                     server,
                     space,
                     dataAccount: { pub: keys.dataAccount.pub, signingSeed: keys.dataAccount.signingSeed },
@@ -1792,6 +1842,8 @@ async function startAuthContext(o) {
             url,
             ...(publicUrl !== undefined ? { publicUrl } : {}),
             cap,
+            closed: owned.closed,
+            connections: () => owned.connections(),
             ...(platformDeps !== undefined ? {
                 platformControlAuthority: makePlatformControlAuthority({
                     ...platformDeps,