@cotal-ai/cli 0.35.0 → 0.36.0

package/dist/commands/up.js

@@ -3,10 +3,10 @@ import { randomUUID } from "node:crypto";
 import { createConnection, createServer } from "node:net";
 import { hostname } from "node:os";
 import { mkdirSync, writeFileSync, readFileSync, existsSync, openSync, statSync, readSync, closeSync, lstatSync, rmSync, realpathSync, } from "node:fs";
-import { join, resolve } from "node:path";
+import { dirname, join, resolve } from "node:path";
 import { isReachable, DEFAULT_SERVER, createSpaceAuth, serverConfig, openServerConfig, validateTlsMaterial, mintCreds, mintLifecycleUid, DEV_OWNER, mintConnectionEvictorCreds, mintMembershipObserverCreds, newIdentity, setupSpaceStreams, reconcileSpaceTtls, standaloneConnectOpts, seedChannelRegistry, ensureDefaultDeliveryClass, mkSecretDir, writeSecretFile, } from "@cotal-ai/core";
 import { connect } from "@nats-io/transport-node";
-import { assertSingleSpaceBroker, assertUserAuthInfo, authDir, getSoleSpaceAuth, getSpaceAuth, hasUserAuthState, listSpaceAccounts, preloadSpaceAccounts, putSpaceAuth, clearCurrent, findMesh, getCurrent, loadMeshes, MEMBERSHIP_RW_CREDS_KEY, recordMesh, meshesForRoot, removeMesh, rotateSystemCreds, setCurrent, staleSystemCreds, SYSTEM_CREDS_FILES, userAuthStateDir, workspaceSecretStore, acquireMaintenanceLock, assertStoreIdentity, assessRestoreClaim, beginOrdinaryResume, bindOrdinaryResumeListener, consumeRetiredMaintenance, markOrdinaryResumeActive, markOrdinaryResumeDegraded, replaceDeadOrdinaryResumeListener, localProcessOwnerStatus, readMaintenanceJournal, readMaintenanceResumeDocument, readStoreIdentity, recordOrdinaryResumeManagerCommit, releaseMaintenanceLock, retireOrdinaryResume, sameStoreIdentity, readBrokerPolicy, writeBrokerPolicy, } from "@cotal-ai/workspace";
+import { assertSingleSpaceBroker, assertUserAuthInfo, authDir, getSoleSpaceAuth, getSpaceAuth, hasUserAuthState, listSpaceAccounts, preloadSpaceAccounts, putSpaceAuth, clearCurrent, findMesh, getCurrent, loadMeshes, connectionEvictorCredsKey, membershipConfigPath, membershipObserverCredsKey, membershipRwCredsKey, MEMBERSHIP_CONFIG_KIND, MEMBERSHIP_RW_CREDS_KIND, recordMesh, meshesForRoot, removeMesh, rotateSystemCreds, setCurrent, staleSystemCreds, SYSTEM_CREDS_FILES, userAuthStateDir, workspaceSecretStore, acquireMaintenanceLock, assertStoreIdentity, assessRestoreClaim, beginOrdinaryResume, bindOrdinaryResumeListener, consumeRetiredMaintenance, markOrdinaryResumeActive, markOrdinaryResumeDegraded, replaceDeadOrdinaryResumeListener, localProcessOwnerStatus, readMaintenanceJournal, readMaintenanceResumeDocument, readStoreIdentity, recordOrdinaryResumeManagerCommit, releaseMaintenanceLock, retireOrdinaryResume, sameStoreIdentity, readBrokerPolicy, writeBrokerPolicy, } from "@cotal-ai/workspace";
 import { ensureAuthService, resolveAuthProvider, stopAuthService } from "../lib/auth-proc.js";
 import { resolveSpace } from "../lib/status.js";
 import { c } from "../ui.js";
@@ -66,7 +66,24 @@ export function upComplete(argv) {
     const items = upFlags.map((f) => ({ value: `--${f.name}`, description: f.description }));
     return { items, directive: items.length ? "nofiles" : "default" };
 }
-export async function up(args) {
+/** `inheritedLock` is the root maintenance lock a recovery re-entry hands down (see the re-entry
+ *  below). Ownership transfers with it: this call releases it at its own release point. */
+export async function up(args, inheritedLock) {
+    if (!inheritedLock)
+        return await runUp(args);
+    // A re-entry owns the inherited lock from the moment it is called, but `runUp` only adopts it
+    // part-way in — an early refusal (an unusable `--runtime`, say) throws before that. Release it
+    // here in exactly that case, so a refused re-entry never leaves the root locked for the reaper.
+    let adopted = false;
+    try {
+        return await runUp(args, inheritedLock, () => { adopted = true; });
+    }
+    finally {
+        if (!adopted)
+            releaseMaintenanceLock(inheritedLock);
+    }
+}
+async function runUp(args, inheritedLock, onAdopt) {
     const values = args.values;
     if (values.restore) {
         if (values.file || values.channels)
@@ -101,6 +118,11 @@ export async function up(args) {
         }
         catch (error) {
             pendingRestores.delete(prepared.attemptId);
+            // The one journal writer deliberately NOT handed a lock, and the reason is structural rather
+            // than stylistic: no lock is held in this frame either way. This invocation returns just below
+            // without ever reaching the `startupLock` acquire, and the re-entry that did hold one released
+            // it in its own `finally` on the way out through this very throw. So the helper must take its
+            // own — passing something here would mean passing `undefined`, which reads as an oversight.
             markPreparedRestoreDegraded(prepared.root, prepared.attemptId, error.message);
             throw error;
         }
@@ -127,6 +149,9 @@ export async function up(args) {
         const lock = acquireMaintenanceLock(root);
         let pending;
         let recoveredRestore;
+        // Set only at the moment the lock is passed to a re-entry, which then owns releasing it. Any
+        // other exit from this block — including a throw after `pending` was built — still releases.
+        let handedOff = false;
         try {
             const journal = readMaintenanceJournal(root);
             if (journal?.state === "restore-ready") {
@@ -271,56 +296,64 @@ export async function up(args) {
             else if (journal) {
                 throw new Error(`cotal up is refused while maintenance state is ${journal.state}; follow the recorded recovery`);
             }
-        }
-        finally {
-            releaseMaintenanceLock(lock);
-        }
-        if (recoveredRestore) {
-            try {
-                await up({
-                    ...args,
-                    values: {
-                        ...values,
-                        __restoreAttempt: recoveredRestore.attemptId,
-                        space: recoveredRestore.space,
-                        server: recoveredRestore.server,
-                        host: recoveredRestore.host,
-                        "store-dir": recoveredRestore.targetPath,
-                        runtime: recoveredRestore.runtime,
-                        detach: recoveredRestore.detached,
-                        open: recoveredRestore.mode === "open",
-                        "user-auth": recoveredRestore.mode === "user",
-                    },
-                });
+            // The re-entries run INSIDE this block, under the lock the journal was just written with, and
+            // inherit it rather than letting it drop. Releasing here and re-acquiring in the nested `up`
+            // left the journal reading `resume-intent` with the lock free — the window a concurrent `up`
+            // could render the whole-broker server.conf through (design doc §4, P2).
+            if (recoveredRestore) {
+                handedOff = true;
+                try {
+                    await up({
+                        ...args,
+                        values: {
+                            ...values,
+                            __restoreAttempt: recoveredRestore.attemptId,
+                            space: recoveredRestore.space,
+                            server: recoveredRestore.server,
+                            host: recoveredRestore.host,
+                            "store-dir": recoveredRestore.targetPath,
+                            runtime: recoveredRestore.runtime,
+                            detach: recoveredRestore.detached,
+                            open: recoveredRestore.mode === "open",
+                            "user-auth": recoveredRestore.mode === "user",
+                        },
+                    }, lock);
+                }
+                catch (error) {
+                    // The nested `up` released the inherited lock on its way out, so this takes its own.
+                    markPendingResumeDegraded(recoveredRestore.attemptId, error instanceof Error ? error.message : String(error));
+                    throw error;
+                }
+                return;
             }
-            catch (error) {
-                markPendingResumeDegraded(recoveredRestore.attemptId, error instanceof Error ? error.message : String(error));
-                throw error;
+            if (pending) {
+                handedOff = true;
+                try {
+                    await up({
+                        ...args,
+                        values: {
+                            ...values,
+                            __ordinaryResumeAttempt: pending.attemptId,
+                            space: pending.space,
+                            server: pending.server,
+                            "store-dir": pending.storeDir,
+                            runtime: pending.runtime,
+                            detach: pending.detached,
+                            open: pending.mode === "open",
+                            "user-auth": pending.mode === "user",
+                        },
+                    }, lock);
+                }
+                catch (error) {
+                    markPendingResumeDegraded(pending.attemptId, error instanceof Error ? error.message : String(error));
+                    throw error;
+                }
+                return;
             }
-            return;
         }
-        if (pending) {
-            try {
-                await up({
-                    ...args,
-                    values: {
-                        ...values,
-                        __ordinaryResumeAttempt: pending.attemptId,
-                        space: pending.space,
-                        server: pending.server,
-                        "store-dir": pending.storeDir,
-                        runtime: pending.runtime,
-                        detach: pending.detached,
-                        open: pending.mode === "open",
-                        "user-auth": pending.mode === "user",
-                    },
-                });
-            }
-            catch (error) {
-                markPendingResumeDegraded(pending.attemptId, error instanceof Error ? error.message : String(error));
-                throw error;
-            }
-            return;
+        finally {
+            if (!handedOff)
+                releaseMaintenanceLock(lock);
         }
     }
     const wantUser = Boolean(values["user-auth"]);
@@ -454,7 +487,31 @@ export async function up(args) {
     if (values.runtime)
         await preflightRuntime(values.runtime);
     const resumeAttempt = values.__restoreAttempt ?? values.__ordinaryResumeAttempt;
-    let startupLock = resumeAttempt ? undefined : acquireMaintenanceLock(cotalRoot());
+    // Held for EVERY `up`, resume re-entry included. A re-entry used to hold no lock at all and still
+    // reach the renderer below, so a concurrent `cotal up` could rewrite the whole-broker server.conf
+    // — every tenant's trust, unserialized — while a resume was mid-flight (design doc §4, P2). A
+    // re-entry INHERITS the recovery's lock rather than taking its own: the recovery journals
+    // `resume-intent` under that lock, so releasing it to re-acquire here would leave exactly the
+    // window the probe catches — the journal already in a resume state with the lock free.
+    //
+    // Inheriting makes handing this lock down MANDATORY, not tidy: the lock is not reentrant, and it
+    // cannot stale-reap its way out either, because the recorded owner is alive — it is us. A helper
+    // that self-acquired would take the "held by a live owner" refusal and fail the resume outright.
+    // So EVERY helper below that journals under this lock takes it as a parameter, on both the restore
+    // and the ordinary side: the two ADOPT paths (proven-restore and proven-ordinary listeners, which
+    // recover by adopting an already-live listener rather than spawning a competitor), the spawn-path
+    // listener binds (detached and foreground, restore and ordinary), the dead-listener replacement,
+    // `completeResumeActivation` and the manager-commit / activation / degraded writers it calls.
+    //
+    // Read that list as a rule, not an inventory. The trap is that a call site with no lock ARGUMENT
+    // says nothing about whether the CALLEE acquires: the restore-side writers took no parameter and
+    // self-acquired, so handing the lock to the sites that visibly named it left `up --restore` dead
+    // on its first pass — it died in the listener bind, before any of the seams. If you add a helper
+    // that journals, give it `heldLock` and hand it down from here; do not audit by call-site shape.
+    // The one caller that does NOT pass it is the prepare-failure path above, where the lock has
+    // genuinely not been taken yet in this frame and the helper is meant to take its own.
+    let startupLock = inheritedLock ?? acquireMaintenanceLock(cotalRoot());
+    onAdopt?.(); // from here the release below owns it, inherited or not
     const releaseStartupLock = () => {
         if (!startupLock)
             return;
@@ -462,7 +519,10 @@ export async function up(args) {
         startupLock = undefined;
     };
     try {
-        if (startupLock)
+        // Gated on the ATTEMPT, not on holding the lock. The lock is now always held, and this refuses
+        // whenever a journal exists — which is precisely the state a resume re-entry is in, so gating
+        // it on the lock would refuse the very resume the lock is here to protect.
+        if (!resumeAttempt)
             assertOrdinaryUpAllowed(cotalRoot(), values["store-dir"] ? resolve(values["store-dir"]) : cotalPath("nats"));
         let server = values.server ?? DEFAULT_SERVER;
         const host = values.host ?? "127.0.0.1";
@@ -480,7 +540,7 @@ export async function up(args) {
                 throw new Error(`restore attempt ${resumeAttempt} re-entry has no bound listener proof; preserving recovery state`);
             const ownerStatus = localProcessOwnerStatus(restoredAttempt.listenerProof.processOwner);
             if (ownerStatus === "alive") {
-                await resumeProvenRestoreListener(restoredAttempt);
+                await resumeProvenRestoreListener(restoredAttempt, startupLock);
                 return;
             }
             if (ownerStatus === "unknown")
@@ -489,13 +549,13 @@ export async function up(args) {
                 throw new Error(`restore attempt ${resumeAttempt} is manager-committed but its bound listener is dead; preserving the commit token and retained suppression`);
             if (await isReachable(restoredAttempt.server))
                 throw new Error(`restore attempt ${resumeAttempt} refuses the occupied foreign listener at ${restoredAttempt.server}`);
-            replacePreparedDeadRestoreListener(restoredAttempt);
+            replacePreparedDeadRestoreListener(restoredAttempt, startupLock);
         }
         const ordinaryAttempt = resumeAttempt ? pendingOrdinaryResumes.get(resumeAttempt) : undefined;
         if (ordinaryAttempt?.adoptProof) {
             // The recovered attempt's exact bound listener is alive: prove it over the wire and adopt it
             // instead of spawning a competitor over the same store.
-            await resumeProvenOrdinaryListener(ordinaryAttempt);
+            await resumeProvenOrdinaryListener(ordinaryAttempt, startupLock);
             return;
         }
         if (ordinaryAttempt?.journalState === "resume-committed")
@@ -713,14 +773,14 @@ export async function up(args) {
                     boundListener: {
                         serverName: restored.serverName,
                         serverNonce: restored.serverNonce,
-                        onSpawn: (pid, startedAt) => bindSpawnedRestoreListener(restored, pid, startedAt),
+                        onSpawn: (pid, startedAt) => bindSpawnedRestoreListener(restored, pid, startedAt, startupLock),
                         verify: async () => { await provePreparedRestoreListener(restored); },
                     },
                 } : ordinaryAttempt ? {
                     boundListener: {
                         serverName: ordinaryAttempt.serverName,
                         serverNonce: ordinaryAttempt.serverNonce,
-                        onSpawn: (pid, startedAt) => bindSpawnedOrdinaryResumeListener(ordinaryAttempt, pid, startedAt),
+                        onSpawn: (pid, startedAt) => bindSpawnedOrdinaryResumeListener(ordinaryAttempt, pid, startedAt, startupLock),
                         verify: async () => { await verifySpawnedOrdinaryListener(ordinaryAttempt); },
                     },
                 } : {}),
@@ -736,7 +796,7 @@ export async function up(args) {
             // exit code, not only in the red line above.
             if (!authService)
                 process.exitCode = 1;
-            await completeResumeActivation(resumeAttempt, controlPlane && authService, !authService ? "normal listener started but the user-auth service is unavailable" : "normal listener started but the control plane is degraded", server);
+            await completeResumeActivation(resumeAttempt, controlPlane && authService, !authService ? "normal listener started but the user-auth service is unavailable" : "normal listener started but the control plane is degraded", server, startupLock);
             return;
         }
         const useAuth = !values.open;
@@ -785,7 +845,7 @@ export async function up(args) {
             process.exit(87);
         if (restored)
             try {
-                bindSpawnedRestoreListener(restored, child.pid ?? 0, listenerStartedAt);
+                bindSpawnedRestoreListener(restored, child.pid ?? 0, listenerStartedAt, startupLock);
             }
             catch (error) {
                 await stopUnboundRestoreListener(child);
@@ -794,7 +854,7 @@ export async function up(args) {
             }
         if (ordinaryAttempt)
             try {
-                bindSpawnedOrdinaryResumeListener(ordinaryAttempt, child.pid ?? 0, listenerStartedAt);
+                bindSpawnedOrdinaryResumeListener(ordinaryAttempt, child.pid ?? 0, listenerStartedAt, startupLock);
             }
             catch (error) {
                 await stopUnboundRestoreListener(child);
@@ -855,7 +915,9 @@ export async function up(args) {
         if (!ready) {
             child.kill("SIGTERM");
             const reason = `nats-server did not become ready at ${server}`;
-            markPendingResumeDegraded(resumeAttempt ?? "", reason);
+            // `startupLock` is already released by here on this path (and the helper then takes its own);
+            // passed anyway so the call stays correct if the release above ever moves.
+            markPendingResumeDegraded(resumeAttempt ?? "", reason, startupLock);
             throw new Error(reason);
         }
         if (restored)
@@ -910,7 +972,7 @@ export async function up(args) {
             });
             if (restored && process.env.COTAL_SMOKE_FAIL_AFTER_RESTORE_LISTENER_READY === "1")
                 throw new Error("smoke-injected failure after restore listener readiness");
-            await completeResumeActivation(resumeAttempt, controlPlane && svc.ok, !svc.ok ? "normal listener started but the user-auth service is unavailable" : "normal listener started but the control plane is degraded", server);
+            await completeResumeActivation(resumeAttempt, controlPlane && svc.ok, !svc.ok ? "normal listener started but the user-auth service is unavailable" : "normal listener started but the control plane is degraded", server, startupLock);
             activationFinished = true;
         }
         await new Promise(() => { });
@@ -943,10 +1005,10 @@ function assertOrdinaryUpAllowed(root, storeDir) {
         throw new Error("ordinary resume must begin through the attempt-bound startup path");
     throw new Error(`cotal up is refused while maintenance state is ${maintenance.state}; follow the recorded restore recovery`);
 }
-function markPendingResumeDegraded(attemptId, reason) {
+function markPendingResumeDegraded(attemptId, reason, heldLock) {
     const ordinary = pendingOrdinaryResumes.get(attemptId);
     if (ordinary) {
-        const lock = acquireMaintenanceLock(ordinary.root);
+        const lock = heldLock ?? acquireMaintenanceLock(ordinary.root);
         try {
             const journal = readMaintenanceJournal(ordinary.root);
             if (journal && (journal.state === "resume-intent" || journal.state === "resume-active"))
@@ -957,13 +1019,14 @@ function markPendingResumeDegraded(attemptId, reason) {
                     }]);
         }
         finally {
-            releaseMaintenanceLock(lock);
+            if (!heldLock)
+                releaseMaintenanceLock(lock);
         }
         return;
     }
     const restored = pendingRestores.get(attemptId);
     if (restored)
-        markPreparedRestoreDegraded(restored.root, restored.attemptId, reason);
+        markPreparedRestoreDegraded(restored.root, restored.attemptId, reason, heldLock);
 }
 async function resumeControlAuth(root, mode) {
     if (mode === "open")
@@ -1021,11 +1084,11 @@ async function stopUnboundRestoreListener(child) {
     if (!await waitForChildExit(child, 5_000))
         throw new Error(`unbound restore listener process ${child.pid ?? "unknown"} did not exit`);
 }
-function bindSpawnedRestoreListener(prepared, pid, startedAt) {
-    bindRestoreListenerOwner(prepared, restoreListenerOwner(pid, prepared.serverNonce, startedAt));
+function bindSpawnedRestoreListener(prepared, pid, startedAt, heldLock) {
+    bindRestoreListenerOwner(prepared, restoreListenerOwner(pid, prepared.serverNonce, startedAt), heldLock);
 }
-function bindRestoreListenerOwner(prepared, processOwner) {
-    bindPreparedRestoreListener(prepared, processOwner);
+function bindRestoreListenerOwner(prepared, processOwner, heldLock) {
+    bindPreparedRestoreListener(prepared, processOwner, heldLock);
     if (process.env.COTAL_SMOKE_EXIT_AFTER_RESTORE_LISTENER_BIND === "1")
         process.exit(86);
 }
@@ -1146,10 +1209,10 @@ async function provePreparedRestoreListener(prepared) {
     }
     return proof;
 }
-function bindSpawnedOrdinaryResumeListener(pending, pid, startedAt) {
+function bindSpawnedOrdinaryResumeListener(pending, pid, startedAt, heldLock) {
     if (!Number.isInteger(pid) || pid <= 0)
         throw new Error("resume listener spawn returned no pid");
-    const lock = acquireMaintenanceLock(pending.root);
+    const lock = heldLock ?? acquireMaintenanceLock(pending.root);
     try {
         const journal = readMaintenanceJournal(pending.root);
         if (!journal || !("ordinaryResume" in journal) || journal.ordinaryResume.attemptId !== pending.attemptId)
@@ -1164,7 +1227,8 @@ function bindSpawnedOrdinaryResumeListener(pending, pid, startedAt) {
         });
     }
     finally {
-        releaseMaintenanceLock(lock);
+        if (!heldLock)
+            releaseMaintenanceLock(lock);
     }
 }
 /** Prove the recovered attempt's live bound listener IS ours end-to-end before adoption: launch
@@ -1235,7 +1299,7 @@ async function verifySpawnedOrdinaryListener(pending) {
     if (info.server_name !== pending.serverName)
         throw new Error(`resume attempt ${pending.attemptId} spawned listener reports a foreign NATS server name`);
 }
-async function resumeProvenOrdinaryListener(pending) {
+async function resumeProvenOrdinaryListener(pending, heldLock) {
     await proveOrdinaryResumeListener(pending);
     const svc = await ensureRecoveredUserAuth(pending);
     // Read the recorded exposure BEFORE re-recording: `recordOurMesh` writes the entry whole, so a
@@ -1259,7 +1323,7 @@ async function resumeProvenOrdinaryListener(pending) {
         resumeAttempt: pending.attemptId,
         resumeCommitToken: pending.managerCommit?.durableCommitToken,
     });
-    await completeResumeActivation(pending.attemptId, controlPlane && svc.ok, !svc.ok ? "adopted resume listener has no user-auth service" : "adopted resume listener has a degraded control plane", pending.server);
+    await completeResumeActivation(pending.attemptId, controlPlane && svc.ok, !svc.ok ? "adopted resume listener has no user-auth service" : "adopted resume listener has a degraded control plane", pending.server, heldLock);
 }
 async function ensureRecoveredUserAuth(prepared) {
     if (prepared.mode !== "user")
@@ -1277,7 +1341,7 @@ async function ensureRecoveredUserAuth(prepared) {
     });
     return startUserAuthService(prepared.space, prepared.server, { prepared: provider, stateDir });
 }
-async function resumeProvenRestoreListener(prepared) {
+async function resumeProvenRestoreListener(prepared, heldLock) {
     await provePreparedRestoreListener(prepared);
     const svc = await ensureRecoveredUserAuth(prepared);
     // Same ordering constraint as the pending-adopt path above: capture the recorded exposure before
@@ -1301,9 +1365,9 @@ async function resumeProvenRestoreListener(prepared) {
         resumeAttempt: prepared.attemptId,
         resumeCommitToken: prepared.managerCommit?.durableCommitToken,
     });
-    await completeResumeActivation(prepared.attemptId, controlPlane && svc.ok, !svc.ok ? "proven restore listener has no user-auth service" : "proven restore listener has a degraded control plane", prepared.server);
+    await completeResumeActivation(prepared.attemptId, controlPlane && svc.ok, !svc.ok ? "proven restore listener has no user-auth service" : "proven restore listener has a degraded control plane", prepared.server, heldLock);
 }
-async function completeResumeActivation(attemptId, healthy, reason, server) {
+async function completeResumeActivation(attemptId, healthy, reason, server, heldLock) {
     if (!attemptId)
         return;
     const ordinary = pendingOrdinaryResumes.get(attemptId);
@@ -1312,7 +1376,7 @@ async function completeResumeActivation(attemptId, healthy, reason, server) {
     if (!pending)
         throw new Error(`resume activation lost attempt context ${attemptId}`);
     if (!healthy) {
-        markPendingResumeDegraded(attemptId, reason);
+        markPendingResumeDegraded(attemptId, reason, heldLock);
         throw new Error(reason);
     }
     let journal = readMaintenanceJournal(pending.root);
@@ -1331,12 +1395,13 @@ async function completeResumeActivation(attemptId, healthy, reason, server) {
         if (!("ordinaryResume" in journal) || journal.ordinaryResume.attemptId !== attemptId)
             throw new Error(`ordinary resume journal does not match attempt ${attemptId}`);
         if (journal.state === "resume-retired") {
-            const lock = acquireMaintenanceLock(ordinary.root);
+            const lock = heldLock ?? acquireMaintenanceLock(ordinary.root);
             try {
                 consumeRetiredMaintenance(lock);
             }
             finally {
-                releaseMaintenanceLock(lock);
+                if (!heldLock)
+                    releaseMaintenanceLock(lock);
             }
             pendingOrdinaryResumes.delete(attemptId);
             return;
@@ -1364,7 +1429,7 @@ async function completeResumeActivation(attemptId, healthy, reason, server) {
             break;
         if (Date.now() >= readinessDeadline) {
             const why = ready.error ?? "no manager answered within the readiness deadline";
-            markPendingResumeDegraded(attemptId, why);
+            markPendingResumeDegraded(attemptId, why, heldLock);
             throw new Error(why);
         }
         await new Promise((resolveWait) => setTimeout(resolveWait, 150));
@@ -1378,13 +1443,13 @@ async function completeResumeActivation(attemptId, healthy, reason, server) {
     if (!resumed.ok) {
         const detail = resumed.data ? ` (${JSON.stringify(resumed.data)})` : "";
         const message = `${resumed.error ?? "retained-agent resume failed"}${detail}`;
-        markPendingResumeDegraded(attemptId, message);
+        markPendingResumeDegraded(attemptId, message, heldLock);
         throw new Error(message);
     }
     if (restored && process.env.COTAL_SMOKE_EXIT_AFTER_RESUME_PRESERVED === "1")
         process.exit(88);
     if (ordinary && !managerCommit) {
-        const lock = acquireMaintenanceLock(ordinary.root);
+        const lock = heldLock ?? acquireMaintenanceLock(ordinary.root);
         try {
             const current = readMaintenanceJournal(ordinary.root);
             if (current?.state !== "resume-active") {
@@ -1398,35 +1463,37 @@ async function completeResumeActivation(attemptId, healthy, reason, server) {
             ordinary.journalState = "resume-active";
         }
         finally {
-            releaseMaintenanceLock(lock);
+            if (!heldLock)
+                releaseMaintenanceLock(lock);
         }
     }
     if (!managerCommit) {
         const committed = await askManager(pending.space, server, "commitResume", { attemptId }, auth, "any", 40_000);
         if (!committed.ok) {
             const message = committed.error ?? "manager resume commit failed";
-            markPendingResumeDegraded(attemptId, message);
+            markPendingResumeDegraded(attemptId, message, heldLock);
             throw new Error(message);
         }
         if (!isManagerCommitResult(committed.data, attemptId)) {
             const message = `manager resume commit returned invalid awaiting-finalize evidence for attempt ${attemptId}`;
-            markPendingResumeDegraded(attemptId, message);
+            markPendingResumeDegraded(attemptId, message, heldLock);
             throw new Error(message);
         }
         managerCommit = committed.data;
         if (ordinary) {
-            const lock = acquireMaintenanceLock(ordinary.root);
+            const lock = heldLock ?? acquireMaintenanceLock(ordinary.root);
             try {
                 recordOrdinaryResumeManagerCommit(lock, managerCommit);
                 ordinary.journalState = "resume-committed";
                 ordinary.managerCommit = managerCommit;
             }
             finally {
-                releaseMaintenanceLock(lock);
+                if (!heldLock)
+                    releaseMaintenanceLock(lock);
             }
         }
         else {
-            recordPreparedRestoreManagerCommit(restored, managerCommit);
+            recordPreparedRestoreManagerCommit(restored, managerCommit, heldLock);
         }
         if (process.env.COTAL_SMOKE_EXIT_AFTER_RESUME_COMMIT === "1")
             process.exit(89);
@@ -1457,19 +1524,20 @@ async function completeResumeActivation(attemptId, healthy, reason, server) {
     if (process.env.COTAL_SMOKE_EXIT_AFTER_RESUME_FINALIZE === "1")
         process.exit(91);
     if (ordinary) {
-        const lock = acquireMaintenanceLock(ordinary.root);
+        const lock = heldLock ?? acquireMaintenanceLock(ordinary.root);
         try {
             retireOrdinaryResume(lock, finalizeEvidence);
             consumeRetiredMaintenance(lock);
         }
         finally {
-            releaseMaintenanceLock(lock);
+            if (!heldLock)
+                releaseMaintenanceLock(lock);
         }
         pendingOrdinaryResumes.delete(attemptId);
     }
     else {
         restored.managerCommit = managerCommit;
-        markPreparedRestoreActive(restored, finalizeEvidence);
+        markPreparedRestoreActive(restored, finalizeEvidence, heldLock);
         restored.cleanupStage();
         pendingRestores.delete(attemptId);
     }
@@ -2340,7 +2408,7 @@ transport, rotateSys = false) {
     if (!auth) {
         auth = await createSpaceAuth(space);
         await putSpaceAuth(store, auth); // strips the $SYS seed at rest, but leaves the in-memory `auth` intact …
-        await provisionMembershipCreds(auth, cotalRoot()); // … so the observer can still be minted here (fresh-space only)
+        await provisionMembershipCreds(auth, cotalRoot(), space); // … so the observer can still be minted here (fresh-space only)
         // A fresh space's $SYS material was just minted from the seed that only exists in this branch, so
         // the ASK is already satisfied, so say so rather than rotating a one-second-old account, and never
         // report a rotation that did not happen.
@@ -2376,7 +2444,7 @@ transport, rotateSys = false) {
         console.log(c.dim("  NOTE: full backups taken before this rotation can no longer be restored (they are bound to the retired trust chain) - take a fresh `cotal backup` once the mesh is up."));
     }
     // The DATA half of the membership bundle, on EVERY path — see healMembershipDataCreds.
-    await healMembershipDataCreds(auth, cotalRoot());
+    await healMembershipDataCreds(auth, cotalRoot(), space);
     // The $SYS creds must be signed by the system account THIS boot is about to put in `server.conf`.
     // A rotation that committed the trust record and then died leaves them stale, unexpired, and
     // broker-dead and, crash-before-either-write, stale in a way that no comparison between the two
@@ -2389,7 +2457,7 @@ transport, rotateSys = false) {
     // same pair, so booting here would silently downgrade revocation to deny-new for the life of the
     // mesh. The repo's posture is to throw rather than degrade, and the recovery is one command that
     // this message names.
-    const stale = staleSystemCreds(cotalRoot(), auth.sys.pub);
+    const stale = staleSystemCreds(cotalRoot(), auth.sys.pub, space);
     if (stale.length)
         throw new Error(`${stale.map((x) => `${x.file} (signed by ${x.iss ? `${x.iss.slice(0, 12)}…` : "an unreadable issuer"})`).join(", ")} ` +
             `${stale.length === 1 ? "is" : "are"} not signed by this space's system account (${auth.sys.pub.slice(0, 12)}…) - ` +
@@ -2528,18 +2596,26 @@ async function assertRootBrokerStopped(root) {
  *  its own error text — advice that could not succeed. Healing here fixes both spellings at once:
  *  the ordinary `up` repairs the data half with no rotation at all, and a rotation now repairs it
  *  too, because this runs after both branches converge. */
-async function healMembershipDataCreds(auth, root) {
+async function healMembershipDataCreds(auth, root, space) {
     try {
         const store = workspaceSecretStore(root);
+        // THE RESOLVERS, not the bare kinds (P7 §2 rule 1). Both reads below are absent-means-MINT, which
+        // is precisely why the location must be resolved through the choke point: a canonical read on a
+        // root whose material is still flat answers "absent" and mints a SECOND live cred beside the one
+        // the running daemons hold. `up` is a workstation composition by construction, so the resolver
+        // gets the FS arm and moves the material on this first touch.
+        const composition = { injected: false, root };
+        const rwKey = membershipRwCredsKey(space, composition);
         const wrote = [];
-        if ((await store.get(MEMBERSHIP_RW_CREDS_KEY)) === undefined) {
-            await store.put(MEMBERSHIP_RW_CREDS_KEY, await mintCreds(auth, newIdentity(), "membership-rw"));
-            wrote.push(MEMBERSHIP_RW_CREDS_KEY);
+        if ((await store.get(rwKey)) === undefined) {
+            await store.put(rwKey, await mintCreds(auth, newIdentity(), "membership-rw"));
+            wrote.push(MEMBERSHIP_RW_CREDS_KIND); // the KIND is what an operator reads, never the segmented key
         }
-        if (!existsSync(cotalPath("membership.json"))) {
-            mkSecretDir(cotalPath()); // harden .cotal/ before the file lands, as the fresh path does
-            writeSecretFile(cotalPath("membership.json"), JSON.stringify({ accountId: auth.account.pub }));
-            wrote.push("membership.json");
+        const configPath = membershipConfigPath(root, space);
+        if (!existsSync(configPath)) {
+            mkSecretDir(dirname(configPath)); // harden the per-space dir before the file lands, as the fresh path does
+            writeSecretFile(configPath, JSON.stringify({ accountId: auth.account.pub }));
+            wrote.push(MEMBERSHIP_CONFIG_KIND);
         }
         if (wrote.length)
             console.log(c.dim(`• membership: provisioned ${wrote.join(" + ")} - the data-account half needs no system-account rotation`));
@@ -2550,7 +2626,7 @@ async function healMembershipDataCreds(auth, root) {
         console.error(c.dim(`• broker-sourced membership not repaired: ${e.message}`));
     }
 }
-async function provisionMembershipCreds(auth, root) {
+async function provisionMembershipCreds(auth, root, space) {
     try {
         const observer = await mintMembershipObserverCreds(auth, newIdentity());
         const rw = await mintCreds(auth, newIdentity(), "membership-rw");
@@ -2559,11 +2635,19 @@ async function provisionMembershipCreds(auth, root) {
         // close a revoked/removed principal's live connections. A space without it degrades to
         // deny-new-only (durable reauth) — surfaced loudly by the removal path, never silent.
         const evictor = await mintConnectionEvictorCreds(auth, newIdentity());
-        mkSecretDir(cotalPath()); // harden .cotal/ before the creds land (born under a private ACL, no race)
-        writeSecretFile(cotalPath(SYSTEM_CREDS_FILES[0]), observer);
-        await workspaceSecretStore(root).put(MEMBERSHIP_RW_CREDS_KEY, rw); // migrated kind: through the seam (0600 FS put)
-        writeSecretFile(cotalPath(SYSTEM_CREDS_FILES[1]), evictor);
-        writeSecretFile(cotalPath("membership.json"), JSON.stringify({ accountId: auth.account.pub }));
+        // All four land through the per-kind resolvers (P7 §2 rule 1), and all four through the store's
+        // `put`, which is the same `mkSecretDir` + atomic write the raw `writeSecretFile` did — the
+        // difference being that it hardens the PER-SPACE dir the material now lives in rather than
+        // `.cotal/` itself. This is the fresh-space branch, so nothing is there to migrate; it calls the
+        // resolvers anyway because a kind with two write paths is a kind that grows two layouts.
+        const composition = { injected: false, root };
+        const store = workspaceSecretStore(root);
+        await store.put(membershipObserverCredsKey(space, composition), observer);
+        await store.put(membershipRwCredsKey(space, composition), rw);
+        await store.put(connectionEvictorCredsKey(space, composition), evictor);
+        const configPath = membershipConfigPath(root, space);
+        mkSecretDir(dirname(configPath));
+        writeSecretFile(configPath, JSON.stringify({ accountId: auth.account.pub }));
     }
     catch (e) {
         console.error(c.dim(`• broker-sourced membership not provisioned: ${e.message}`));