@cotal-ai/cli 0.57.0 → 0.59.0

package/dist/commands/up.js

@@ -2,11 +2,13 @@ import { spawn } from "node:child_process";
 import { createHash, randomUUID } from "node:crypto";
 import { createConnection, createServer } from "node:net";
 import { hostname } from "node:os";
-import { mkdirSync, writeFileSync, readFileSync, existsSync, openSync, statSync, readSync, closeSync, lstatSync, rmSync, realpathSync, } from "node:fs";
+import { mkdirSync, writeFileSync, readFileSync, existsSync, openSync, statSync, readSync, closeSync, lstatSync, rmSync, realpathSync, readdirSync, } from "node:fs";
 import { dirname, join, resolve } from "node:path";
-import { isReachable, DEFAULT_SERVER, createSpaceAuth, serverConfig, openServerConfig, validateTlsMaterial, mintCreds, mintLifecycleUid, DEV_OWNER, mintConnectionEvictorCreds, mintMembershipObserverCreds, newIdentity, setupSpaceStreams, reconcileSpaceTtls, standaloneConnectOpts, seedChannelRegistry, ensureDefaultDeliveryClass, mkSecretDir, writeSecretFile, } from "@cotal-ai/core";
+import { isReachable, DEFAULT_SERVER, createSpaceAuth, serverConfig, openServerConfig, validateTlsMaterial, mintCreds, mintLifecycleUid, DEV_OWNER, mintConnectionEvictorCreds, mintMembershipObserverCreds, newIdentity, setupSpaceStreams, reconcileSpaceTtls, ttlBuckets, standaloneConnectOpts, requireBrokerFloor, parseServerVersion, presenceBucket, seedChannelRegistry, ensureDefaultDeliveryClass, mkSecretDir, writeSecretFile, } from "@cotal-ai/core";
 import { connect } from "@nats-io/transport-node";
-import { assertSingleSpaceBroker, assertUserAuthInfo, authDir, getSoleSpaceAuth, getSpaceAuth, hasUserAuthState, listSpaceAccounts, preloadSpaceAccounts, putSpaceAuth, clearCurrent, findMesh, getCurrent, loadMeshes, connectionEvictorCredsKey, membershipConfigPath, membershipObserverCredsKey, membershipRwCredsKey, MEMBERSHIP_CONFIG_KIND, MEMBERSHIP_RW_CREDS_KIND, parsePositiveIntegerFlag, recordMesh, meshesForRoot, removeMesh, rotateSystemCreds, setCurrent, staleSystemCreds, SYSTEM_CREDS_FILES, userAuthStateDir, userAuthSpacesOnDisk, workspaceSecretStore, acquireMaintenanceLock, assertStoreIdentity, assessRestoreClaim, beginOrdinaryResume, bindOrdinaryResumeListener, consumeRetiredMaintenance, markOrdinaryResumeActive, markOrdinaryResumeDegraded, replaceDeadOrdinaryResumeListener, localProcessOwnerStatus, readMaintenanceJournal, readMaintenanceResumeDocument, seatCheckpointDir, readStoreIdentity, recordOrdinaryResumeManagerCommit, releaseMaintenanceLock, retireOrdinaryResume, sameStoreIdentity, readBrokerPolicy, writeBrokerPolicy, removeIdentityPin, writeIdentityPin, localProcessPath, MANAGER_PIDFILE, assertManagerCanSpare, verifyIdentityPin, isLoopbackHost, } from "@cotal-ai/workspace";
+import { jetstreamManager, StorageType } from "@nats-io/jetstream";
+import { Kvm } from "@nats-io/kv";
+import { assertSingleSpaceBroker, assertUserAuthInfo, authDir, getSoleSpaceAuth, getSpaceAuth, hasUserAuthState, listSpaceAccounts, preloadSpaceAccounts, putSpaceAuth, clearCurrent, findMesh, getCurrent, loadMeshes, connectionEvictorCredsKey, membershipConfigPath, membershipObserverCredsKey, membershipRwCredsKey, MEMBERSHIP_CONFIG_KIND, MEMBERSHIP_RW_CREDS_KIND, parsePositiveIntegerFlag, recordMesh, meshesForRoot, removeMesh, rotateSystemCreds, setCurrent, staleSystemCreds, SYSTEM_CREDS_FILES, userAuthStateDir, userAuthSpacesOnDisk, workspaceSecretStore, acquireMaintenanceLock, assertStoreIdentity, assessRestoreClaim, beginOrdinaryResume, bindOrdinaryResumeListener, consumeRetiredMaintenance, markOrdinaryResumeActive, markOrdinaryResumeDegraded, replaceDeadOrdinaryResumeListener, localProcessOwnerStatus, readMaintenanceJournal, readMaintenanceResumeDocument, seatCheckpointDir, readStoreIdentity, recordOrdinaryResumeManagerCommit, releaseMaintenanceLock, retireOrdinaryResume, sameStoreIdentity, readBrokerPolicy, writeBrokerPolicy, removePidPair, writePidPair, localProcessPath, MANAGER_PIDFILE, assertManagerCanSpare, verifyIdentityPin, isLoopbackHost, connectUserControlOrThrow, userViewAuth, } from "@cotal-ai/workspace";
 import { ensureAuthService, resolveAuthProvider, stopAuthService } from "../lib/auth-proc.js";
 import { resolveSpace } from "../lib/status.js";
 import { c } from "../ui.js";
@@ -584,8 +586,6 @@ async function runUp(args, inheritedLock, onAdopt) {
         // Gated on the ATTEMPT, not on holding the lock. The lock is now always held, and this refuses
         // whenever a journal exists — which is precisely the state a resume re-entry is in, so gating
         // it on the lock would refuse the very resume the lock is here to protect.
-        if (!resumeAttempt)
-            assertOrdinaryUpAllowed(cotalRoot(), values["store-dir"] ? resolve(values["store-dir"]) : cotalPath("nats"));
         let server = values.server ?? DEFAULT_SERVER;
         const host = values.host ?? "127.0.0.1";
         // `--host` is the BIND address; `server` is the URL the readiness probe, the mesh registry, and
@@ -596,6 +596,46 @@ async function runUp(args, inheritedLock, onAdopt) {
         // pair rather than starting a broker nothing can reach.
         if (values.host)
             server = reconcileHostAndServer(values.host, values.server);
+        // A same-root repair reopens the store the mesh record names, ONCE, before the ordinary-up
+        // assertion below reads --store-dir (#2218): an ordinary `up` on a space whose broker died must
+        // not silently fall back to the root's default store while the record still names a custom one.
+        // Never runs during a resume attempt, which keeps its own store checks. The reachability probe
+        // here is reused as `listenerReachable` further down instead of probing twice.
+        let repairProbe;
+        if (!resumeAttempt) {
+            const repairSpace = values.space ?? resolveSpace(process.cwd());
+            const repairRoot = cotalRoot();
+            const heldForRepair = loadMeshes().find((m) => m.origin !== "manual" && m.origin !== "catalog" && m.server === server && m.root === repairRoot && m.space === repairSpace);
+            if (heldForRepair) {
+                repairProbe = await isReachable(server);
+                if (!repairProbe) {
+                    if (values["store-dir"] !== undefined) {
+                        const requestedStoreDir = resolve(values["store-dir"]);
+                        if (heldForRepair.storeDir !== undefined && requestedStoreDir !== heldForRepair.storeDir) {
+                            console.error(c.red(`✗ mesh "${repairSpace}" is recorded at ${server} with store ${heldForRepair.storeDir} - a stopped broker can't change --store-dir (it is fixed at start); repair it with \`cotal up --store-dir ${heldForRepair.storeDir}\`, or \`cotal down\` it first`));
+                            process.exit(1);
+                        }
+                    }
+                    else if (heldForRepair.storeDir !== undefined) {
+                        values["store-dir"] = heldForRepair.storeDir;
+                        console.log(c.dim(`reopening the recorded store ${heldForRepair.storeDir}`));
+                    }
+                    else {
+                        const defaultStore = cotalPath("nats");
+                        let hasEntries = false;
+                        try {
+                            hasEntries = readdirSync(join(defaultStore, "jetstream")).length > 0;
+                        }
+                        catch { /* absent: no jetstream dir yet */ }
+                        if (!hasEntries) {
+                            console.error(c.yellow(`! mesh "${repairSpace}" was recorded before the store directory was kept, so this repair opens ${defaultStore}; if the previous broker used --store-dir, stop this one with \`cotal down\` and repair with that flag (its Store Directory: line is in ${repairRoot}/.cotal/nats.log)`));
+                        }
+                    }
+                }
+            }
+        }
+        if (!resumeAttempt)
+            assertOrdinaryUpAllowed(cotalRoot(), values["store-dir"] ? resolve(values["store-dir"]) : cotalPath("nats"));
         const restoredAttempt = resumeAttempt ? pendingRestores.get(resumeAttempt) : undefined;
         if (restoredAttempt?.reentry) {
             if (!restoredAttempt.listenerProof)
@@ -622,7 +662,7 @@ async function runUp(args, inheritedLock, onAdopt) {
         }
         if (ordinaryAttempt?.journalState === "resume-committed")
             throw new Error(`resume attempt ${resumeAttempt} is manager-committed but has no adoptable bound listener; preserving the commit token and retained suppression`);
-        const listenerReachable = await isReachable(server);
+        const listenerReachable = repairProbe !== undefined ? repairProbe : await isReachable(server);
         if (resumeAttempt && listenerReachable)
             throw new Error(`resume attempt ${resumeAttempt} refuses the unproven occupied listener at ${server}`);
         if (listenerReachable) {
@@ -678,6 +718,13 @@ async function runUp(args, inheritedLock, onAdopt) {
                     console.error(c.red(`✗ mesh "${held.space}" is already running at ${server} - a running broker can't change --max-file-store (it is fixed at start); \`cotal down\` it first, then \`cotal up --max-file-store ${maxFileStore}\``));
                     process.exit(1);
                 }
+                // The store directory is fixed the same way: an explicit --store-dir on a live refresh can
+                // only be refused when it disagrees with what the running broker actually opened. Absent
+                // `held.storeDir` (a pre-field record) has nothing to compare against, so it is not checked.
+                if (values["store-dir"] !== undefined && held.storeDir !== undefined && resolve(values["store-dir"]) !== held.storeDir) {
+                    console.error(c.red(`✗ mesh "${held.space}" is already running at ${server} - a running broker can't change --store-dir (it is fixed at start); \`cotal down\` it first, then \`cotal up --store-dir ${resolve(values["store-dir"])}\``));
+                    process.exit(1);
+                }
                 // `--max-sessions` is fixed when the manager starts. A refresh reuses a live manager as-is
                 // (`ensureManager` returns immediately), then used to persist the requested ceiling anyway.
                 // That printed `✓ already running` over an unchanged plane and left MeshEntry lying about it.
@@ -746,6 +793,9 @@ async function runUp(args, inheritedLock, onAdopt) {
                         // a same-root caller holds the space's signing material either way.
                         reconcileCreds = await mintCreds(spaceAuth, newIdentity(), "provisioner");
                     }
+                    const v = await brokerVersion(server, held.space, reconcileCreds);
+                    if (belowPresenceSafeBroker(v.version) && v.presenceFileBacked)
+                        console.error(c.yellow(`! nats-server ${v.version} is below 2.14.5 and the presence bucket of "${held.space}" is file-backed: it can latch after a stall and close the mesh to new joins (#1356); put a nats-server 2.14.5 or newer on PATH (it wins over the bundled one)`));
                     for (const r of await reconcileSpaceTtls({ servers: server, space: held.space, creds: reconcileCreds }))
                         console.log(c.dim(`  reconciled ${r.stream} TTL ${r.fromMs === 0 ? "none" : `${r.fromMs}ms`} -> ${r.toMs}ms`));
                 }
@@ -826,7 +876,7 @@ async function runUp(args, inheritedLock, onAdopt) {
                 // no transport decision to record — and `recordOurMesh` writes the entry whole, so omitting
                 // the field here would erase the requirement on every bare refresh, exactly the way dropping
                 // `attachHost` would silently demote the mesh to loopback.
-                recordOurMesh({ space: held.space, server, root, mode: held.mode, ...(held.tlsRequired !== undefined ? { tlsRequired: held.tlsRequired } : {}), ...(userAuth ? { userAuth } : {}), ...(heldAttachHost ? { attachHost: heldAttachHost } : {}), ...(heldMaxSessions !== undefined ? { maxSessions: heldMaxSessions } : {}), ...(held.maxFileStore !== undefined ? { maxFileStore: held.maxFileStore } : {}), ts: new Date().toISOString() }, "refresh");
+                recordOurMesh({ space: held.space, server, root, mode: held.mode, ...(held.tlsRequired !== undefined ? { tlsRequired: held.tlsRequired } : {}), ...(userAuth ? { userAuth } : {}), ...(heldAttachHost ? { attachHost: heldAttachHost } : {}), ...(heldMaxSessions !== undefined ? { maxSessions: heldMaxSessions } : {}), ...(held.maxFileStore !== undefined ? { maxFileStore: held.maxFileStore } : {}), ...(held.storeDir !== undefined ? { storeDir: held.storeDir } : {}), ts: new Date().toISOString() }, "refresh");
                 return;
             }
             const who = held ? `mesh "${held.space}" (${held.root})` : "a broker not started here";
@@ -947,9 +997,9 @@ async function runUp(args, inheritedLock, onAdopt) {
         const listenerStartedAt = new Date().toISOString();
         const child = spawn(bin, natsArgs, { stdio: "inherit" });
         let activationFinished = !resumeAttempt;
-        if (child.pid)
-            writeFileSync(cotalPath("nats.pid"), String(child.pid));
-        writeIdentityPin(cotalPath("nats.pid"), child.pid ?? 0); // #969: pin pid to process start
+        if (!child.pid)
+            throw new Error("nats-server spawned with no pid");
+        writePidPair(cotalPath("nats.pid"), child.pid); // #969/#1238: publish the pair by rename, never torn
         if (restored && process.env.COTAL_SMOKE_EXIT_AFTER_RESTORE_LISTENER_SPAWN === "1")
             process.exit(87);
         if (restored)
@@ -1041,28 +1091,46 @@ async function runUp(args, inheritedLock, onAdopt) {
         process.on("SIGTERM", stop);
         // The broker is gone — drop it from the registry (and the `current` pointer if it was the default)
         // so a later `cotal spawn` doesn't try to join a dead mesh.
-        child.on("exit", async (code) => {
-            removeIdentityPin(cotalPath("nats.pid"));
-            rmSync(cotalPath("nats.pid"), { force: true });
+        child.on("exit", async (code, signal) => {
+            removePidPair(cotalPath("nats.pid"), String(child.pid));
             // Logged, never silently swallowed; the daemon kill runs in stopDelivery's finally regardless.
             await stopDelivery(undefined, undefined, space).catch((e) => console.error(`! delivery teardown: ${e.message}`));
             await stopManager(undefined, undefined, undefined, space).catch((e) => console.error(`! manager teardown: ${e.message}`));
             await stopAuthService(space).catch((e) => console.error(`! auth teardown: ${e.message}`));
-            // Only unrecord if the registry still points at THIS broker. A newer broker for the same space
-            // (a concurrent `up`, or a different-port re-up that recorded after us) may have replaced our
-            // record — removing by name would clobber the live winner and hide it from the registry.
-            // …and only if it is still OUR kind of record. A concurrent `cotal meshes add --force` can
-            // replace it with a hand-registered one carrying the same server + root; that record outlives
-            // this broker by design (it is the operator's, and only they remove it), so unrecording it on
-            // our exit would delete a registration this process never owned.
-            const mine = findMesh(space);
-            if (mine && mine.origin !== "manual" && mine.server === server && mine.root === cotalRoot()) {
-                removeMesh(space);
-                if (getCurrent() === space)
-                    clearCurrent();
+            // Unrecording (and the exit code below) both depend on WHY the broker is gone. `stopping` is
+            // true only for the intended shutdown `stop()` drives (Ctrl-C, SIGTERM) — everything else here
+            // (a crash, an OOM kill, a signal from outside `up`) leaves the record and its store in place.
+            if (stopping) {
+                // Only unrecord if the registry still points at THIS broker. A newer broker for the same space
+                // (a concurrent `up`, or a different-port re-up that recorded after us) may have replaced our
+                // record — removing by name would clobber the live winner and hide it from the registry.
+                // …and only if it is still OUR kind of record. A concurrent `cotal meshes add --force` can
+                // replace it with a hand-registered one carrying the same server + root; that record outlives
+                // this broker by design (it is the operator's, and only they remove it), so unrecording it on
+                // our exit would delete a registration this process never owned.
+                const mine = findMesh(space);
+                if (mine && mine.origin !== "manual" && mine.server === server && mine.root === cotalRoot()) {
+                    removeMesh(space);
+                    if (getCurrent() === space)
+                        clearCurrent();
+                }
+                if (activationFinished)
+                    process.exit(code ?? 0);
+            }
+            else {
+                // The record is kept ON PURPOSE: it still names the store this broker opened, so a repair
+                // `up` reopens that SAME store instead of falling back to the root's default, and a
+                // supervising unit's restart takes the refresh path against the recorded store rather than
+                // racing a second broker into existence. Foreground stdio is inherited (its own log lines
+                // are already on this terminal, above), so there is no `.cotal/nats.log` to name here.
+                const detail = signal ? `signal ${signal}` : `code ${code}`;
+                console.error(c.red(`✗ nats-server exited unexpectedly (${detail}); the mesh "${space}" stays recorded (\`cotal meshes\` lists it offline); log: see the terminal output above; repair: \`cotal up --server ${server} --space ${space}\``));
+                // A foreground broker under `Restart=on-failure` must exit non-zero on a crash so the unit
+                // restarts it; exiting 0 here (as an unconditional `code ?? 0` used to) looked like a clean
+                // stop and left the unit sitting dead.
+                if (activationFinished)
+                    process.exit(code ?? 1);
             }
-            if (activationFinished)
-                process.exit(code ?? 0);
         });
         const ready = await waitReady(server, setup?.creds);
         if (!ready) {
@@ -1073,6 +1141,9 @@ async function runUp(args, inheritedLock, onAdopt) {
             markPendingResumeDegraded(resumeAttempt ?? "", reason, startupLock);
             throw new Error(reason);
         }
+        const v = await brokerVersion(server, space, setup?.creds);
+        if (belowPresenceSafeBroker(v.version) && v.presenceFileBacked)
+            console.error(c.yellow(`! nats-server ${v.version} is below 2.14.5 and the presence bucket of "${space}" is file-backed: it can latch after a stall and close the mesh to new joins (#1356); put a nats-server 2.14.5 or newer on PATH (it wins over the bundled one)`));
         if (restored)
             await provePreparedRestoreListener(restored);
         // Listener ready — commit the transport decision (S5: not before start).
@@ -1080,6 +1151,8 @@ async function runUp(args, inheritedLock, onAdopt) {
         {
             if (!resumeAttempt)
                 await postStart(server, space, setup, seedFile);
+            else
+                await recreateMemoryBuckets(server, space, setup);
             // USER MODE: the auth service comes up FIRST among the daemons — until its callout answers,
             // every user-mode connect to this broker is denied, so `up` must not report a usable user mesh
             // (nor let agents race it) on a half-started auth plane. (Foreground `up` doesn't exit here, so
@@ -1110,6 +1183,7 @@ async function runUp(args, inheritedLock, onAdopt) {
                 ...(effectiveAttachHost ? { attachHost: effectiveAttachHost } : {}),
                 ...(effectiveMaxSessions !== undefined ? { maxSessions: effectiveMaxSessions } : {}),
                 ...(maxFileStore !== undefined ? { maxFileStore } : {}),
+                storeDir,
                 ts: new Date().toISOString(),
             }, "started");
             // Bring up the delivery daemon WITH the server (auth mode only — it self-gates on `.cotal/auth`).
@@ -1204,6 +1278,28 @@ async function resumeControlAuth(root, mode) {
         epCaller: { owner: DEV_OWNER, actor: identity.id, uid },
     };
 }
+/** The caller for the resume's admin-gated manager asks on a user mesh. The gate reads the CURRENT
+ *  ledger, so they ride the logged-in operator's manager view, the principal the preserve cut used:
+ *  a static instrument carries a caller the ledger has no row for. The view selects a registered
+ *  manager instance, so it is taken after the readiness wait. Each ask is its own connection and a
+ *  bearer lives no longer than the IdP proof behind it, so every call after the first takes a fresh
+ *  bearer from the view's source, pinned to the same principal and manager instance. */
+async function userManagerAuth(space) {
+    const conn = await connectUserControlOrThrow({ space });
+    const view = await userViewAuth(conn, "manager-caller");
+    let first = view.bearer;
+    return async () => {
+        const bearer = first ?? await view.source();
+        first = undefined;
+        return {
+            bearer,
+            sentinelCreds: view.sentinelCreds,
+            tls: conn.tls,
+            epCaller: { owner: view.owner, actor: view.actor, uid: view.lifecycleUid },
+            managerInstanceId: view.managerInstanceId,
+        };
+    };
+}
 function restoreListenerOwner(pid, nonce, startedAt) {
     if (!Number.isInteger(pid) || pid <= 0)
         throw new Error("restore listener spawn returned no pid");
@@ -1476,6 +1572,7 @@ async function resumeProvenOrdinaryListener(pending, heldLock) {
         ...(adoptAttachHost ? { attachHost: adoptAttachHost } : {}),
         ...(adoptMaxSessions !== undefined ? { maxSessions: adoptMaxSessions } : {}),
         ...(pending.maxFileStore !== undefined ? { maxFileStore: pending.maxFileStore } : {}),
+        storeDir: resolve(pending.storeDir),
         ts: new Date().toISOString(),
     }, "started");
     const controlPlane = await startDeliveryWithBroker(pending.space, pending.server, adoptedTlsRequired(pending.root), {
@@ -1521,6 +1618,7 @@ async function resumeProvenRestoreListener(prepared, heldLock) {
         ...(svc.userAuth ? { userAuth: svc.userAuth } : {}),
         ...(restoreAttachHost ? { attachHost: restoreAttachHost } : {}),
         ...(restoreMaxSessions !== undefined ? { maxSessions: restoreMaxSessions } : {}),
+        storeDir: resolve(prepared.targetPath),
         ts: new Date().toISOString(),
     }, "started");
     const controlPlane = await startDeliveryWithBroker(prepared.space, prepared.server, adoptedTlsRequired(prepared.root), {
@@ -1599,12 +1697,28 @@ async function completeResumeActivation(attemptId, healthy, reason, server, held
         }
         await new Promise((resolveWait) => setTimeout(resolveWait, 150));
     }
+    let userAuth;
+    // The auth for one admin-gated ask. A failed user-mode renewal degrades the journal only where a
+    // refusal of that ask would.
+    const askAuth = async (degrade) => {
+        if (pending.mode !== "user")
+            return auth;
+        try {
+            userAuth ??= await userManagerAuth(pending.space);
+            return await userAuth();
+        }
+        catch (e) {
+            if (degrade)
+                markPendingResumeDegraded(attemptId, e instanceof Error ? e.message : String(e), heldLock);
+            throw e;
+        }
+    };
     const resumed = await askManager(pending.space, server, "resumePreserved", {
         attemptId,
         inventory: restored?.selection === "registry"
             ? { ...pending.inventory, agents: [] }
             : pending.inventory,
-    }, auth, "any", 10 * 60 * 1000);
+    }, await askAuth(true), "any", 10 * 60 * 1000);
     if (!resumed.ok) {
         const detail = resumed.data ? ` (${JSON.stringify(resumed.data)})` : "";
         const message = `${resumed.error ?? "retained-agent resume failed"}${detail}`;
@@ -1633,7 +1747,7 @@ async function completeResumeActivation(attemptId, healthy, reason, server, held
         }
     }
     if (!managerCommit) {
-        const committed = await askManager(pending.space, server, "commitResume", { attemptId }, auth, "any", 40_000);
+        const committed = await askManager(pending.space, server, "commitResume", { attemptId }, await askAuth(true), "any", 40_000);
         if (!committed.ok) {
             const message = committed.error ?? "manager resume commit failed";
             markPendingResumeDegraded(attemptId, message, heldLock);
@@ -1664,7 +1778,7 @@ async function completeResumeActivation(attemptId, healthy, reason, server, held
             process.exit(89);
     }
     else {
-        const recommitted = await askManager(pending.space, server, "commitResume", { attemptId }, auth, "any", 40_000);
+        const recommitted = await askManager(pending.space, server, "commitResume", { attemptId }, await askAuth(false), "any", 40_000);
         // A surviving manager that already finalized legitimately answers {state:"active"} with the
         // exact durable token: accept both committed shapes idempotently, then reissue the token-bound
         // finalize (itself idempotent) below.
@@ -1676,7 +1790,7 @@ async function completeResumeActivation(attemptId, healthy, reason, server, held
         if (!exactToken)
             throw new Error(`replacement manager did not recover the durable commit token for attempt ${attemptId}`);
     }
-    const finalized = await askManager(pending.space, server, "finalizeResume", { attemptId, durableCommitToken: managerCommit.durableCommitToken }, auth, "any", 40_000);
+    const finalized = await askManager(pending.space, server, "finalizeResume", { attemptId, durableCommitToken: managerCommit.durableCommitToken }, await askAuth(false), "any", 40_000);
     if (!finalized.ok)
         throw new Error(finalized.error ?? `manager resume finalize failed for attempt ${attemptId}`);
     if (!isManagerFinalizeResult(finalized.data, attemptId))
@@ -2041,8 +2155,9 @@ opts) {
     const child = spawn(bin, args, { detached: true, stdio: ["ignore", fd, fd] });
     closeSync(fd);
     if (opts.boundListener) {
-        writeFileSync(cotalPath("nats.pid"), String(child.pid));
-        writeIdentityPin(cotalPath("nats.pid"), child.pid ?? 0); // #969: pin pid to process start
+        if (!child.pid)
+            throw new Error("nats-server spawned with no pid");
+        writePidPair(cotalPath("nats.pid"), child.pid); // #969/#1238: publish the pair by rename, never torn
         if (process.env.COTAL_SMOKE_EXIT_AFTER_RESTORE_LISTENER_SPAWN === "1")
             process.exit(87);
         try {
@@ -2062,14 +2177,25 @@ opts) {
     tailing = false;
     if (!ready) {
         child.kill("SIGTERM");
-        if (opts.boundListener)
-            removeIdentityPin(cotalPath("nats.pid"));
-        rmSync(cotalPath("nats.pid"), { force: true });
+        removePidPair(cotalPath("nats.pid"), String(child.pid));
         throw new Error(`nats-server did not become reachable at ${server} - see ${logPath}`);
     }
-    if (!opts.boundListener)
-        writeFileSync(cotalPath("nats.pid"), String(child.pid));
-    writeIdentityPin(cotalPath("nats.pid"), child.pid ?? 0); // #969: pin pid to process start
+    let brokerVer;
+    try {
+        brokerVer = await brokerVersion(server, space, setup?.creds);
+    }
+    catch (e) {
+        child.kill("SIGTERM");
+        removePidPair(cotalPath("nats.pid"), String(child.pid));
+        throw e;
+    }
+    if (belowPresenceSafeBroker(brokerVer.version) && brokerVer.presenceFileBacked)
+        console.error(c.yellow(`! nats-server ${brokerVer.version} is below 2.14.5 and the presence bucket of "${space}" is file-backed: it can latch after a stall and close the mesh to new joins (#1356); put a nats-server 2.14.5 or newer on PATH (it wins over the bundled one)`));
+    if (!opts.boundListener) {
+        if (!child.pid)
+            throw new Error("nats-server spawned with no pid");
+        writePidPair(cotalPath("nats.pid"), child.pid); // #969/#1238: publish the pair by rename, never torn
+    }
     if (opts.boundListener)
         await opts.boundListener.verify();
     // POST-START MUST NOT LEAVE AN ORPHAN LISTENER.
@@ -2097,13 +2223,15 @@ opts) {
             }
             catch { /* already gone */ }
             try {
-                removeIdentityPin(cotalPath("nats.pid"));
-                rmSync(cotalPath("nats.pid"), { force: true });
+                removePidPair(cotalPath("nats.pid"), String(child.pid));
             }
             catch { /* best effort */ }
             throw e;
         }
     }
+    else {
+        await recreateMemoryBuckets(server, space, setup);
+    }
     // USER MODE: the auth service comes up FIRST among the daemons (see the foreground path).
     const svc = await startUserAuthService(space, server, setup, opts.publicExchange);
     // Record BEFORE the control plane: the manager's fail-closed mode detection needs the
@@ -2125,6 +2253,7 @@ opts) {
         ...(effectiveAttachHost ? { attachHost: effectiveAttachHost } : {}),
         ...(effectiveMaxSessions !== undefined ? { maxSessions: effectiveMaxSessions } : {}),
         ...(opts.maxFileStore !== undefined ? { maxFileStore: opts.maxFileStore } : {}),
+        storeDir,
         ts: new Date().toISOString(),
     }, "started");
     // Commit policy BEFORE delivery launch (S9). Listener is proved; refuse paths never reach here.
@@ -2303,6 +2432,10 @@ function journaledMaxFileStore(value) {
  *  asserted directly by the registry smoke, since reaching each branch through a full `up` would
  *  need a live broker per case. */
 export const recordOurMeshForTest = (m, provenance) => recordOurMesh(m, provenance);
+/** Record this mesh in the registry, and set it as the `current` default when there's no usable one
+ *  — i.e. the first mesh, OR when `current` dangles at a space that's no longer in the registry (a
+ *  ghost pointer is not a default). Never silently redirect a `current` that still resolves to a live
+ *  mesh; just say another is the default and how to switch. */
 function recordOurMesh(m, provenance) {
     const cur = getCurrent();
     const usableCurrent = cur && findMesh(cur) ? cur : undefined; // compute before recording m
@@ -2356,6 +2489,43 @@ async function waitReady(server, creds) {
     }
     return false;
 }
+/** The broker's version, and whether `space`'s presence stream exists file-backed. Cotal creates
+ *  it in memory (#1356), so only a bucket an older cotal created is file-backed; an absent one
+ *  will be created in memory by `setupSpaceStreams`. */
+async function brokerVersion(server, space, creds) {
+    const nc = await connect({ servers: server, ...standaloneConnectOpts({ creds, tls: false }) });
+    try {
+        requireBrokerFloor(nc);
+        const version = nc.info?.version ?? "";
+        let presenceFileBacked = false;
+        try {
+            const info = await (await jetstreamManager(nc)).streams.info(`KV_${presenceBucket(space)}`);
+            presenceFileBacked = info.config.storage === StorageType.File;
+        }
+        catch (e) {
+            const err = e;
+            if (err.api_error?.err_code !== 10059 && !/stream not found/i.test(err.message ?? ""))
+                throw e;
+        }
+        return { version, presenceFileBacked };
+    }
+    finally {
+        await nc.drain();
+    }
+}
+// #1356: the presence bucket's file-backed latch is fixed in nats-server 2.14.5. Below that,
+// `cotal up` warns when the space's presence bucket is file-backed (never refuses: 2.14.0 meets the
+// SPEC §13.12 floor and is the bundled broker). A memory-backed bucket has no file store to latch.
+function belowPresenceSafeBroker(version) {
+    const p = parseServerVersion(version);
+    if (!p)
+        return false;
+    if (p.major !== 2)
+        return p.major < 2;
+    if (p.minor !== 14)
+        return p.minor < 14;
+    return p.patch < 5;
+}
 /** Wildcard binds mean "every interface", so they are NOT an address a client can be told to dial:
  *  the probe/registry URL stays whatever it already is (loopback by default), which the wildcard
  *  bind necessarily covers. */
@@ -2463,6 +2633,23 @@ async function postStart(server, space, setup, seedFile) {
         deliveryClass: setup ? "durable" : "live",
     });
 }
+/** A resume or restore skips `postStart` because the preserved store already holds every canonical
+ *  stream. A memory-backed bucket is the exception: the broker stop that preserved the store emptied
+ *  it, stream and all (#1356 made presence memory-backed). Recreate those, and only those, from the
+ *  same `ttlBuckets` list `setupSpaceStreams` creates them from, before any daemon opens them. */
+async function recreateMemoryBuckets(server, space, setup) {
+    const nc = await connect({ servers: server, ...standaloneConnectOpts({ creds: setup?.creds, tls: false }) });
+    try {
+        requireBrokerFloor(nc);
+        const kvm = new Kvm(nc);
+        for (const [bucket, ttl, storage] of ttlBuckets(space))
+            if (storage === StorageType.Memory)
+                await kvm.create(bucket, { ttl, storage });
+    }
+    finally {
+        await nc.drain();
+    }
+}
 /** Load the declarative channels-config file to seed the registry. An explicit `--channels`
  *  path that's missing is a hard error; the default `.cotal/channels.json` is optional (absent
  *  ⇒ nothing to seed). */
@@ -2476,21 +2663,6 @@ function loadChannelsFile(explicit) {
     }
     return JSON.parse(readFileSync(path, "utf8"));
 }
-/** Ensure the space's trust material exists, render a server config, and mint a privileged
- *  setup creds (used to pre-create streams once the server is up). The account signing key
- *  in `.cotal/auth` is what `cotal mint` and the manager later use to issue per-agent creds.
- *
- *  USER MODE (`user` set): additionally run the registered auth provider's `prepareServer` over the
- *  SPACE-SCOPED state dir (`.cotal/auth/<space>/` — the multi-space-ready layout; nothing user-auth
- *  lives flat) and preload its extra account(s) into the broker config. The inverse is fail-closed:
- *  a space whose user-auth state exists MUST keep being started with --user-auth — regenerating the
- *  config without the callout account would silently break every sentinel connect.
- *
- *  `rotateSys` (`cotal up --rotate-sys`) is the class-3 renewal for an EXISTING space: rotate the
- *  system account, re-mint `membership-observer.creds` + `connection-evictor.creds` against the
- *  successor, and render the config from the ROTATED record so the broker this `up` starts is the one
- *  that trusts them. It belongs here because this is the single site that both owns the trust record
- *  and renders `server.conf`; anywhere else would publish creds the live broker cannot honor. */
 /**
  * Turn the `--tls-cert`/`--tls-key` pair into a validated {@link BrokerTransport}, or `plaintext`
  * when neither was given.
@@ -2690,6 +2862,21 @@ function writeOpenBrokerConf(storeDir, opts) {
     writeFileSync(confPath, openServerConfig({ port: opts.port, host: opts.host, storeDir, maxFileStore: opts.maxFileStore, transport: opts.transport }));
     return confPath;
 }
+/** Ensure the space's trust material exists, render a server config, and mint a privileged
+ *  setup creds (used to pre-create streams once the server is up). The account signing key
+ *  in `.cotal/auth` is what `cotal mint` and the manager later use to issue per-agent creds.
+ *
+ *  USER MODE (`user` set): additionally run the registered auth provider's `prepareServer` over the
+ *  SPACE-SCOPED state dir (`.cotal/auth/<space>/` — the multi-space-ready layout; nothing user-auth
+ *  lives flat) and preload its extra account(s) into the broker config. The inverse is fail-closed:
+ *  a space whose user-auth state exists MUST keep being started with --user-auth — regenerating the
+ *  config without the callout account would silently break every sentinel connect.
+ *
+ *  `rotateSys` (`cotal up --rotate-sys`) is the class-3 renewal for an EXISTING space: rotate the
+ *  system account, re-mint `membership-observer.creds` + `connection-evictor.creds` against the
+ *  successor, and render the config from the ROTATED record so the broker this `up` starts is the one
+ *  that trusts them. It belongs here because this is the single site that both owns the trust record
+ *  and renders `server.conf`; anywhere else would publish creds the live broker cannot honor. */
 async function authSetup(storeDir, server, space, host = "127.0.0.1", user, 
 // NON-OPTIONAL, and it used to default to `{ kind: "plaintext" }`. That default was the hole: the
 // manifest path called this with five arguments and silently got a cleartext listener while the
@@ -2868,23 +3055,6 @@ async function assertRootBrokerStopped(root) {
             throw new Error(`--rotate-sys: mesh "${m.space}" for this root is still reachable at ${m.server} - stop it first (\`cotal down\`), then rotate`);
     }
 }
-/** Mint the two scoped creds the delivery daemon's membership feed loads (broker-sourced graph
- *  membership), at the FRESH `cotal up` while the in-memory `$SYS` signing seed still exists:
- *   - `membership-observer.creds` — SYSTEM-account CONNZ reader (the only window it can be minted: the
- *     `$SYS` seed is never persisted).
- *   - `membership-rw.creds` — DATA-account members-read + feed-write.
- *   - `membership.json` — the DATA account id (the CONNZ/event subjects pin it; non-secret, but kept
- *     0600 alongside the creds).
- *  All 0600. Best-effort: a failure logs and leaves the feed disabled (the graph degrades to traffic-
- *  only, delivery is untouched). Runs only on a FRESH space (the `if (!auth)` branch); a normal down/up
- *  keeps `.cotal/auth` + these creds and reuses them. A space provisioned before this feature has no
- *  in-memory `$SYS` seed, so it gains membership only when its auth is regenerated (a fresh `.cotal/auth`)
- *  — a documented migration property, not a silent no-op.
- *  Coupling: `cotal clean all` deletes this identity-derived set (removeLocalState in clean.ts) —
- *  a cred added here must be added to that removal list too. `membership-rw.creds` is a MIGRATED kind:
- *  its write/read/delete all go through the {@link SecretStore} seam (here, the feed reader, and clean),
- *  so the renewal owner can re-sign it into a hosted store. The observer / evictor / config stay on the
- *  raw FS (static $SYS creds + non-secret config, not renewable kinds). */
 /** Provision the DATA-account half of the membership bundle, on EVERY `up` rather than only a fresh
  *  space. Idempotent: it writes only what is absent and is silent when both are present.
  *
@@ -2934,6 +3104,23 @@ async function healMembershipDataCreds(auth, root, space) {
         console.error(c.dim(`• broker-sourced membership not repaired: ${e.message}`));
     }
 }
+/** Mint the two scoped creds the delivery daemon's membership feed loads (broker-sourced graph
+ *  membership), at the FRESH `cotal up` while the in-memory `$SYS` signing seed still exists:
+ *   - `membership-observer.creds` — SYSTEM-account CONNZ reader (the only window it can be minted: the
+ *     `$SYS` seed is never persisted).
+ *   - `membership-rw.creds` — DATA-account members-read + feed-write.
+ *   - `membership.json` — the DATA account id (the CONNZ/event subjects pin it; non-secret, but kept
+ *     0600 alongside the creds).
+ *  All 0600. Best-effort: a failure logs and leaves the feed disabled (the graph degrades to traffic-
+ *  only, delivery is untouched). Runs only on a FRESH space (the `if (!auth)` branch); a normal down/up
+ *  keeps `.cotal/auth` + these creds and reuses them. A space provisioned before this feature has no
+ *  in-memory `$SYS` seed, so it gains membership only when its auth is regenerated (a fresh `.cotal/auth`)
+ *  — a documented migration property, not a silent no-op.
+ *  Coupling: `cotal clean all` deletes this identity-derived set (removeLocalState in clean.ts) —
+ *  a cred added here must be added to that removal list too. `membership-rw.creds` is a MIGRATED kind:
+ *  its write/read/delete all go through the {@link SecretStore} seam (here, the feed reader, and clean),
+ *  so the renewal owner can re-sign it into a hosted store. The observer / evictor / config stay on the
+ *  raw FS (static $SYS creds + non-secret config, not renewable kinds). */
 async function provisionMembershipCreds(auth, root, space) {
     try {
         const observer = await mintMembershipObserverCreds(auth, newIdentity());