@cotal-ai/cli 0.72.1 → 0.76.0

package/dist/commands/up.js

@@ -2,7 +2,7 @@ import { spawn } from "node:child_process";
 import { createHash, randomUUID } from "node:crypto";
 import { createConnection, createServer } from "node:net";
 import { hostname } from "node:os";
-import { mkdirSync, writeFileSync, readFileSync, existsSync, openSync, statSync, readSync, closeSync, lstatSync, rmSync, realpathSync, readdirSync, } from "node:fs";
+import { mkdirSync, writeFileSync, readFileSync, existsSync, openSync, statSync, readSync, closeSync, lstatSync, realpathSync, readdirSync, } from "node:fs";
 import { dirname, join, resolve } from "node:path";
 import { isReachable, DEFAULT_SERVER, createSpaceAuth, serverConfig, openServerConfig, validateTlsMaterial, mintCreds, mintLifecycleUid, DEV_OWNER, mintConnectionEvictorCreds, mintMembershipObserverCreds, newIdentity, setupSpaceStreams, reconcileSpaceTtls, ttlBuckets, standaloneConnectOpts, requireBrokerFloor, parseServerVersion, presenceBucket, seedChannelRegistry, ensureDefaultDeliveryClass, LEASE_TTL_MS, mkSecretDir, writeSecretFile, } from "@cotal-ai/core";
 import { connect } from "@nats-io/transport-node";
@@ -903,8 +903,19 @@ async function runUp(args, inheritedLock, onAdopt) {
                 process.exit(1);
             }
         }
+        const restored = resumeAttempt ? pendingRestores.get(resumeAttempt) : undefined;
+        const boundListener = restored ? {
+            serverName: restored.serverName,
+            serverNonce: restored.serverNonce,
+            onSpawn: (pid, startedAt) => bindSpawnedRestoreListener(restored, pid, startedAt, startupLock),
+            verify: async () => { await provePreparedRestoreListener(restored); },
+        } : ordinaryAttempt ? {
+            serverName: ordinaryAttempt.serverName,
+            serverNonce: ordinaryAttempt.serverNonce,
+            onSpawn: (pid, startedAt) => bindSpawnedOrdinaryResumeListener(ordinaryAttempt, pid, startedAt, startupLock),
+            verify: async () => { await verifySpawnedOrdinaryListener(ordinaryAttempt); },
+        } : undefined;
         if (values.detach) {
-            const restored = resumeAttempt ? pendingRestores.get(resumeAttempt) : undefined;
             const { pid, source, authService, controlPlane, delivery, manager } = await startMeshDetached({
                 transport,
                 server,
@@ -925,21 +936,7 @@ async function runUp(args, inheritedLock, onAdopt) {
                 noManager,
                 resumeAttempt,
                 resumeCommitToken: restored?.managerCommit?.durableCommitToken ?? ordinaryAttempt?.managerCommit?.durableCommitToken,
-                ...(restored ? {
-                    boundListener: {
-                        serverName: restored.serverName,
-                        serverNonce: restored.serverNonce,
-                        onSpawn: (pid, startedAt) => bindSpawnedRestoreListener(restored, pid, startedAt, startupLock),
-                        verify: async () => { await provePreparedRestoreListener(restored); },
-                    },
-                } : ordinaryAttempt ? {
-                    boundListener: {
-                        serverName: ordinaryAttempt.serverName,
-                        serverNonce: ordinaryAttempt.serverNonce,
-                        onSpawn: (pid, startedAt) => bindSpawnedOrdinaryResumeListener(ordinaryAttempt, pid, startedAt, startupLock),
-                        verify: async () => { await verifySpawnedOrdinaryListener(ordinaryAttempt); },
-                    },
-                } : {}),
+                boundListener,
             });
             // Transport policy is committed inside startMeshDetached before delivery launch (S5+S9).
             console.log(c.dim(`Started nats-server (${source}).`));
@@ -979,7 +976,6 @@ async function runUp(args, inheritedLock, onAdopt) {
         assertServesDialHost(transport, new URL(server).hostname);
         const setup = useAuth ? await authSetup(storeDir, server, space, host, wantUser ? { idpUrl: values.idp } : undefined, transport, values["rotate-sys"], maxFileStore) : undefined;
         const port = Number(new URL(server).port) || 4222;
-        const restored = resumeAttempt ? pendingRestores.get(resumeAttempt) : undefined;
         // Both modes go through a RENDERER, never bare CLI flags. Open mode used to start from
         // `-js -sd … -p … -a …`, which never called a renderer at all — so the required transport union
         // protected the auth path and was silent on the open one, and a cert/key pair passed to an
@@ -987,9 +983,7 @@ async function runUp(args, inheritedLock, onAdopt) {
         const confPath = setup ? setup.confPath : writeOpenBrokerConf(storeDir, { port, host, transport, maxFileStore });
         const natsArgs = [
             "-c", confPath,
-            ...(restored ? ["--name", restored.serverName]
-                : ordinaryAttempt ? ["--name", ordinaryAttempt.serverName]
-                    : []),
+            ...(boundListener ? ["--name", boundListener.serverName] : []),
         ];
         const { bin, source } = await resolveNatsServer();
         console.log(c.dim(`Starting nats-server (JetStream, ${useAuth ? "JWT auth" : "OPEN/no-auth"}, ${source}) - store: ${storeDir}, bind: ${host}`));
@@ -997,29 +991,7 @@ async function runUp(args, inheritedLock, onAdopt) {
         const listenerStartedAt = new Date().toISOString();
         const child = spawn(bin, natsArgs, { stdio: "inherit" });
         let activationFinished = !resumeAttempt;
-        if (!child.pid)
-            throw new Error("nats-server spawned with no pid");
-        writePidPair(cotalPath("nats.pid"), child.pid); // #969/#1238: publish the pair by rename, never torn
-        if (restored && process.env.COTAL_SMOKE_EXIT_AFTER_RESTORE_LISTENER_SPAWN === "1")
-            process.exit(87);
-        if (restored)
-            try {
-                bindSpawnedRestoreListener(restored, child.pid ?? 0, listenerStartedAt, startupLock);
-            }
-            catch (error) {
-                await stopUnboundRestoreListener(child);
-                removeMatchingNatsPid(child.pid ?? 0);
-                throw error;
-            }
-        if (ordinaryAttempt)
-            try {
-                bindSpawnedOrdinaryResumeListener(ordinaryAttempt, child.pid ?? 0, listenerStartedAt, startupLock);
-            }
-            catch (error) {
-                await stopUnboundRestoreListener(child);
-                removeMatchingNatsPid(child.pid ?? 0);
-                throw error;
-            }
+        await recordSpawnedListener(child, listenerStartedAt, boundListener);
         releaseStartupLock();
         child.on("error", (err) => {
             console.error(c.red(`Failed to start nats-server: ${err.message}`));
@@ -1072,6 +1044,8 @@ async function runUp(args, inheritedLock, onAdopt) {
         // The broker is gone — drop it from the registry (and the `current` pointer if it was the default)
         // so a later `cotal spawn` doesn't try to join a dead mesh.
         child.on("exit", async (code, signal) => {
+            if (failedListeners.has(child))
+                return;
             removePidPair(cotalPath("nats.pid"), String(child.pid));
             // One teardown at a time: a stop started while another is in flight is refused the reservation
             // that one holds, so wait for a Ctrl-C teardown already running instead of racing it.
@@ -1114,20 +1088,7 @@ async function runUp(args, inheritedLock, onAdopt) {
                     process.exit(code ?? 1);
             }
         });
-        const ready = await waitReady(server, setup?.creds);
-        if (!ready) {
-            child.kill("SIGTERM");
-            const reason = `nats-server did not become ready at ${server}`;
-            // `startupLock` is already released by here on this path (and the helper then takes its own);
-            // passed anyway so the call stays correct if the release above ever moves.
-            markPendingResumeDegraded(resumeAttempt ?? "", reason, startupLock);
-            throw new Error(reason);
-        }
-        const v = await brokerVersion(server, space, setup?.creds);
-        if (belowPresenceSafeBroker(v.version) && v.presenceFileBacked)
-            console.error(c.yellow(`! nats-server ${v.version} is below 2.14.5 and the presence bucket of "${space}" is file-backed: it can latch after a stall and close the mesh to new joins (#1356); put a nats-server 2.14.5 or newer on PATH (it wins over the bundled one)`));
-        if (restored)
-            await provePreparedRestoreListener(restored);
+        await awaitSpawnedListener({ child, server, space, creds: setup?.creds, log: "the terminal output above", boundListener });
         {
             // A DAEMON THAT DIES UNDER THIS RUNNING BROKER IS RESTARTED HERE (#2469). It exits on its own once
             // it cannot reach the broker, and a starved host makes a running broker look unreachable, so it can
@@ -1294,15 +1255,6 @@ function restoreListenerOwner(pid, nonce, startedAt) {
         throw new Error("restore listener spawn returned no pid");
     return { pid, host: hostname(), startedAt, id: `restore-listener-${nonce}` };
 }
-function removeMatchingNatsPid(pid) {
-    const path = cotalPath("nats.pid");
-    try {
-        const stat = lstatSync(path);
-        if (stat.isFile() && !stat.isSymbolicLink() && readFileSync(path, "utf8") === String(pid))
-            rmSync(path);
-    }
-    catch { /* absent, changed, or not owned by this spawn */ }
-}
 function waitForChildExit(child, timeoutMs) {
     if (child.exitCode !== null || child.signalCode !== null)
         return Promise.resolve(true);
@@ -1317,15 +1269,23 @@ function waitForChildExit(child, timeoutMs) {
         child.once("exit", onExit);
     });
 }
-async function stopUnboundRestoreListener(child) {
-    if (child.exitCode !== null || child.signalCode !== null)
-        return;
+/** Listeners {@link stopFailedListener} is stopping. Their launch reports its own error, so the
+ *  foreground's broker-exit handler must not report that exit as a crash and exit first. */
+const failedListeners = new WeakSet();
+/**
+ * Stop a listener whose launch failed before it served. A signal is not proof of exit, so SIGTERM
+ * escalates to SIGKILL and `nats.pid` is removed only once the exit is observed. A listener that
+ * outlives both keeps its record and fails loud with its pid.
+ */
+async function stopFailedListener(child) {
+    failedListeners.add(child);
     child.kill("SIGTERM");
-    if (await waitForChildExit(child, 5_000))
-        return;
-    child.kill("SIGKILL");
-    if (!await waitForChildExit(child, 5_000))
-        throw new Error(`unbound restore listener process ${child.pid ?? "unknown"} did not exit`);
+    if (!await waitForChildExit(child, 5_000)) {
+        child.kill("SIGKILL");
+        if (!await waitForChildExit(child, 5_000))
+            throw new Error(`nats-server process ${child.pid ?? "unknown"} did not exit after SIGKILL`);
+    }
+    removePidPair(cotalPath("nats.pid"), String(child.pid));
 }
 function bindSpawnedRestoreListener(prepared, pid, startedAt, heldLock) {
     bindRestoreListenerOwner(prepared, restoreListenerOwner(pid, prepared.serverNonce, startedAt), heldLock);
@@ -2093,6 +2053,55 @@ tlsRequired, mgr) {
         return undefined;
     }
 }
+/**
+ * Publish a spawned listener's pid, and bind a maintenance-bound one to its attempt, before anything
+ * waits on it. A launch that dies before its listener serves then leaves `nats.pid` for `cotal down`
+ * and a journal naming the exact process for recovery to adopt or retire. A bind that throws stops
+ * the listener. Both launch modes call it right after the spawn.
+ */
+async function recordSpawnedListener(child, startedAt, bound) {
+    if (!child.pid)
+        throw new Error("nats-server spawned with no pid");
+    writePidPair(cotalPath("nats.pid"), child.pid); // #969/#1238: publish the pair by rename, never torn
+    if (!bound)
+        return;
+    if (process.env.COTAL_SMOKE_EXIT_AFTER_RESTORE_LISTENER_SPAWN === "1")
+        process.exit(87);
+    try {
+        bound.onSpawn(child.pid, startedAt);
+    }
+    catch (error) {
+        await stopFailedListener(child);
+        throw error;
+    }
+}
+/**
+ * Everything between spawning a listener and serving it: readiness, the broker version read and a
+ * bound listener's identity check. Foreground `up` and {@link startMeshDetached} both call it, so a
+ * guard added here reaches both launch modes (#2924).
+ *
+ * A listener that never answers or whose version read throws is stopped and its `nats.pid` removed
+ * before `up` fails. Nothing has recorded the mesh yet, so a listener left running would hold the
+ * port with no registry entry. A bound listener that fails its identity check is kept: its
+ * maintenance journal is already bound to it.
+ */
+async function awaitSpawnedListener(l) {
+    const { child, server, space } = l;
+    let facts;
+    try {
+        if (!(await waitReady(server, l.creds)))
+            throw new Error(`nats-server did not become reachable at ${server} - see ${l.log}`);
+        facts = await brokerVersion(server, space, l.creds);
+    }
+    catch (e) {
+        await stopFailedListener(child);
+        throw e;
+    }
+    if (belowPresenceSafeBroker(facts.version) && facts.presenceFileBacked)
+        console.error(c.yellow(`! nats-server ${facts.version} is below 2.14.5 and the presence bucket of "${space}" is file-backed: it can latch after a stall and close the mesh to new joins (#1356); put a nats-server 2.14.5 or newer on PATH (it wins over the bundled one)`));
+    if (l.boundListener)
+        await l.boundListener.verify();
+}
 /**
  * Everything a launch runs once its listener answers: the space setup, the user-auth service, the
  * mesh record, the transport commit and the control plane. Foreground `up` and
@@ -2112,14 +2121,7 @@ async function serveReadyListener(l) {
             await postStart(server, space, setup, l.seedFile);
         }
         catch (e) {
-            try {
-                child.kill("SIGTERM");
-            }
-            catch { /* already gone */ }
-            try {
-                removePidPair(cotalPath("nats.pid"), String(child.pid));
-            }
-            catch { /* best effort */ }
+            await stopFailedListener(child);
             throw e;
         }
     }
@@ -2224,50 +2226,17 @@ opts) {
     const listenerStartedAt = new Date().toISOString();
     const child = spawn(bin, args, { detached: true, stdio: ["ignore", fd, fd] });
     closeSync(fd);
-    if (opts.boundListener) {
-        if (!child.pid)
-            throw new Error("nats-server spawned with no pid");
-        writePidPair(cotalPath("nats.pid"), child.pid); // #969/#1238: publish the pair by rename, never torn
-        if (process.env.COTAL_SMOKE_EXIT_AFTER_RESTORE_LISTENER_SPAWN === "1")
-            process.exit(87);
-        try {
-            opts.boundListener.onSpawn(child.pid ?? 0, listenerStartedAt);
-        }
-        catch (error) {
-            await stopUnboundRestoreListener(child);
-            removeMatchingNatsPid(child.pid ?? 0);
-            throw error;
-        }
-    }
+    await recordSpawnedListener(child, listenerStartedAt, opts.boundListener);
     child.unref();
     let tailing = Boolean(opts.onLine);
     if (opts.onLine)
         tailLines(logPath, startOffset, opts.onLine, () => !tailing);
-    const ready = await waitReady(server, setup?.creds);
-    tailing = false;
-    if (!ready) {
-        child.kill("SIGTERM");
-        removePidPair(cotalPath("nats.pid"), String(child.pid));
-        throw new Error(`nats-server did not become reachable at ${server} - see ${logPath}`);
-    }
-    let brokerVer;
     try {
-        brokerVer = await brokerVersion(server, space, setup?.creds);
-    }
-    catch (e) {
-        child.kill("SIGTERM");
-        removePidPair(cotalPath("nats.pid"), String(child.pid));
-        throw e;
+        await awaitSpawnedListener({ child, server, space, creds: setup?.creds, log: logPath, boundListener: opts.boundListener });
     }
-    if (belowPresenceSafeBroker(brokerVer.version) && brokerVer.presenceFileBacked)
-        console.error(c.yellow(`! nats-server ${brokerVer.version} is below 2.14.5 and the presence bucket of "${space}" is file-backed: it can latch after a stall and close the mesh to new joins (#1356); put a nats-server 2.14.5 or newer on PATH (it wins over the bundled one)`));
-    if (!opts.boundListener) {
-        if (!child.pid)
-            throw new Error("nats-server spawned with no pid");
-        writePidPair(cotalPath("nats.pid"), child.pid); // #969/#1238: publish the pair by rename, never torn
+    finally {
+        tailing = false;
     }
-    if (opts.boundListener)
-        await opts.boundListener.verify();
     const { authService, controlPlane } = await serveReadyListener({
         child, server, space, storeDir, setup, seedFile, transport,
         host: opts.host,