@cotal-ai/cli 0.61.0 → 0.63.0

package/dist/commands/up.js

@@ -4,7 +4,7 @@ import { createConnection, createServer } from "node:net";
 import { hostname } from "node:os";
 import { mkdirSync, writeFileSync, readFileSync, existsSync, openSync, statSync, readSync, closeSync, lstatSync, rmSync, realpathSync, readdirSync, } from "node:fs";
 import { dirname, join, resolve } from "node:path";
-import { isReachable, DEFAULT_SERVER, createSpaceAuth, serverConfig, openServerConfig, validateTlsMaterial, mintCreds, mintLifecycleUid, DEV_OWNER, mintConnectionEvictorCreds, mintMembershipObserverCreds, newIdentity, setupSpaceStreams, reconcileSpaceTtls, ttlBuckets, standaloneConnectOpts, requireBrokerFloor, parseServerVersion, presenceBucket, seedChannelRegistry, ensureDefaultDeliveryClass, mkSecretDir, writeSecretFile, } from "@cotal-ai/core";
+import { isReachable, DEFAULT_SERVER, createSpaceAuth, serverConfig, openServerConfig, validateTlsMaterial, mintCreds, mintLifecycleUid, DEV_OWNER, mintConnectionEvictorCreds, mintMembershipObserverCreds, newIdentity, setupSpaceStreams, reconcileSpaceTtls, ttlBuckets, standaloneConnectOpts, requireBrokerFloor, parseServerVersion, presenceBucket, seedChannelRegistry, ensureDefaultDeliveryClass, LEASE_TTL_MS, mkSecretDir, writeSecretFile, } from "@cotal-ai/core";
 import { connect } from "@nats-io/transport-node";
 import { jetstreamManager, StorageType } from "@nats-io/jetstream";
 import { Kvm } from "@nats-io/kv";
@@ -17,7 +17,7 @@ import { resolveNatsServer } from "../lib/nats-bin.js";
 import { cotalPath, cotalRoot } from "../lib/paths.js";
 import { renderDetachedSummary } from "../lib/up-report.js";
 import { detachedSystemdSupervisionWarning } from "../lib/systemd-supervision.js";
-import { deliveryUp, ensureControlPlane, stopDelivery } from "../lib/delivery-proc.js";
+import { deliveryStoppedByDown, deliveryUp, ensureControlPlane, ensureDelivery, stopDelivery } from "../lib/delivery-proc.js";
 import { RESPONDER_UNBOUND_CONSEQUENCE } from "../lib/delivery-responder.js";
 import { displayCmd } from "../lib/self-exec.js";
 import { liveManagerWouldApplyMaxSessions, managerHasDeliveryMarker, managerLogDisplayPath, managerRecordState, managerUp, stopManager } from "../lib/manager-proc.js";
@@ -26,6 +26,7 @@ import { loadManifest } from "../lib/manifest/index.js";
 import { buildLaunchSpec, genRunId, manifestToChannels, preflightConnectors, writeLaunchSpec } from "../lib/manifest/apply.js";
 import { renderUpPlan, renderInherited, renderWarnings } from "../lib/manifest/render.js";
 import { failManifest } from "./topology.js";
+import { reserveStop } from "./down.js";
 import { extensionNames, preflightRuntime } from "../ext-loader.js";
 import { completingFlagValue } from "../lib/completion.js";
 import { askManager } from "../lib/control.js";
@@ -808,7 +809,7 @@ async function runUp(args, inheritedLock, onAdopt) {
                 // failure copy says "restart it with `cotal up`"), so a refresh must re-ensure the service —
                 // never just reprint "already running" over a dead callout. No broker config is (re)written
                 // here, so healing on a bare `cotal up` is safe: the mode can't drift, only the daemon heals.
-                let userAuth = held.userAuth;
+                let userAuth;
                 if (held.mode === "user") {
                     const auth = await getSpaceAuth(workspaceSecretStore(root), held.space);
                     if (!auth) {
@@ -868,15 +869,11 @@ async function runUp(args, inheritedLock, onAdopt) {
                     if (!controlPlane)
                         process.exitCode = 1;
                 }
-                const heldAttachHost = attachHostFor(held.space, values.host);
-                const heldMaxSessions = maxSessionsFor(held.space, maxSessions);
                 // A broker was already answering here — this branch starts nothing, so it must not claim the
-                // record as ours (see `Provenance`).
-                // `tlsRequired` is CARRIED FORWARD, not re-derived. This branch starts no listener, so it has
-                // no transport decision to record — and `recordOurMesh` writes the entry whole, so omitting
-                // the field here would erase the requirement on every bare refresh, exactly the way dropping
-                // `attachHost` would silently demote the mesh to loopback.
-                recordOurMesh({ space: held.space, server, root, mode: held.mode, ...(held.tlsRequired !== undefined ? { tlsRequired: held.tlsRequired } : {}), ...(userAuth ? { userAuth } : {}), ...(heldAttachHost ? { attachHost: heldAttachHost } : {}), ...(heldMaxSessions !== undefined ? { maxSessions: heldMaxSessions } : {}), ...(held.maxFileStore !== undefined ? { maxFileStore: held.maxFileStore } : {}), ...(held.storeDir !== undefined ? { storeDir: held.storeDir } : {}), ts: new Date().toISOString() }, "refresh");
+                // record as ours (see `Provenance`), and it passes only what this invocation decided.
+                // `recordOurMesh` carries everything else (`tlsRequired`, the recorded host and session
+                // ceiling, the store) from the record as it stands at write time, not from `held`.
+                recordOurMesh({ space: held.space, server, root, mode: held.mode, ...(userAuth ? { userAuth } : {}), ...(values.host ? { attachHost: values.host } : {}), ...(maxSessions !== undefined ? { maxSessions } : {}), ts: new Date().toISOString() }, "refresh");
                 return;
             }
             const who = held ? `mesh "${held.space}" (${held.root})` : "a broker not started here";
@@ -1047,8 +1044,11 @@ async function runUp(args, inheritedLock, onAdopt) {
             void (async () => {
                 const managerContext = { root: cotalRoot(), space };
                 let spared;
+                let spareSeats;
                 let legacyManagerSpareUnverified = false;
                 const managerPidPath = localProcessPath(MANAGER_PIDFILE, managerContext);
+                const stopMarker = `${managerPidPath}.stopping`;
+                let reserved = false;
                 if (existsSync(managerPidPath)) {
                     const pin = verifyIdentityPin(managerPidPath);
                     if (pin.kind === "legacy")
@@ -1056,7 +1056,12 @@ async function runUp(args, inheritedLock, onAdopt) {
                     else if (pin.kind === "match") {
                         spared = await listManagerSeatsForSpare(managerContext);
                         try {
-                            assertManagerCanSpare(managerContext, undefined, pin.record);
+                            // Taken before the capability read and held until the manager is gone, as bare `down`
+                            // holds it, so a concurrent `cotal down` can neither stop this manager nor arm a reap
+                            // while this stop is in flight.
+                            reserveStop("manager", stopMarker);
+                            reserved = true;
+                            spareSeats = assertManagerCanSpare(managerContext, undefined, pin.record);
                         }
                         catch (e) {
                             // THE CAPABILITY ASSERT IS THE SIGNAL GATE, the same rule bare `down` enforces inside
@@ -1065,6 +1070,8 @@ async function runUp(args, inheritedLock, onAdopt) {
                             // release the latch so the stack keeps running in the foreground; the operator ends it
                             // with `cotal down --with-agents` from another terminal, and the broker-exit handler
                             // below already ends `up` when the broker goes.
+                            if (reserved)
+                                rmSync(stopMarker, { force: true });
                             console.error(c.red(`! teardown: ${e.message}`));
                             console.error(c.red(`the stack is still running; to take managed agents with it, run: cotal down --with-agents`));
                             stopping = false;
@@ -1078,11 +1085,15 @@ async function runUp(args, inheritedLock, onAdopt) {
                     if (legacyManagerSpareUnverified)
                         printLegacyManagerSpareUncertainty();
                     else if (spared)
-                        printSparedAgents(spared);
+                        printSparedAgents(spared, spareSeats);
                 }
                 catch (e) {
                     console.error(`! manager teardown: ${e.message}`);
                 }
+                finally {
+                    if (reserved)
+                        rmSync(stopMarker, { force: true });
+                }
                 await stopAuthService(space).catch((e) => console.error(`! auth teardown: ${e.message}`));
                 child.kill("SIGTERM");
             })();
@@ -1186,11 +1197,62 @@ async function runUp(args, inheritedLock, onAdopt) {
                 storeDir,
                 ts: new Date().toISOString(),
             }, "started");
+            // A DAEMON THAT DIES UNDER THIS RUNNING BROKER IS RESTARTED HERE (#2469). It exits on its own once
+            // it cannot reach the broker, and a starved host makes a running broker look unreachable, so it can
+            // end while this broker serves on. Nothing else brings it back, and every static retirement then
+            // fails on the ctl.delivery-admin rail and holds its name until an operator re-runs `cotal up`. A
+            // clean exit or SIGTERM/SIGINT is a deliberate stop (`cotal down delivery`) and stays stopped, and so
+            // is the SIGKILL `down` escalates to when a starved daemon outlives its grace, also when either ends a
+            // replacement this loop is still waiting on: that attempt then fails, and retrying it would undo the
+            // stop. A failed attempt waits out the lease TTL, the longest a dead holder's lease blocks its
+            // replacement. No daemon is alive during that wait for `down` to signal, so a stop there shows only as
+            // the dead daemon's record going away. Recovery is announced only once the responder is bound.
+            const meshServing = () => !stopping && child.exitCode === null && child.signalCode === null;
+            let restartingDelivery = false;
+            let deliveryStopped = false;
+            const onDeliveryExit = (code, signal) => {
+                if (code === 0 || signal === "SIGTERM" || signal === "SIGINT" || (signal !== null && deliveryStoppedByDown(space))) {
+                    deliveryStopped = true;
+                    return;
+                }
+                if (restartingDelivery || !meshServing())
+                    return;
+                restartingDelivery = true;
+                console.error(c.yellow(`! delivery daemon exited (${signal ? `signal ${signal}` : `code ${code}`}) while nats-server is running - restarting it`));
+                void (async () => {
+                    while (meshServing()) {
+                        let failure;
+                        try {
+                            const ensured = await ensureDelivery({ space, server, tls: transport.kind === "tls-required", onDeliveryExit });
+                            // An abnormal exit during this attempt was dropped above, so the record decides.
+                            if (deliveryUp(space)) {
+                                // An unbound responder was reported by ensureDelivery, and this daemon's exit re-arms the restart.
+                                if (ensured.responderBound)
+                                    console.error(c.green(`✓ delivery daemon running again${ensured.pid !== undefined ? ` (pid ${ensured.pid})` : ""}`));
+                                break;
+                            }
+                            failure = "no delivery daemon is running after the attempt";
+                        }
+                        catch (e) {
+                            failure = e.message;
+                        }
+                        if (deliveryStopped)
+                            break;
+                        console.error(c.yellow(`! delivery restart failed, retrying in ${LEASE_TTL_MS / 1000}s: ${failure}`));
+                        const recorded = !deliveryStoppedByDown(space);
+                        await new Promise((r) => setTimeout(r, LEASE_TTL_MS));
+                        if (deliveryStopped || (recorded && deliveryStoppedByDown(space)))
+                            break;
+                    }
+                    restartingDelivery = false;
+                })();
+            };
             // Bring up the delivery daemon WITH the server (auth mode only — it self-gates on `.cotal/auth`).
             // It is part of the server, so `cotal up` starts it by default; open dev mode has no daemon.
             // Class-2 credential renewal is NOT wired here: the MANAGER is the renewal owner (it is resident
             // in every mesh mode — foreground, --detach, refresh — where this foreground process is not).
             const controlPlane = await startDeliveryWithBroker(space, server, transport.kind === "tls-required", {
+                onDeliveryExit,
                 runtime: values.runtime,
                 // The address the broker was bound to. This is what lets `cotal attach` reach this manager
                 // from another machine; without it the attach face stays loopback-only, so exposing terminals
@@ -2441,13 +2503,15 @@ function recordOurMesh(m, provenance) {
     const usableCurrent = cur && findMesh(cur) ? cur : undefined; // compute before recording m
     const prior = findMesh(m.space);
     const origin = provenance === "refresh" && (prior?.origin === "manual" || prior?.origin === "catalog") ? prior.origin : "up";
-    // A REFRESH starts nothing: it concluded the mesh is up from reachability alone, and rebuilds `m`
-    // from what THIS launch knows, which is never the operator's past decisions. `origin` was already
-    // carried across for that reason; the overlay acceptance is the same class and was not, so a
-    // no-op refresh silently erased a consent the operator had given. A `started` takeover may
-    // replace it (that launch really is the mesh now); a refresh may not quietly drop it.
-    const unencryptedOverlay = provenance === "refresh" && m.unencryptedOverlay === undefined ? prior?.unencryptedOverlay : m.unencryptedOverlay;
-    recordMesh({ ...m, origin, ...(unencryptedOverlay !== undefined ? { unencryptedOverlay } : {}) });
+    // A REFRESH starts nothing: it concluded the mesh is up from reachability alone, so `m` holds only
+    // what THIS launch decided, and every other field is the record as it stands now. That covers the
+    // operator's past decisions (origin, overlay consent, TLS requirement) and anything another writer
+    // committed while the refresh ensured the control plane; rebuilding from the refresh's earlier
+    // read reverted those and still exited 0. A `started` takeover may replace the record outright
+    // (that launch really is the mesh now).
+    if (provenance === "refresh" && !prior)
+        throw new Error(`mesh "${m.space}" was removed from the registry while \`cotal up\` refreshed it - not re-recording it from an earlier read`);
+    recordMesh(provenance === "refresh" ? { ...prior, ...m, origin } : { ...m, origin });
     if (!usableCurrent) {
         setCurrent(m.space);
         return;