@cotal-ai/cli 0.60.0 → 0.62.0

package/dist/commands/up.js

@@ -4,7 +4,7 @@ import { createConnection, createServer } from "node:net";
 import { hostname } from "node:os";
 import { mkdirSync, writeFileSync, readFileSync, existsSync, openSync, statSync, readSync, closeSync, lstatSync, rmSync, realpathSync, readdirSync, } from "node:fs";
 import { dirname, join, resolve } from "node:path";
-import { isReachable, DEFAULT_SERVER, createSpaceAuth, serverConfig, openServerConfig, validateTlsMaterial, mintCreds, mintLifecycleUid, DEV_OWNER, mintConnectionEvictorCreds, mintMembershipObserverCreds, newIdentity, setupSpaceStreams, reconcileSpaceTtls, ttlBuckets, standaloneConnectOpts, requireBrokerFloor, parseServerVersion, presenceBucket, seedChannelRegistry, ensureDefaultDeliveryClass, mkSecretDir, writeSecretFile, } from "@cotal-ai/core";
+import { isReachable, DEFAULT_SERVER, createSpaceAuth, serverConfig, openServerConfig, validateTlsMaterial, mintCreds, mintLifecycleUid, DEV_OWNER, mintConnectionEvictorCreds, mintMembershipObserverCreds, newIdentity, setupSpaceStreams, reconcileSpaceTtls, ttlBuckets, standaloneConnectOpts, requireBrokerFloor, parseServerVersion, presenceBucket, seedChannelRegistry, ensureDefaultDeliveryClass, LEASE_TTL_MS, mkSecretDir, writeSecretFile, } from "@cotal-ai/core";
 import { connect } from "@nats-io/transport-node";
 import { jetstreamManager, StorageType } from "@nats-io/jetstream";
 import { Kvm } from "@nats-io/kv";
@@ -17,7 +17,7 @@ import { resolveNatsServer } from "../lib/nats-bin.js";
 import { cotalPath, cotalRoot } from "../lib/paths.js";
 import { renderDetachedSummary } from "../lib/up-report.js";
 import { detachedSystemdSupervisionWarning } from "../lib/systemd-supervision.js";
-import { deliveryUp, ensureControlPlane, stopDelivery } from "../lib/delivery-proc.js";
+import { deliveryStoppedByDown, deliveryUp, ensureControlPlane, ensureDelivery, stopDelivery } from "../lib/delivery-proc.js";
 import { RESPONDER_UNBOUND_CONSEQUENCE } from "../lib/delivery-responder.js";
 import { displayCmd } from "../lib/self-exec.js";
 import { liveManagerWouldApplyMaxSessions, managerHasDeliveryMarker, managerLogDisplayPath, managerRecordState, managerUp, stopManager } from "../lib/manager-proc.js";
@@ -26,6 +26,7 @@ import { loadManifest } from "../lib/manifest/index.js";
 import { buildLaunchSpec, genRunId, manifestToChannels, preflightConnectors, writeLaunchSpec } from "../lib/manifest/apply.js";
 import { renderUpPlan, renderInherited, renderWarnings } from "../lib/manifest/render.js";
 import { failManifest } from "./topology.js";
+import { reserveStop } from "./down.js";
 import { extensionNames, preflightRuntime } from "../ext-loader.js";
 import { completingFlagValue } from "../lib/completion.js";
 import { askManager } from "../lib/control.js";
@@ -1047,8 +1048,11 @@ async function runUp(args, inheritedLock, onAdopt) {
             void (async () => {
                 const managerContext = { root: cotalRoot(), space };
                 let spared;
+                let spareSeats;
                 let legacyManagerSpareUnverified = false;
                 const managerPidPath = localProcessPath(MANAGER_PIDFILE, managerContext);
+                const stopMarker = `${managerPidPath}.stopping`;
+                let reserved = false;
                 if (existsSync(managerPidPath)) {
                     const pin = verifyIdentityPin(managerPidPath);
                     if (pin.kind === "legacy")
@@ -1056,7 +1060,12 @@ async function runUp(args, inheritedLock, onAdopt) {
                     else if (pin.kind === "match") {
                         spared = await listManagerSeatsForSpare(managerContext);
                         try {
-                            assertManagerCanSpare(managerContext, undefined, pin.record);
+                            // Taken before the capability read and held until the manager is gone, as bare `down`
+                            // holds it, so a concurrent `cotal down` can neither stop this manager nor arm a reap
+                            // while this stop is in flight.
+                            reserveStop("manager", stopMarker);
+                            reserved = true;
+                            spareSeats = assertManagerCanSpare(managerContext, undefined, pin.record);
                         }
                         catch (e) {
                             // THE CAPABILITY ASSERT IS THE SIGNAL GATE, the same rule bare `down` enforces inside
@@ -1065,6 +1074,8 @@ async function runUp(args, inheritedLock, onAdopt) {
                             // release the latch so the stack keeps running in the foreground; the operator ends it
                             // with `cotal down --with-agents` from another terminal, and the broker-exit handler
                             // below already ends `up` when the broker goes.
+                            if (reserved)
+                                rmSync(stopMarker, { force: true });
                             console.error(c.red(`! teardown: ${e.message}`));
                             console.error(c.red(`the stack is still running; to take managed agents with it, run: cotal down --with-agents`));
                             stopping = false;
@@ -1078,11 +1089,15 @@ async function runUp(args, inheritedLock, onAdopt) {
                     if (legacyManagerSpareUnverified)
                         printLegacyManagerSpareUncertainty();
                     else if (spared)
-                        printSparedAgents(spared);
+                        printSparedAgents(spared, spareSeats);
                 }
                 catch (e) {
                     console.error(`! manager teardown: ${e.message}`);
                 }
+                finally {
+                    if (reserved)
+                        rmSync(stopMarker, { force: true });
+                }
                 await stopAuthService(space).catch((e) => console.error(`! auth teardown: ${e.message}`));
                 child.kill("SIGTERM");
             })();
@@ -1186,11 +1201,62 @@ async function runUp(args, inheritedLock, onAdopt) {
                 storeDir,
                 ts: new Date().toISOString(),
             }, "started");
+            // A DAEMON THAT DIES UNDER THIS RUNNING BROKER IS RESTARTED HERE (#2469). It exits on its own once
+            // it cannot reach the broker, and a starved host makes a running broker look unreachable, so it can
+            // end while this broker serves on. Nothing else brings it back, and every static retirement then
+            // fails on the ctl.delivery-admin rail and holds its name until an operator re-runs `cotal up`. A
+            // clean exit or SIGTERM/SIGINT is a deliberate stop (`cotal down delivery`) and stays stopped, and so
+            // is the SIGKILL `down` escalates to when a starved daemon outlives its grace, also when either ends a
+            // replacement this loop is still waiting on: that attempt then fails, and retrying it would undo the
+            // stop. A failed attempt waits out the lease TTL, the longest a dead holder's lease blocks its
+            // replacement. No daemon is alive during that wait for `down` to signal, so a stop there shows only as
+            // the dead daemon's record going away. Recovery is announced only once the responder is bound.
+            const meshServing = () => !stopping && child.exitCode === null && child.signalCode === null;
+            let restartingDelivery = false;
+            let deliveryStopped = false;
+            const onDeliveryExit = (code, signal) => {
+                if (code === 0 || signal === "SIGTERM" || signal === "SIGINT" || (signal !== null && deliveryStoppedByDown(space))) {
+                    deliveryStopped = true;
+                    return;
+                }
+                if (restartingDelivery || !meshServing())
+                    return;
+                restartingDelivery = true;
+                console.error(c.yellow(`! delivery daemon exited (${signal ? `signal ${signal}` : `code ${code}`}) while nats-server is running - restarting it`));
+                void (async () => {
+                    while (meshServing()) {
+                        let failure;
+                        try {
+                            const ensured = await ensureDelivery({ space, server, tls: transport.kind === "tls-required", onDeliveryExit });
+                            // An abnormal exit during this attempt was dropped above, so the record decides.
+                            if (deliveryUp(space)) {
+                                // An unbound responder was reported by ensureDelivery, and this daemon's exit re-arms the restart.
+                                if (ensured.responderBound)
+                                    console.error(c.green(`✓ delivery daemon running again${ensured.pid !== undefined ? ` (pid ${ensured.pid})` : ""}`));
+                                break;
+                            }
+                            failure = "no delivery daemon is running after the attempt";
+                        }
+                        catch (e) {
+                            failure = e.message;
+                        }
+                        if (deliveryStopped)
+                            break;
+                        console.error(c.yellow(`! delivery restart failed, retrying in ${LEASE_TTL_MS / 1000}s: ${failure}`));
+                        const recorded = !deliveryStoppedByDown(space);
+                        await new Promise((r) => setTimeout(r, LEASE_TTL_MS));
+                        if (deliveryStopped || (recorded && deliveryStoppedByDown(space)))
+                            break;
+                    }
+                    restartingDelivery = false;
+                })();
+            };
             // Bring up the delivery daemon WITH the server (auth mode only — it self-gates on `.cotal/auth`).
             // It is part of the server, so `cotal up` starts it by default; open dev mode has no daemon.
             // Class-2 credential renewal is NOT wired here: the MANAGER is the renewal owner (it is resident
             // in every mesh mode — foreground, --detach, refresh — where this foreground process is not).
             const controlPlane = await startDeliveryWithBroker(space, server, transport.kind === "tls-required", {
+                onDeliveryExit,
                 runtime: values.runtime,
                 // The address the broker was bound to. This is what lets `cotal attach` reach this manager
                 // from another machine; without it the attach face stays loopback-only, so exposing terminals