@cotal-ai/cli 0.85.0 → 0.87.0

package/dist/commands/up.js

@@ -20,7 +20,7 @@ import { detachedSystemdSupervisionWarning } from "../lib/systemd-supervision.js
 import { deliveryStoppedByDown, deliveryUp, ensureControlPlane, ensureDelivery, stopDelivery } from "../lib/delivery-proc.js";
 import { RESPONDER_UNBOUND_CONSEQUENCE } from "../lib/delivery-responder.js";
 import { displayCmd } from "../lib/self-exec.js";
-import { liveManagerWouldApplyMaxSessions, managerHasDeliveryMarker, managerLogDisplayPath, managerRecordState, managerUp, stopManager } from "../lib/manager-proc.js";
+import { clearManagerArtifacts, liveManagerWouldApplyMaxSessions, managerHasDeliveryMarker, managerLogDisplayPath, managerRecordState, managerUp, stopManager } from "../lib/manager-proc.js";
 import { loadManifest } from "../lib/manifest/index.js";
 import { buildLaunchSpec, genRunId, manifestToChannels, preflightConnectors, writeLaunchSpec } from "../lib/manifest/apply.js";
 import { renderUpPlan, renderInherited, renderWarnings } from "../lib/manifest/render.js";
@@ -1034,6 +1034,7 @@ async function runUp(args, inheritedLock, onAdopt) {
             teardown = (async () => {
                 try {
                     await stopManager(space);
+                    clearManagerArtifacts(space);
                 }
                 catch (e) {
                     console.error(c.red(`! teardown: ${e.message}`));
@@ -1058,7 +1059,7 @@ async function runUp(args, inheritedLock, onAdopt) {
             // that one holds, so wait for a Ctrl-C teardown already running instead of racing it.
             await teardown;
             await stopDelivery(space).catch((e) => console.error(`! delivery teardown: ${e.message}`));
-            await stopManager(space).catch((e) => console.error(`! manager teardown: ${e.message}`));
+            await stopManager(space).then(() => clearManagerArtifacts(space)).catch((e) => console.error(`! manager teardown: ${e.message}`));
             await stopAuthService(space).catch((e) => console.error(`! auth teardown: ${e.message}`));
             // Unrecording (and the exit code below) both depend on WHY the broker is gone. `stopping` is
             // true only for the intended shutdown `stop()` drives (Ctrl-C, SIGTERM) — everything else here
@@ -1363,29 +1364,31 @@ function readNatsInfo(endpoint, timeoutMs = 2_000) {
         socket.on("close", () => finish(new Error("restore listener closed before NATS INFO")));
     });
 }
-async function provePreparedRestoreListener(prepared) {
-    const proof = prepared.listenerProof;
+/** Prove a recovered attempt's live bound listener IS ours end-to-end before adoption: launch
+ *  binding, durable journal match, local pid + exact pidfile, store identity, mesh identity, the
+ *  raw NATS INFO server name, and an authenticated connect confirming name + JetStream. Each attempt
+ *  kind supplies only its journal match, which yields the store identity the journal recorded. */
+async function proveBoundListener(kind, attempt, proof, journalStore, store) {
+    const label = `${kind} attempt ${attempt.attemptId}`;
     if (!proof)
-        throw new Error(`restore attempt ${prepared.attemptId} has no bound listener proof`);
-    if (proof.attemptId !== prepared.attemptId || proof.serverName !== prepared.serverName ||
-        proof.serverNonce !== prepared.serverNonce || proof.serverEndpoint !== prepared.server ||
+        throw new Error(`${label} has no bound listener proof`);
+    if (proof.attemptId !== attempt.attemptId || proof.serverName !== attempt.serverName ||
+        proof.serverNonce !== attempt.serverNonce || proof.serverEndpoint !== attempt.server ||
         proof.serverName !== `${proof.attemptId}-${proof.serverNonce}` || !/^[0-9a-f]{32}$/.test(proof.serverNonce) ||
-        proof.processOwner.id !== `restore-listener-${proof.serverNonce}`)
-        throw new Error(`restore attempt ${prepared.attemptId} listener proof does not match its launch record`);
-    const journal = readMaintenanceJournal(prepared.root);
-    if (!journal || (journal.state !== "commit-intent" && journal.state !== "manager-committed" && journal.state !== "degraded" && journal.state !== "active") ||
-        journal.restore.attemptId !== prepared.attemptId || !journal.listenerProof ||
-        !sameRestoreListenerProof(journal.listenerProof, proof) || journal.launch.server !== proof.serverEndpoint)
-        throw new Error(`restore attempt ${prepared.attemptId} listener proof does not exactly match durable recovery state`);
+        proof.processOwner.id !== `${kind}-listener-${proof.serverNonce}`)
+        throw new Error(`${label} listener proof does not match its launch record`);
+    const recorded = journalStore(proof);
+    if (!recorded)
+        throw new Error(`${label} listener proof does not exactly match durable recovery state`);
     if (proof.processOwner.host !== hostname())
-        throw new Error(`restore attempt ${prepared.attemptId} listener process ownership is not live and local`);
+        throw new Error(`${label} listener process ownership is not live and local`);
     try {
         process.kill(proof.processOwner.pid, 0);
     }
     catch {
-        throw new Error(`restore attempt ${prepared.attemptId} listener process ownership is not live and local`);
+        throw new Error(`${label} listener process ownership is not live and local`);
     }
-    const pidPath = join(prepared.root, ".cotal", "nats.pid");
+    const pidPath = join(attempt.root, ".cotal", "nats.pid");
     let exactPidFile = false;
     try {
         const stat = lstatSync(pidPath);
@@ -1394,33 +1397,42 @@ async function provePreparedRestoreListener(prepared) {
     }
     catch { /* absent or unprovable */ }
     if (!exactPidFile)
-        throw new Error(`restore attempt ${prepared.attemptId} listener pidfile does not match its process proof`);
-    const target = readStoreIdentity(prepared.targetPath);
-    if (!sameStoreIdentity(target, proof.target) || !sameStoreIdentity(journal.restore.target, proof.target))
-        throw new Error(`restore attempt ${prepared.attemptId} target identity changed`);
-    const mesh = findMesh(prepared.space);
-    if (mesh && (mesh.server !== prepared.server || mesh.root !== prepared.root || mesh.mode !== prepared.mode))
-        throw new Error(`restore attempt ${prepared.attemptId} conflicts with the recorded mesh identity`);
+        throw new Error(`${label} listener pidfile does not match its process proof`);
+    if (!sameStoreIdentity(readStoreIdentity(store.path), proof.target) || !sameStoreIdentity(recorded, proof.target))
+        throw new Error(`${label} ${store.changed}`);
+    const mesh = findMesh(attempt.space);
+    if (mesh && (mesh.server !== attempt.server || mesh.root !== attempt.root || mesh.mode !== attempt.mode))
+        throw new Error(`${label} conflicts with the recorded mesh identity`);
     const info = await readNatsInfo(proof.serverEndpoint);
     if (info.server_name !== proof.serverName)
-        throw new Error(`restore attempt ${prepared.attemptId} reached a foreign NATS server name/nonce`);
-    const auth = await resumeControlAuth(prepared.root, prepared.mode);
+        throw new Error(`${label} reached a foreign NATS server name/nonce`);
+    const auth = await resumeControlAuth(attempt.root, attempt.mode);
     const nc = await connect({
-        servers: prepared.server,
+        servers: attempt.server,
         ...standaloneConnectOpts({ ...auth, /* not yet wired to a recorded transport - see broker-policy/MeshEntry work */ tls: false }),
         maxReconnectAttempts: 0,
     });
     try {
         if (nc.info?.server_name !== proof.serverName)
-            throw new Error(`restore attempt ${prepared.attemptId} authenticated to a foreign NATS server name/nonce`);
+            throw new Error(`${label} authenticated to a foreign NATS server name/nonce`);
         if (!nc.info.jetstream)
-            throw new Error(`restore attempt ${prepared.attemptId} listener has no JetStream`);
+            throw new Error(`${label} listener has no JetStream`);
     }
     finally {
         await nc.drain().catch(() => { });
     }
     return proof;
 }
+function provePreparedRestoreListener(prepared) {
+    return proveBoundListener("restore", prepared, prepared.listenerProof, (proof) => {
+        const journal = readMaintenanceJournal(prepared.root);
+        if (!journal || (journal.state !== "commit-intent" && journal.state !== "manager-committed" && journal.state !== "degraded" && journal.state !== "active") ||
+            journal.restore.attemptId !== prepared.attemptId || !journal.listenerProof ||
+            !sameRestoreListenerProof(journal.listenerProof, proof) || journal.launch.server !== proof.serverEndpoint)
+            return undefined;
+        return journal.restore.target;
+    }, { path: prepared.targetPath, changed: "target identity changed" });
+}
 function bindSpawnedOrdinaryResumeListener(pending, pid, startedAt, heldLock) {
     if (!Number.isInteger(pid) || pid <= 0)
         throw new Error("resume listener spawn returned no pid");
@@ -1443,66 +1455,15 @@ function bindSpawnedOrdinaryResumeListener(pending, pid, startedAt, heldLock) {
             releaseMaintenanceLock(lock);
     }
 }
-/** Prove the recovered attempt's live bound listener IS ours end-to-end before adoption: launch
- *  binding, durable journal match, local pid + exact pidfile, source identity, mesh identity, the
- *  raw NATS INFO server name, and an authenticated connect confirming name + JetStream. */
-async function proveOrdinaryResumeListener(pending) {
-    const proof = pending.adoptProof;
-    if (!proof)
-        throw new Error(`resume attempt ${pending.attemptId} has no bound listener proof`);
-    if (proof.attemptId !== pending.attemptId || proof.serverName !== pending.serverName ||
-        proof.serverNonce !== pending.serverNonce || proof.serverEndpoint !== pending.server ||
-        proof.serverName !== `${proof.attemptId}-${proof.serverNonce}` || !/^[0-9a-f]{32}$/.test(proof.serverNonce) ||
-        proof.processOwner.id !== `resume-listener-${proof.serverNonce}`)
-        throw new Error(`resume attempt ${pending.attemptId} listener proof does not match its launch record`);
-    const journal = readMaintenanceJournal(pending.root);
-    if (!journal || !("ordinaryResume" in journal) || journal.ordinaryResume.attemptId !== pending.attemptId ||
-        !journal.listenerProof || !sameRestoreListenerProof(journal.listenerProof, proof) ||
-        journal.ordinaryResume.launch.server !== proof.serverEndpoint)
-        throw new Error(`resume attempt ${pending.attemptId} listener proof does not exactly match durable recovery state`);
-    if (proof.processOwner.host !== hostname())
-        throw new Error(`resume attempt ${pending.attemptId} listener process ownership is not live and local`);
-    try {
-        process.kill(proof.processOwner.pid, 0);
-    }
-    catch {
-        throw new Error(`resume attempt ${pending.attemptId} listener process ownership is not live and local`);
-    }
-    const pidPath = join(pending.root, ".cotal", "nats.pid");
-    let exactPidFile = false;
-    try {
-        const stat = lstatSync(pidPath);
-        exactPidFile = stat.isFile() && !stat.isSymbolicLink() &&
-            readFileSync(pidPath, "utf8") === String(proof.processOwner.pid);
-    }
-    catch { /* absent or unprovable */ }
-    if (!exactPidFile)
-        throw new Error(`resume attempt ${pending.attemptId} listener pidfile does not match its process proof`);
-    const source = readStoreIdentity(pending.storeDir);
-    if (!sameStoreIdentity(source, proof.target) || !sameStoreIdentity(journal.source, proof.target))
-        throw new Error(`resume attempt ${pending.attemptId} preserved source identity changed`);
-    const mesh = findMesh(pending.space);
-    if (mesh && (mesh.server !== pending.server || mesh.root !== pending.root || mesh.mode !== pending.mode))
-        throw new Error(`resume attempt ${pending.attemptId} conflicts with the recorded mesh identity`);
-    const info = await readNatsInfo(proof.serverEndpoint);
-    if (info.server_name !== proof.serverName)
-        throw new Error(`resume attempt ${pending.attemptId} reached a foreign NATS server name/nonce`);
-    const auth = await resumeControlAuth(pending.root, pending.mode);
-    const nc = await connect({
-        servers: pending.server,
-        ...standaloneConnectOpts({ ...auth, /* not yet wired to a recorded transport - see broker-policy/MeshEntry work */ tls: false }),
-        maxReconnectAttempts: 0,
-    });
-    try {
-        if (nc.info?.server_name !== proof.serverName)
-            throw new Error(`resume attempt ${pending.attemptId} authenticated to a foreign NATS server name/nonce`);
-        if (!nc.info.jetstream)
-            throw new Error(`resume attempt ${pending.attemptId} listener has no JetStream`);
-    }
-    finally {
-        await nc.drain().catch(() => { });
-    }
-    return proof;
+function proveOrdinaryResumeListener(pending) {
+    return proveBoundListener("resume", pending, pending.adoptProof, (proof) => {
+        const journal = readMaintenanceJournal(pending.root);
+        if (!journal || !("ordinaryResume" in journal) || journal.ordinaryResume.attemptId !== pending.attemptId ||
+            !journal.listenerProof || !sameRestoreListenerProof(journal.listenerProof, proof) ||
+            journal.ordinaryResume.launch.server !== proof.serverEndpoint)
+            return undefined;
+        return journal.source;
+    }, { path: pending.storeDir, changed: "preserved source identity changed" });
 }
 /** Light wire check for a freshly SPAWNED ordinary-resume listener: the greeted server must carry
  *  the attempt's exact name/nonce (the full seven-point prove is for adopting a survivor). */
@@ -1962,6 +1923,7 @@ async function upManifest(file, opts) {
     // lives at this address) would win the fresh mesh's lease and the launch manager would refuse.
     // Stop it, so the manager started below WITH the launch spec is THE manager.
     await stopManager(m.space);
+    clearManagerArtifacts(m.space);
     let pid;
     let controlPlane = false;
     let authService = true;