@cotal-ai/cli 0.68.0 → 0.69.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/describe.d.ts.map +1 -1
- package/dist/commands/describe.js +2 -2
- package/dist/commands/describe.js.map +1 -1
- package/dist/commands/down.d.ts +0 -23
- package/dist/commands/down.d.ts.map +1 -1
- package/dist/commands/down.js +8 -217
- package/dist/commands/down.js.map +1 -1
- package/dist/commands/endpoints.js +2 -2
- package/dist/commands/endpoints.js.map +1 -1
- package/dist/commands/meshes.d.ts.map +1 -1
- package/dist/commands/meshes.js +5 -3
- package/dist/commands/meshes.js.map +1 -1
- package/dist/commands/mint.d.ts.map +1 -1
- package/dist/commands/mint.js +8 -2
- package/dist/commands/mint.js.map +1 -1
- package/dist/commands/setup.d.ts.map +1 -1
- package/dist/commands/setup.js +4 -4
- package/dist/commands/setup.js.map +1 -1
- package/dist/commands/spawn.d.ts.map +1 -1
- package/dist/commands/spawn.js +98 -64
- package/dist/commands/spawn.js.map +1 -1
- package/dist/commands/status.d.ts.map +1 -1
- package/dist/commands/status.js +3 -4
- package/dist/commands/status.js.map +1 -1
- package/dist/commands/sync.d.ts.map +1 -1
- package/dist/commands/sync.js +31 -25
- package/dist/commands/sync.js.map +1 -1
- package/dist/commands/up.d.ts +0 -2
- package/dist/commands/up.d.ts.map +1 -1
- package/dist/commands/up.js +139 -199
- package/dist/commands/up.js.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -9
- package/dist/index.js.map +1 -1
- package/dist/lib/control.d.ts +1 -13
- package/dist/lib/control.d.ts.map +1 -1
- package/dist/lib/control.js +1 -30
- package/dist/lib/control.js.map +1 -1
- package/dist/lib/delivery-proc.d.ts +1 -2
- package/dist/lib/delivery-proc.d.ts.map +1 -1
- package/dist/lib/delivery-proc.js +10 -9
- package/dist/lib/delivery-proc.js.map +1 -1
- package/dist/lib/local-process-stop.d.ts +29 -0
- package/dist/lib/local-process-stop.d.ts.map +1 -0
- package/dist/lib/local-process-stop.js +196 -0
- package/dist/lib/local-process-stop.js.map +1 -0
- package/dist/lib/manager-proc.d.ts +17 -17
- package/dist/lib/manager-proc.d.ts.map +1 -1
- package/dist/lib/manager-proc.js +57 -107
- package/dist/lib/manager-proc.js.map +1 -1
- package/dist/lib/manifest/ledger.js +1 -1
- package/dist/lib/manifest/prepare.d.ts.map +1 -1
- package/dist/lib/manifest/prepare.js +2 -2
- package/dist/lib/manifest/prepare.js.map +1 -1
- package/dist/lib/self-exec.d.ts +1 -1
- package/dist/lib/self-exec.js +1 -1
- package/dist/lib/status.d.ts +0 -1
- package/dist/lib/status.d.ts.map +1 -1
- package/dist/lib/status.js +2 -25
- package/dist/lib/status.js.map +1 -1
- package/dist/lib/teardown-spare.js +3 -3
- package/dist/lib/teardown-spare.js.map +1 -1
- package/dist/ui.d.ts +9 -9
- package/dist/ui.d.ts.map +1 -1
- package/dist/ui.js +16 -18
- package/dist/ui.js.map +1 -1
- package/package.json +4 -4
package/dist/commands/up.js
CHANGED
|
@@ -8,7 +8,7 @@ import { isReachable, DEFAULT_SERVER, createSpaceAuth, serverConfig, openServerC
|
|
|
8
8
|
import { connect } from "@nats-io/transport-node";
|
|
9
9
|
import { jetstreamManager, StorageType } from "@nats-io/jetstream";
|
|
10
10
|
import { Kvm } from "@nats-io/kv";
|
|
11
|
-
import { assertSingleSpaceBroker, assertUserAuthInfo, authDir, getSoleSpaceAuth, getSpaceAuth, hasUserAuthState, listSpaceAccounts, preloadSpaceAccounts, putSpaceAuth, clearCurrent, findMesh, getCurrent, loadMeshes, connectionEvictorCredsKey, membershipConfigPath, membershipObserverCredsKey, membershipRwCredsKey, MEMBERSHIP_CONFIG_KIND, MEMBERSHIP_RW_CREDS_KIND, parsePositiveIntegerFlag, recordMesh, meshesForRoot, removeMesh, rotateSystemCreds, setCurrent, staleSystemCreds, SYSTEM_CREDS_FILES, userAuthStateDir, userAuthSpacesOnDisk, workspaceSecretStore, acquireMaintenanceLock, assertStoreIdentity, assessRestoreClaim, beginOrdinaryResume, bindOrdinaryResumeListener, consumeRetiredMaintenance, markOrdinaryResumeActive, markOrdinaryResumeDegraded, replaceDeadOrdinaryResumeListener, localProcessOwnerStatus, readMaintenanceJournal, readMaintenanceResumeDocument, seatCheckpointDir, readStoreIdentity, recordOrdinaryResumeManagerCommit, releaseMaintenanceLock, retireOrdinaryResume, sameStoreIdentity, readBrokerPolicy, writeBrokerPolicy, removePidPair, writePidPair,
|
|
11
|
+
import { assertSingleSpaceBroker, assertUserAuthInfo, authDir, getSoleSpaceAuth, getSpaceAuth, hasUserAuthState, listSpaceAccounts, preloadSpaceAccounts, putSpaceAuth, clearCurrent, findMesh, getCurrent, loadMeshes, connectionEvictorCredsKey, membershipConfigPath, membershipObserverCredsKey, membershipRwCredsKey, MEMBERSHIP_CONFIG_KIND, MEMBERSHIP_RW_CREDS_KIND, parsePositiveIntegerFlag, recordMesh, meshesForRoot, removeMesh, rotateSystemCreds, setCurrent, staleSystemCreds, SYSTEM_CREDS_FILES, userAuthStateDir, userAuthSpacesOnDisk, workspaceSecretStore, acquireMaintenanceLock, assertStoreIdentity, assessRestoreClaim, beginOrdinaryResume, bindOrdinaryResumeListener, consumeRetiredMaintenance, markOrdinaryResumeActive, markOrdinaryResumeDegraded, replaceDeadOrdinaryResumeListener, localProcessOwnerStatus, readMaintenanceJournal, readMaintenanceResumeDocument, seatCheckpointDir, readStoreIdentity, recordOrdinaryResumeManagerCommit, releaseMaintenanceLock, retireOrdinaryResume, sameStoreIdentity, readBrokerPolicy, writeBrokerPolicy, removePidPair, writePidPair, isLoopbackHost, connectUserControlOrThrow, userViewAuth, } from "@cotal-ai/workspace";
|
|
12
12
|
import { ensureAuthService, resolveAuthProvider, stopAuthService } from "../lib/auth-proc.js";
|
|
13
13
|
import { resolveSpace } from "../lib/status.js";
|
|
14
14
|
import { c } from "../ui.js";
|
|
@@ -21,12 +21,10 @@ import { deliveryStoppedByDown, deliveryUp, ensureControlPlane, ensureDelivery,
|
|
|
21
21
|
import { RESPONDER_UNBOUND_CONSEQUENCE } from "../lib/delivery-responder.js";
|
|
22
22
|
import { displayCmd } from "../lib/self-exec.js";
|
|
23
23
|
import { liveManagerWouldApplyMaxSessions, managerHasDeliveryMarker, managerLogDisplayPath, managerRecordState, managerUp, stopManager } from "../lib/manager-proc.js";
|
|
24
|
-
import { listManagerSeatsForSpare, printLegacyManagerSpareUncertainty, printSparedAgents } from "../lib/teardown-spare.js";
|
|
25
24
|
import { loadManifest } from "../lib/manifest/index.js";
|
|
26
25
|
import { buildLaunchSpec, genRunId, manifestToChannels, preflightConnectors, writeLaunchSpec } from "../lib/manifest/apply.js";
|
|
27
26
|
import { renderUpPlan, renderInherited, renderWarnings } from "../lib/manifest/render.js";
|
|
28
27
|
import { failManifest } from "./topology.js";
|
|
29
|
-
import { reserveStop } from "./down.js";
|
|
30
28
|
import { extensionNames, preflightRuntime } from "../ext-loader.js";
|
|
31
29
|
import { completingFlagValue } from "../lib/completion.js";
|
|
32
30
|
import { askManager } from "../lib/control.js";
|
|
@@ -868,6 +866,12 @@ async function runUp(args, inheritedLock, onAdopt) {
|
|
|
868
866
|
});
|
|
869
867
|
if (!controlPlane)
|
|
870
868
|
process.exitCode = 1;
|
|
869
|
+
// #883: a refresh that HEALED a missing manager must say so — the summary line otherwise reads
|
|
870
|
+
// identically to a refresh that found everything already up, and the operator cannot tell which
|
|
871
|
+
// world they are in without a second command. Only here does a started manager mean one was
|
|
872
|
+
// missing: a launch starts its manager as a matter of course (#2480).
|
|
873
|
+
if (controlPlane?.started)
|
|
874
|
+
console.log(c.green(`✓ restored in the background: manager (pid ${controlPlane.pid})`));
|
|
871
875
|
}
|
|
872
876
|
// A broker was already answering here — this branch starts nothing, so it must not claim the
|
|
873
877
|
// record as ours (see `Provenance`), and it passes only what this invocation decided.
|
|
@@ -936,7 +940,6 @@ async function runUp(args, inheritedLock, onAdopt) {
|
|
|
936
940
|
verify: async () => { await verifySpawnedOrdinaryListener(ordinaryAttempt); },
|
|
937
941
|
},
|
|
938
942
|
} : {}),
|
|
939
|
-
skipPostStart: Boolean(resumeAttempt),
|
|
940
943
|
});
|
|
941
944
|
// Transport policy is committed inside startMeshDetached before delivery launch (S5+S9).
|
|
942
945
|
console.log(c.dim(`Started nats-server (${source}).`));
|
|
@@ -1023,19 +1026,22 @@ async function runUp(args, inheritedLock, onAdopt) {
|
|
|
1023
1026
|
if (!resumeAttempt)
|
|
1024
1027
|
process.exit(1);
|
|
1025
1028
|
});
|
|
1026
|
-
// The control plane is coupled to the broker: stop the
|
|
1029
|
+
// The control plane is coupled to the broker: stop the detached manager AND the delivery daemon
|
|
1027
1030
|
// (AND the space's user-auth service) when this `up` stops (Ctrl-C), so none outlives the broker
|
|
1028
1031
|
// it serves — a surviving manager would reconnect-loop invisibly against the dead (or the NEXT)
|
|
1029
1032
|
// broker (the documented orphan-supervisor failure mode). All kill by pidfile, symmetric; the
|
|
1030
1033
|
// auth service's pid is space-scoped so no other space's daemon can ever be hit.
|
|
1034
|
+
// The manager goes first, in `cotal down`'s order, through the stop `cotal down` runs (#1307): it
|
|
1035
|
+
// holds the stop reservation, asserts the exact manager can spare its seats before any signal,
|
|
1036
|
+
// reports the seats left behind, and escalates a wedged manager to SIGKILL. When that stop throws
|
|
1037
|
+
// the manager may still be serving, so NOTHING else is signalled — no delivery, no auth, no broker:
|
|
1038
|
+
// the refusal is printed with the reap route and the latch is released, so the stack keeps running
|
|
1039
|
+
// in the foreground; the operator ends it with `cotal down --with-agents` from another terminal,
|
|
1040
|
+
// and the broker-exit handler below already ends `up` when the broker goes.
|
|
1031
1041
|
// stopDelivery is the stop `cotal down delivery` performs (reservation, SIGKILL escalation); the
|
|
1032
1042
|
// rest of the teardown must run even if it fails — the failure is logged, never swallowed
|
|
1033
|
-
// silently.
|
|
1034
|
-
//
|
|
1035
|
-
// while the manager still answers, the spare capability of the exact recorded process is asserted
|
|
1036
|
-
// before any signal, and the seats left behind are reported with the reap route. Teardown is
|
|
1037
|
-
// AWAITED before the broker is signalled — the same order the broker-exit handler below uses — so
|
|
1038
|
-
// manager teardown never races a broker that is already going away.
|
|
1043
|
+
// silently. Teardown is AWAITED before the broker is signalled, so it never races a broker that is
|
|
1044
|
+
// already going away.
|
|
1039
1045
|
let stopping = false;
|
|
1040
1046
|
let teardown;
|
|
1041
1047
|
const stop = () => {
|
|
@@ -1047,58 +1053,16 @@ async function runUp(args, inheritedLock, onAdopt) {
|
|
|
1047
1053
|
if (child.exitCode !== null || child.signalCode !== null)
|
|
1048
1054
|
return;
|
|
1049
1055
|
teardown = (async () => {
|
|
1050
|
-
const managerContext = { root: cotalRoot(), space };
|
|
1051
|
-
let spared;
|
|
1052
|
-
let spareSeats;
|
|
1053
|
-
let legacyManagerSpareUnverified = false;
|
|
1054
|
-
const managerPidPath = localProcessPath(MANAGER_PIDFILE, managerContext);
|
|
1055
|
-
const stopMarker = `${managerPidPath}.stopping`;
|
|
1056
|
-
let reserved = false;
|
|
1057
|
-
if (existsSync(managerPidPath)) {
|
|
1058
|
-
const pin = verifyIdentityPin(managerPidPath);
|
|
1059
|
-
if (pin.kind === "legacy")
|
|
1060
|
-
legacyManagerSpareUnverified = true;
|
|
1061
|
-
else if (pin.kind === "match") {
|
|
1062
|
-
spared = await listManagerSeatsForSpare(managerContext);
|
|
1063
|
-
try {
|
|
1064
|
-
// Taken before the capability read and held until the manager is gone, as bare `down`
|
|
1065
|
-
// holds it, so a concurrent `cotal down` can neither stop this manager nor arm a reap
|
|
1066
|
-
// while this stop is in flight.
|
|
1067
|
-
reserveStop("manager", stopMarker);
|
|
1068
|
-
reserved = true;
|
|
1069
|
-
spareSeats = assertManagerCanSpare(managerContext, undefined, pin.record);
|
|
1070
|
-
}
|
|
1071
|
-
catch (e) {
|
|
1072
|
-
// THE CAPABILITY ASSERT IS THE SIGNAL GATE, the same rule bare `down` enforces inside
|
|
1073
|
-
// its beforeSignal hook: a throw there signals nothing. Signal NOTHING here either — no
|
|
1074
|
-
// manager, no delivery, no auth, no broker — print the refusal with the reap route, and
|
|
1075
|
-
// release the latch so the stack keeps running in the foreground; the operator ends it
|
|
1076
|
-
// with `cotal down --with-agents` from another terminal, and the broker-exit handler
|
|
1077
|
-
// below already ends `up` when the broker goes.
|
|
1078
|
-
if (reserved)
|
|
1079
|
-
rmSync(stopMarker, { force: true });
|
|
1080
|
-
console.error(c.red(`! teardown: ${e.message}`));
|
|
1081
|
-
console.error(c.red(`the stack is still running; to take managed agents with it, run: cotal down --with-agents`));
|
|
1082
|
-
stopping = false;
|
|
1083
|
-
return;
|
|
1084
|
-
}
|
|
1085
|
-
}
|
|
1086
|
-
}
|
|
1087
|
-
await stopDelivery(space).catch((e) => console.error(`! delivery teardown: ${e.message}`));
|
|
1088
1056
|
try {
|
|
1089
|
-
await stopManager(
|
|
1090
|
-
if (legacyManagerSpareUnverified)
|
|
1091
|
-
printLegacyManagerSpareUncertainty();
|
|
1092
|
-
else if (spared)
|
|
1093
|
-
printSparedAgents(spared, spareSeats);
|
|
1057
|
+
await stopManager(space);
|
|
1094
1058
|
}
|
|
1095
1059
|
catch (e) {
|
|
1096
|
-
console.error(`!
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
rmSync(stopMarker, { force: true });
|
|
1060
|
+
console.error(c.red(`! teardown: ${e.message}`));
|
|
1061
|
+
console.error(c.red(`the stack is still running; to take managed agents with it, run: cotal down --with-agents`));
|
|
1062
|
+
stopping = false;
|
|
1063
|
+
return;
|
|
1101
1064
|
}
|
|
1065
|
+
await stopDelivery(space).catch((e) => console.error(`! delivery teardown: ${e.message}`));
|
|
1102
1066
|
await stopAuthService(space).catch((e) => console.error(`! auth teardown: ${e.message}`));
|
|
1103
1067
|
child.kill("SIGTERM");
|
|
1104
1068
|
})();
|
|
@@ -1109,11 +1073,11 @@ async function runUp(args, inheritedLock, onAdopt) {
|
|
|
1109
1073
|
// so a later `cotal spawn` doesn't try to join a dead mesh.
|
|
1110
1074
|
child.on("exit", async (code, signal) => {
|
|
1111
1075
|
removePidPair(cotalPath("nats.pid"), String(child.pid));
|
|
1112
|
-
// One teardown at a time: a
|
|
1113
|
-
//
|
|
1076
|
+
// One teardown at a time: a stop started while another is in flight is refused the reservation
|
|
1077
|
+
// that one holds, so wait for a Ctrl-C teardown already running instead of racing it.
|
|
1114
1078
|
await teardown;
|
|
1115
1079
|
await stopDelivery(space).catch((e) => console.error(`! delivery teardown: ${e.message}`));
|
|
1116
|
-
await stopManager(
|
|
1080
|
+
await stopManager(space).catch((e) => console.error(`! manager teardown: ${e.message}`));
|
|
1117
1081
|
await stopAuthService(space).catch((e) => console.error(`! auth teardown: ${e.message}`));
|
|
1118
1082
|
// Unrecording (and the exit code below) both depend on WHY the broker is gone. `stopping` is
|
|
1119
1083
|
// true only for the intended shutdown `stop()` drives (Ctrl-C, SIGTERM) — everything else here
|
|
@@ -1164,46 +1128,7 @@ async function runUp(args, inheritedLock, onAdopt) {
|
|
|
1164
1128
|
console.error(c.yellow(`! nats-server ${v.version} is below 2.14.5 and the presence bucket of "${space}" is file-backed: it can latch after a stall and close the mesh to new joins (#1356); put a nats-server 2.14.5 or newer on PATH (it wins over the bundled one)`));
|
|
1165
1129
|
if (restored)
|
|
1166
1130
|
await provePreparedRestoreListener(restored);
|
|
1167
|
-
// Listener ready — commit the transport decision (S5: not before start).
|
|
1168
|
-
commitTransportPolicy(meshRoot, transport);
|
|
1169
1131
|
{
|
|
1170
|
-
if (!resumeAttempt)
|
|
1171
|
-
await postStart(server, space, setup, seedFile);
|
|
1172
|
-
else
|
|
1173
|
-
await recreateMemoryBuckets(server, space, setup);
|
|
1174
|
-
// USER MODE: the auth service comes up FIRST among the daemons — until its callout answers,
|
|
1175
|
-
// every user-mode connect to this broker is denied, so `up` must not report a usable user mesh
|
|
1176
|
-
// (nor let agents race it) on a half-started auth plane. (Foreground `up` doesn't exit here, so
|
|
1177
|
-
// `ok` has no exit code to carry — the red consequence line above is the operator signal.)
|
|
1178
|
-
const svc = await startUserAuthService(space, server, setup, publicExchange);
|
|
1179
|
-
// Record BEFORE the control plane comes up: the manager's fail-closed mode detection requires
|
|
1180
|
-
// an authoritative registry entry (marker-without-registry is a refused start, not a guess),
|
|
1181
|
-
// so the record must exist by the time it boots. A manager/delivery failure after this leaves
|
|
1182
|
-
// a recorded-but-degraded mesh — the documented, healable posture.
|
|
1183
|
-
// Resolve exposure BEFORE recording. This path also serves a RESUME of an already-recorded mesh
|
|
1184
|
-
// (`down --preserve-state` then a bare `up`), where the operator's `--host` lives only in the
|
|
1185
|
-
// registry — and the record below is written whole, so reading it afterwards would find the
|
|
1186
|
-
// field this very call had just erased.
|
|
1187
|
-
const effectiveAttachHost = attachHostFor(space, values.host);
|
|
1188
|
-
const effectiveMaxSessions = maxSessionsFor(space, maxSessions);
|
|
1189
|
-
recordOurMesh({
|
|
1190
|
-
space, server, root: cotalRoot(),
|
|
1191
|
-
mode: setup?.prepared ? "user" : useAuth ? "auth" : "open",
|
|
1192
|
-
// Written ALWAYS, as a boolean, unlike `attachHost` below. Absence and `false` resolve
|
|
1193
|
-
// identically for clients, so an omitted field would be indistinguishable from a deliberate
|
|
1194
|
-
// plaintext mesh — and this is the one field whose whole purpose is that the answer was
|
|
1195
|
-
// stated rather than defaulted.
|
|
1196
|
-
tlsRequired: transport.kind === "tls-required",
|
|
1197
|
-
...(svc.userAuth ? { userAuth: svc.userAuth } : {}),
|
|
1198
|
-
// Only a real decision is persisted — an explicit `--host` now, or one carried forward from a
|
|
1199
|
-
// previous launch. The bare case stays absent rather than recording the loopback default as
|
|
1200
|
-
// though the operator had chosen it.
|
|
1201
|
-
...(effectiveAttachHost ? { attachHost: effectiveAttachHost } : {}),
|
|
1202
|
-
...(effectiveMaxSessions !== undefined ? { maxSessions: effectiveMaxSessions } : {}),
|
|
1203
|
-
...(maxFileStore !== undefined ? { maxFileStore } : {}),
|
|
1204
|
-
storeDir,
|
|
1205
|
-
ts: new Date().toISOString(),
|
|
1206
|
-
}, "started");
|
|
1207
1132
|
// A DAEMON THAT DIES UNDER THIS RUNNING BROKER IS RESTARTED HERE (#2469). It exits on its own once
|
|
1208
1133
|
// it cannot reach the broker, and a starved host makes a running broker look unreachable, so it can
|
|
1209
1134
|
// end while this broker serves on. Nothing else brings it back, and every static retirement then
|
|
@@ -1254,26 +1179,21 @@ async function runUp(args, inheritedLock, onAdopt) {
|
|
|
1254
1179
|
restartingDelivery = false;
|
|
1255
1180
|
})();
|
|
1256
1181
|
};
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
runtime: values.runtime,
|
|
1264
|
-
// The address the broker was bound to. This is what lets `cotal attach` reach this manager
|
|
1265
|
-
// from another machine; without it the attach face stays loopback-only, so exposing terminals
|
|
1266
|
-
// is never a side effect of anything but the operator binding the mesh somewhere reachable.
|
|
1267
|
-
attachHost: effectiveAttachHost,
|
|
1268
|
-
maxSessions: effectiveMaxSessions,
|
|
1269
|
-
noManager,
|
|
1182
|
+
const { authService, controlPlane } = await serveReadyListener({
|
|
1183
|
+
child, server, space, storeDir, setup, seedFile, transport,
|
|
1184
|
+
host: values.host,
|
|
1185
|
+
maxSessions,
|
|
1186
|
+
maxFileStore,
|
|
1187
|
+
publicExchange,
|
|
1270
1188
|
resumeAttempt,
|
|
1271
1189
|
resumeCommitToken: restored?.managerCommit?.durableCommitToken ?? ordinaryAttempt?.managerCommit?.durableCommitToken,
|
|
1272
|
-
|
|
1190
|
+
runtime: values.runtime,
|
|
1191
|
+
noManager,
|
|
1192
|
+
onDeliveryExit,
|
|
1273
1193
|
});
|
|
1274
1194
|
if (restored && process.env.COTAL_SMOKE_FAIL_AFTER_RESTORE_LISTENER_READY === "1")
|
|
1275
1195
|
throw new Error("smoke-injected failure after restore listener readiness");
|
|
1276
|
-
await completeResumeActivation(resumeAttempt, controlPlane &&
|
|
1196
|
+
await completeResumeActivation(resumeAttempt, controlPlane !== undefined && authService, !authService ? "normal listener started but the user-auth service is unavailable" : "normal listener started but the control plane is degraded", server, startupLock);
|
|
1277
1197
|
activationFinished = true;
|
|
1278
1198
|
}
|
|
1279
1199
|
await new Promise(() => { });
|
|
@@ -1651,7 +1571,7 @@ async function resumeProvenOrdinaryListener(pending, heldLock) {
|
|
|
1651
1571
|
resumeAttempt: pending.attemptId,
|
|
1652
1572
|
resumeCommitToken: pending.managerCommit?.durableCommitToken,
|
|
1653
1573
|
});
|
|
1654
|
-
await completeResumeActivation(pending.attemptId, controlPlane && svc.ok, !svc.ok ? "adopted resume listener has no user-auth service" : "adopted resume listener has a degraded control plane", pending.server, heldLock);
|
|
1574
|
+
await completeResumeActivation(pending.attemptId, controlPlane !== undefined && svc.ok, !svc.ok ? "adopted resume listener has no user-auth service" : "adopted resume listener has a degraded control plane", pending.server, heldLock);
|
|
1655
1575
|
}
|
|
1656
1576
|
async function ensureRecoveredUserAuth(prepared) {
|
|
1657
1577
|
if (prepared.mode !== "user")
|
|
@@ -1697,7 +1617,7 @@ async function resumeProvenRestoreListener(prepared, heldLock) {
|
|
|
1697
1617
|
resumeAttempt: prepared.attemptId,
|
|
1698
1618
|
resumeCommitToken: prepared.managerCommit?.durableCommitToken,
|
|
1699
1619
|
});
|
|
1700
|
-
await completeResumeActivation(prepared.attemptId, controlPlane && svc.ok, !svc.ok ? "proven restore listener has no user-auth service" : "proven restore listener has a degraded control plane", prepared.server, heldLock);
|
|
1620
|
+
await completeResumeActivation(prepared.attemptId, controlPlane !== undefined && svc.ok, !svc.ok ? "proven restore listener has no user-auth service" : "proven restore listener has a degraded control plane", prepared.server, heldLock);
|
|
1701
1621
|
}
|
|
1702
1622
|
async function completeResumeActivation(attemptId, healthy, reason, server, heldLock) {
|
|
1703
1623
|
if (!attemptId)
|
|
@@ -2061,7 +1981,7 @@ async function upManifest(file, opts) {
|
|
|
2061
1981
|
// A leftover detached manager (its broker is gone — the reachability check above proved nothing
|
|
2062
1982
|
// lives at this address) would win the fresh mesh's lease and the launch manager would refuse.
|
|
2063
1983
|
// Stop it, so the manager started below WITH the launch spec is THE manager.
|
|
2064
|
-
await stopManager(
|
|
1984
|
+
await stopManager(m.space);
|
|
2065
1985
|
let pid;
|
|
2066
1986
|
let controlPlane = false;
|
|
2067
1987
|
let authService = true;
|
|
@@ -2131,8 +2051,9 @@ function applyUpOverrides(prepared, o) {
|
|
|
2131
2051
|
* cotal_spawn find a manager without any setup side effect. Coupled to the broker by the
|
|
2132
2052
|
* daemon's watchdog + the `up`/`down` teardown. `mgr` rides through to the manager start — the
|
|
2133
2053
|
* `up -f` path hands THE manager its runtime + resolved launch spec here (one manager per space,
|
|
2134
|
-
* so the launch can never be a second supervise). Returns
|
|
2135
|
-
* caller whose output claims a manager (the `up -f` launching
|
|
2054
|
+
* so the launch can never be a second supervise). Returns the control plane it ensured, or
|
|
2055
|
+
* `undefined` when it is degraded, so a caller whose output claims a manager (the `up -f` launching
|
|
2056
|
+
* line) can tell the truth. */
|
|
2136
2057
|
async function startDeliveryWithBroker(space, server,
|
|
2137
2058
|
/**
|
|
2138
2059
|
* Whether this listener requires TLS, decided by the CALLER.
|
|
@@ -2153,11 +2074,6 @@ async function startDeliveryWithBroker(space, server,
|
|
|
2153
2074
|
tlsRequired, mgr) {
|
|
2154
2075
|
try {
|
|
2155
2076
|
const plane = await ensureControlPlane({ space, server, tls: tlsRequired, ...(mgr ?? {}) });
|
|
2156
|
-
// #883: a refresh that HEALED a missing manager must say so — the summary line otherwise reads
|
|
2157
|
-
// identically to a refresh that found everything already up, and the operator cannot tell which
|
|
2158
|
-
// world they are in without a second command.
|
|
2159
|
-
if (plane.started)
|
|
2160
|
-
console.log(c.green(`✓ restored in the background: manager (pid ${plane.pid})`));
|
|
2161
2077
|
// #1576: `up` either BINDS the responder or SAYS SO HERE. The delivery daemon is a hard
|
|
2162
2078
|
// dependency of spawn, retirement and join, and this function used to return `true` for a boot
|
|
2163
2079
|
// that started a daemon whose responder never bound — so `cotal up` printed its success banner
|
|
@@ -2167,16 +2083,101 @@ tlsRequired, mgr) {
|
|
|
2167
2083
|
if (plane.responderBound === false)
|
|
2168
2084
|
console.error(c.yellow(`! delivery responder did not bind before this boot finished - ${RESPONDER_UNBOUND_CONSEQUENCE}`) +
|
|
2169
2085
|
c.dim(`\n The daemon process is running and the wait is open-ended; boot durable joins reconcile by themselves once it binds (agents do NOT need respawning).\n Watch it with \`${displayCmd()} status --components\`.`));
|
|
2170
|
-
return
|
|
2086
|
+
return plane;
|
|
2171
2087
|
}
|
|
2172
2088
|
catch (e) {
|
|
2173
2089
|
// Non-fatal (live messaging is unaffected) — but never SILENT: without the manager,
|
|
2174
2090
|
// `spawn --detach` / cotal_spawn have no responder, and the operator must hear it here,
|
|
2175
2091
|
// not as an unexplained "no manager reachable" later.
|
|
2176
2092
|
console.error(c.dim(`! control plane degraded: ${e.message} - durable delivery/manager may be down; start one with: cotal supervise`));
|
|
2177
|
-
return
|
|
2093
|
+
return undefined;
|
|
2178
2094
|
}
|
|
2179
2095
|
}
|
|
2096
|
+
/**
|
|
2097
|
+
* Everything a launch runs once its listener answers: the space setup, the user-auth service, the
|
|
2098
|
+
* mesh record, the transport commit and the control plane. Foreground `up` and
|
|
2099
|
+
* {@link startMeshDetached} both call it, so the two launch modes cannot drift apart (#2496).
|
|
2100
|
+
*
|
|
2101
|
+
* A fresh boot whose setup throws stops the listener and removes `nats.pid` before rethrowing the
|
|
2102
|
+
* original error. Nothing has recorded the mesh yet, so a listener left running would hold the port
|
|
2103
|
+
* with no registry entry for `cotal down` to reach. A channel seed with an unparseable value fails
|
|
2104
|
+
* here. A resume keeps its listener: its maintenance journal is already bound to it.
|
|
2105
|
+
*/
|
|
2106
|
+
async function serveReadyListener(l) {
|
|
2107
|
+
const { child, server, space, setup, transport } = l;
|
|
2108
|
+
if (l.resumeAttempt)
|
|
2109
|
+
await recreateMemoryBuckets(server, space, setup);
|
|
2110
|
+
else {
|
|
2111
|
+
try {
|
|
2112
|
+
await postStart(server, space, setup, l.seedFile);
|
|
2113
|
+
}
|
|
2114
|
+
catch (e) {
|
|
2115
|
+
try {
|
|
2116
|
+
child.kill("SIGTERM");
|
|
2117
|
+
}
|
|
2118
|
+
catch { /* already gone */ }
|
|
2119
|
+
try {
|
|
2120
|
+
removePidPair(cotalPath("nats.pid"), String(child.pid));
|
|
2121
|
+
}
|
|
2122
|
+
catch { /* best effort */ }
|
|
2123
|
+
throw e;
|
|
2124
|
+
}
|
|
2125
|
+
}
|
|
2126
|
+
// USER MODE: the auth service comes up FIRST among the daemons — until its callout answers,
|
|
2127
|
+
// every user-mode connect to this broker is denied, so `up` must not report a usable user mesh
|
|
2128
|
+
// (nor let agents race it) on a half-started auth plane.
|
|
2129
|
+
const svc = await startUserAuthService(space, server, setup, l.publicExchange);
|
|
2130
|
+
// Record BEFORE the control plane comes up: the manager's fail-closed mode detection requires
|
|
2131
|
+
// an authoritative registry entry (marker-without-registry is a refused start, not a guess),
|
|
2132
|
+
// so the record must exist by the time it boots. A manager/delivery failure after this leaves
|
|
2133
|
+
// a recorded-but-degraded mesh — the documented, healable posture.
|
|
2134
|
+
// Resolve exposure BEFORE recording. A RESUME of an already-recorded mesh (`down
|
|
2135
|
+
// --preserve-state` then a bare `up`) carries the operator's `--host` only in the registry, and
|
|
2136
|
+
// the record below is written whole, so reading it afterwards would find the field this very call
|
|
2137
|
+
// had just erased.
|
|
2138
|
+
const attachHost = attachHostFor(space, l.host);
|
|
2139
|
+
const maxSessions = maxSessionsFor(space, l.maxSessions);
|
|
2140
|
+
recordOurMesh({
|
|
2141
|
+
space, server, root: cotalRoot(),
|
|
2142
|
+
mode: setup?.prepared ? "user" : setup ? "auth" : "open",
|
|
2143
|
+
// Written ALWAYS, as a boolean, unlike `attachHost` below. Absence and `false` resolve
|
|
2144
|
+
// identically for clients, so an omitted field would be indistinguishable from a deliberate
|
|
2145
|
+
// plaintext mesh — and this is the one field whose whole purpose is that the answer was
|
|
2146
|
+
// stated rather than defaulted.
|
|
2147
|
+
tlsRequired: transport.kind === "tls-required",
|
|
2148
|
+
...(svc.userAuth ? { userAuth: svc.userAuth } : {}),
|
|
2149
|
+
// Only a real decision is persisted — an explicit `--host` now, or one carried forward from a
|
|
2150
|
+
// previous launch. The bare case stays absent rather than recording the loopback default as
|
|
2151
|
+
// though the operator had chosen it.
|
|
2152
|
+
...(attachHost ? { attachHost } : {}),
|
|
2153
|
+
...(maxSessions !== undefined ? { maxSessions } : {}),
|
|
2154
|
+
...(l.maxFileStore !== undefined ? { maxFileStore: l.maxFileStore } : {}),
|
|
2155
|
+
storeDir: l.storeDir,
|
|
2156
|
+
ts: new Date().toISOString(),
|
|
2157
|
+
}, "started");
|
|
2158
|
+
// Commit the policy only for a listener that is up and recorded (S5), and before the control
|
|
2159
|
+
// plane (S9). startDeliveryWithBroker is HANDED the transport decision, so it does not depend on
|
|
2160
|
+
// the file at all.
|
|
2161
|
+
commitTransportPolicy(cotalRoot(), transport);
|
|
2162
|
+
// The delivery daemon self-gates on `.cotal/auth`, so an open mesh has none. Class-2 credential
|
|
2163
|
+
// renewal is NOT wired here: the MANAGER is the renewal owner (it is resident in every mesh mode,
|
|
2164
|
+
// where a foreground `up` process is not).
|
|
2165
|
+
const controlPlane = await startDeliveryWithBroker(space, server, transport.kind === "tls-required", {
|
|
2166
|
+
onDeliveryExit: l.onDeliveryExit,
|
|
2167
|
+
runtime: l.runtime,
|
|
2168
|
+
launch: l.launch,
|
|
2169
|
+
// The address the broker was bound to. This is what lets `cotal attach` reach this manager
|
|
2170
|
+
// from another machine; without it the attach face stays loopback-only, so exposing terminals
|
|
2171
|
+
// is never a side effect of anything but the operator binding the mesh somewhere reachable.
|
|
2172
|
+
attachHost,
|
|
2173
|
+
maxSessions,
|
|
2174
|
+
noManager: l.noManager,
|
|
2175
|
+
resumeAttempt: l.resumeAttempt,
|
|
2176
|
+
resumeCommitToken: l.resumeCommitToken,
|
|
2177
|
+
wsPort: setup?.wsPort, // P2 item 6: the console session client's broker ws port
|
|
2178
|
+
});
|
|
2179
|
+
return { authService: svc.ok, controlPlane };
|
|
2180
|
+
}
|
|
2180
2181
|
/**
|
|
2181
2182
|
* Start a background nats-server (JetStream), wait until it's reachable, pre-create the
|
|
2182
2183
|
* space's streams, and leave it running detached (pid in `.cotal/nats.pid`). Used by
|
|
@@ -2267,85 +2268,24 @@ opts) {
|
|
|
2267
2268
|
}
|
|
2268
2269
|
if (opts.boundListener)
|
|
2269
2270
|
await opts.boundListener.verify();
|
|
2270
|
-
|
|
2271
|
-
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
|
|
2278
|
-
// This is reachable BECAUSE of TLS and cannot happen on `main`: the post-start client verifies the
|
|
2279
|
-
// certificate, so a private CA without `NODE_EXTRA_CA_CERTS` fails here — after the listener is up.
|
|
2280
|
-
// The feature introduced the state, so the feature tears it down.
|
|
2281
|
-
//
|
|
2282
|
-
// Deliberately narrow: this is a teardown on the failure path, not a restructuring of the launch
|
|
2283
|
-
// sequence. The listener is stopped and the pid file removed, then the original error is rethrown
|
|
2284
|
-
// unchanged — the operator needs the certificate error, not a message about cleanup.
|
|
2285
|
-
if (!opts.skipPostStart) {
|
|
2286
|
-
try {
|
|
2287
|
-
await postStart(server, space, setup, seedFile);
|
|
2288
|
-
}
|
|
2289
|
-
catch (e) {
|
|
2290
|
-
try {
|
|
2291
|
-
child.kill("SIGTERM");
|
|
2292
|
-
}
|
|
2293
|
-
catch { /* already gone */ }
|
|
2294
|
-
try {
|
|
2295
|
-
removePidPair(cotalPath("nats.pid"), String(child.pid));
|
|
2296
|
-
}
|
|
2297
|
-
catch { /* best effort */ }
|
|
2298
|
-
throw e;
|
|
2299
|
-
}
|
|
2300
|
-
}
|
|
2301
|
-
else {
|
|
2302
|
-
await recreateMemoryBuckets(server, space, setup);
|
|
2303
|
-
}
|
|
2304
|
-
// USER MODE: the auth service comes up FIRST among the daemons (see the foreground path).
|
|
2305
|
-
const svc = await startUserAuthService(space, server, setup, opts.publicExchange);
|
|
2306
|
-
// Record BEFORE the control plane: the manager's fail-closed mode detection needs the
|
|
2307
|
-
// authoritative registry entry at boot (marker-without-registry refuses). Detached: the entry
|
|
2308
|
-
// outlives this process — `cotal down` removes it.
|
|
2309
|
-
// Same capture-before-record as the foreground path: this also runs for a bare `up --detach` that
|
|
2310
|
-
// RESUMES an already-recorded mesh, whose exposure decision exists only in the registry entry the
|
|
2311
|
-
// call below rewrites.
|
|
2312
|
-
const effectiveAttachHost = attachHostFor(space, opts.host);
|
|
2313
|
-
const effectiveMaxSessions = maxSessionsFor(space, opts.maxSessions);
|
|
2314
|
-
recordOurMesh({
|
|
2315
|
-
space, server, root: cotalRoot(),
|
|
2316
|
-
mode: setup?.prepared ? "user" : useAuth ? "auth" : "open",
|
|
2317
|
-
// The detached listener is started from `transport` a few lines above, so this is the same
|
|
2318
|
-
// decision that shaped the config file — not a re-derivation.
|
|
2319
|
-
tlsRequired: transport.kind === "tls-required",
|
|
2320
|
-
...(svc.userAuth ? { userAuth: svc.userAuth } : {}),
|
|
2321
|
-
// Persist only a real decision — declared now, or carried forward — never the loopback default.
|
|
2322
|
-
...(effectiveAttachHost ? { attachHost: effectiveAttachHost } : {}),
|
|
2323
|
-
...(effectiveMaxSessions !== undefined ? { maxSessions: effectiveMaxSessions } : {}),
|
|
2324
|
-
...(opts.maxFileStore !== undefined ? { maxFileStore: opts.maxFileStore } : {}),
|
|
2325
|
-
storeDir,
|
|
2326
|
-
ts: new Date().toISOString(),
|
|
2327
|
-
}, "started");
|
|
2328
|
-
// Commit policy BEFORE delivery launch (S9). Listener is proved; refuse paths never reach here.
|
|
2329
|
-
// startDeliveryWithBroker is HANDED the transport decision, so it does not depend on the file at all.
|
|
2330
|
-
commitTransportPolicy(cotalRoot(), transport);
|
|
2331
|
-
// Bring up the delivery daemon WITH the detached broker (auth mode only; `cotal down` tears both down).
|
|
2332
|
-
const controlPlane = await startDeliveryWithBroker(space, server, transport.kind === "tls-required", {
|
|
2271
|
+
const { authService, controlPlane } = await serveReadyListener({
|
|
2272
|
+
child, server, space, storeDir, setup, seedFile, transport,
|
|
2273
|
+
host: opts.host,
|
|
2274
|
+
maxSessions: opts.maxSessions,
|
|
2275
|
+
maxFileStore: opts.maxFileStore,
|
|
2276
|
+
publicExchange: opts.publicExchange,
|
|
2277
|
+
resumeAttempt: opts.resumeAttempt,
|
|
2278
|
+
resumeCommitToken: opts.resumeCommitToken,
|
|
2333
2279
|
runtime: opts.runtime,
|
|
2334
2280
|
launch: opts.launch,
|
|
2335
|
-
// See the foreground path: the broker's bind address is what makes attach reachable off-box.
|
|
2336
|
-
attachHost: effectiveAttachHost,
|
|
2337
|
-
maxSessions: effectiveMaxSessions,
|
|
2338
2281
|
noManager: opts.noManager,
|
|
2339
|
-
resumeAttempt: opts.resumeAttempt,
|
|
2340
|
-
resumeCommitToken: opts.resumeCommitToken,
|
|
2341
|
-
wsPort: setup?.wsPort, // P2 item 6: the console session client's broker ws port
|
|
2342
2282
|
});
|
|
2343
2283
|
return {
|
|
2344
2284
|
server,
|
|
2345
2285
|
pid: child.pid ?? 0,
|
|
2346
2286
|
source,
|
|
2347
|
-
controlPlane,
|
|
2348
|
-
authService
|
|
2287
|
+
controlPlane: controlPlane !== undefined,
|
|
2288
|
+
authService,
|
|
2349
2289
|
delivery: useAuth && deliveryUp(space),
|
|
2350
2290
|
manager: managerUp(space),
|
|
2351
2291
|
};
|