@cotal-ai/cli 0.67.0 → 0.69.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/describe.d.ts.map +1 -1
- package/dist/commands/describe.js +2 -2
- package/dist/commands/describe.js.map +1 -1
- package/dist/commands/down.d.ts +0 -23
- package/dist/commands/down.d.ts.map +1 -1
- package/dist/commands/down.js +8 -207
- package/dist/commands/down.js.map +1 -1
- package/dist/commands/endpoints.js +2 -2
- package/dist/commands/endpoints.js.map +1 -1
- package/dist/commands/meshes-add.d.ts +4 -1
- package/dist/commands/meshes-add.d.ts.map +1 -1
- package/dist/commands/meshes-add.js +10 -7
- package/dist/commands/meshes-add.js.map +1 -1
- package/dist/commands/meshes-wizard.js +2 -2
- package/dist/commands/meshes-wizard.js.map +1 -1
- package/dist/commands/meshes.d.ts.map +1 -1
- package/dist/commands/meshes.js +7 -5
- package/dist/commands/meshes.js.map +1 -1
- package/dist/commands/mint.d.ts.map +1 -1
- package/dist/commands/mint.js +8 -2
- package/dist/commands/mint.js.map +1 -1
- package/dist/commands/setup.d.ts +0 -6
- package/dist/commands/setup.d.ts.map +1 -1
- package/dist/commands/setup.js +29 -26
- package/dist/commands/setup.js.map +1 -1
- package/dist/commands/spawn.d.ts.map +1 -1
- package/dist/commands/spawn.js +101 -66
- package/dist/commands/spawn.js.map +1 -1
- package/dist/commands/status.d.ts.map +1 -1
- package/dist/commands/status.js +17 -10
- package/dist/commands/status.js.map +1 -1
- package/dist/commands/sync.d.ts.map +1 -1
- package/dist/commands/sync.js +31 -25
- package/dist/commands/sync.js.map +1 -1
- package/dist/commands/up.d.ts +0 -2
- package/dist/commands/up.d.ts.map +1 -1
- package/dist/commands/up.js +149 -202
- package/dist/commands/up.js.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -17
- package/dist/index.js.map +1 -1
- package/dist/lib/control.d.ts +2 -20
- package/dist/lib/control.d.ts.map +1 -1
- package/dist/lib/control.js +1 -30
- package/dist/lib/control.js.map +1 -1
- package/dist/lib/delivery-proc.d.ts +8 -8
- package/dist/lib/delivery-proc.d.ts.map +1 -1
- package/dist/lib/delivery-proc.js +27 -101
- package/dist/lib/delivery-proc.js.map +1 -1
- package/dist/lib/local-process-stop.d.ts +29 -0
- package/dist/lib/local-process-stop.d.ts.map +1 -0
- package/dist/lib/local-process-stop.js +196 -0
- package/dist/lib/local-process-stop.js.map +1 -0
- package/dist/lib/manager-proc.d.ts +17 -17
- package/dist/lib/manager-proc.d.ts.map +1 -1
- package/dist/lib/manager-proc.js +57 -107
- package/dist/lib/manager-proc.js.map +1 -1
- package/dist/lib/manifest/ledger.js +1 -1
- package/dist/lib/manifest/prepare.d.ts.map +1 -1
- package/dist/lib/manifest/prepare.js +2 -2
- package/dist/lib/manifest/prepare.js.map +1 -1
- package/dist/lib/self-exec.d.ts +1 -1
- package/dist/lib/self-exec.js +1 -1
- package/dist/lib/status.d.ts +0 -1
- package/dist/lib/status.d.ts.map +1 -1
- package/dist/lib/status.js +2 -25
- package/dist/lib/status.js.map +1 -1
- package/dist/lib/teardown-spare.js +3 -3
- package/dist/lib/teardown-spare.js.map +1 -1
- package/dist/ui.d.ts +9 -9
- package/dist/ui.d.ts.map +1 -1
- package/dist/ui.js +16 -18
- package/dist/ui.js.map +1 -1
- package/package.json +4 -4
package/dist/commands/up.js
CHANGED
|
@@ -8,7 +8,7 @@ import { isReachable, DEFAULT_SERVER, createSpaceAuth, serverConfig, openServerC
|
|
|
8
8
|
import { connect } from "@nats-io/transport-node";
|
|
9
9
|
import { jetstreamManager, StorageType } from "@nats-io/jetstream";
|
|
10
10
|
import { Kvm } from "@nats-io/kv";
|
|
11
|
-
import { assertSingleSpaceBroker, assertUserAuthInfo, authDir, getSoleSpaceAuth, getSpaceAuth, hasUserAuthState, listSpaceAccounts, preloadSpaceAccounts, putSpaceAuth, clearCurrent, findMesh, getCurrent, loadMeshes, connectionEvictorCredsKey, membershipConfigPath, membershipObserverCredsKey, membershipRwCredsKey, MEMBERSHIP_CONFIG_KIND, MEMBERSHIP_RW_CREDS_KIND, parsePositiveIntegerFlag, recordMesh, meshesForRoot, removeMesh, rotateSystemCreds, setCurrent, staleSystemCreds, SYSTEM_CREDS_FILES, userAuthStateDir, userAuthSpacesOnDisk, workspaceSecretStore, acquireMaintenanceLock, assertStoreIdentity, assessRestoreClaim, beginOrdinaryResume, bindOrdinaryResumeListener, consumeRetiredMaintenance, markOrdinaryResumeActive, markOrdinaryResumeDegraded, replaceDeadOrdinaryResumeListener, localProcessOwnerStatus, readMaintenanceJournal, readMaintenanceResumeDocument, seatCheckpointDir, readStoreIdentity, recordOrdinaryResumeManagerCommit, releaseMaintenanceLock, retireOrdinaryResume, sameStoreIdentity, readBrokerPolicy, writeBrokerPolicy, removePidPair, writePidPair,
|
|
11
|
+
import { assertSingleSpaceBroker, assertUserAuthInfo, authDir, getSoleSpaceAuth, getSpaceAuth, hasUserAuthState, listSpaceAccounts, preloadSpaceAccounts, putSpaceAuth, clearCurrent, findMesh, getCurrent, loadMeshes, connectionEvictorCredsKey, membershipConfigPath, membershipObserverCredsKey, membershipRwCredsKey, MEMBERSHIP_CONFIG_KIND, MEMBERSHIP_RW_CREDS_KIND, parsePositiveIntegerFlag, recordMesh, meshesForRoot, removeMesh, rotateSystemCreds, setCurrent, staleSystemCreds, SYSTEM_CREDS_FILES, userAuthStateDir, userAuthSpacesOnDisk, workspaceSecretStore, acquireMaintenanceLock, assertStoreIdentity, assessRestoreClaim, beginOrdinaryResume, bindOrdinaryResumeListener, consumeRetiredMaintenance, markOrdinaryResumeActive, markOrdinaryResumeDegraded, replaceDeadOrdinaryResumeListener, localProcessOwnerStatus, readMaintenanceJournal, readMaintenanceResumeDocument, seatCheckpointDir, readStoreIdentity, recordOrdinaryResumeManagerCommit, releaseMaintenanceLock, retireOrdinaryResume, sameStoreIdentity, readBrokerPolicy, writeBrokerPolicy, removePidPair, writePidPair, isLoopbackHost, connectUserControlOrThrow, userViewAuth, } from "@cotal-ai/workspace";
|
|
12
12
|
import { ensureAuthService, resolveAuthProvider, stopAuthService } from "../lib/auth-proc.js";
|
|
13
13
|
import { resolveSpace } from "../lib/status.js";
|
|
14
14
|
import { c } from "../ui.js";
|
|
@@ -21,12 +21,10 @@ import { deliveryStoppedByDown, deliveryUp, ensureControlPlane, ensureDelivery,
|
|
|
21
21
|
import { RESPONDER_UNBOUND_CONSEQUENCE } from "../lib/delivery-responder.js";
|
|
22
22
|
import { displayCmd } from "../lib/self-exec.js";
|
|
23
23
|
import { liveManagerWouldApplyMaxSessions, managerHasDeliveryMarker, managerLogDisplayPath, managerRecordState, managerUp, stopManager } from "../lib/manager-proc.js";
|
|
24
|
-
import { listManagerSeatsForSpare, printLegacyManagerSpareUncertainty, printSparedAgents } from "../lib/teardown-spare.js";
|
|
25
24
|
import { loadManifest } from "../lib/manifest/index.js";
|
|
26
25
|
import { buildLaunchSpec, genRunId, manifestToChannels, preflightConnectors, writeLaunchSpec } from "../lib/manifest/apply.js";
|
|
27
26
|
import { renderUpPlan, renderInherited, renderWarnings } from "../lib/manifest/render.js";
|
|
28
27
|
import { failManifest } from "./topology.js";
|
|
29
|
-
import { reserveStop } from "./down.js";
|
|
30
28
|
import { extensionNames, preflightRuntime } from "../ext-loader.js";
|
|
31
29
|
import { completingFlagValue } from "../lib/completion.js";
|
|
32
30
|
import { askManager } from "../lib/control.js";
|
|
@@ -868,6 +866,12 @@ async function runUp(args, inheritedLock, onAdopt) {
|
|
|
868
866
|
});
|
|
869
867
|
if (!controlPlane)
|
|
870
868
|
process.exitCode = 1;
|
|
869
|
+
// #883: a refresh that HEALED a missing manager must say so — the summary line otherwise reads
|
|
870
|
+
// identically to a refresh that found everything already up, and the operator cannot tell which
|
|
871
|
+
// world they are in without a second command. Only here does a started manager mean one was
|
|
872
|
+
// missing: a launch starts its manager as a matter of course (#2480).
|
|
873
|
+
if (controlPlane?.started)
|
|
874
|
+
console.log(c.green(`✓ restored in the background: manager (pid ${controlPlane.pid})`));
|
|
871
875
|
}
|
|
872
876
|
// A broker was already answering here — this branch starts nothing, so it must not claim the
|
|
873
877
|
// record as ours (see `Provenance`), and it passes only what this invocation decided.
|
|
@@ -936,7 +940,6 @@ async function runUp(args, inheritedLock, onAdopt) {
|
|
|
936
940
|
verify: async () => { await verifySpawnedOrdinaryListener(ordinaryAttempt); },
|
|
937
941
|
},
|
|
938
942
|
} : {}),
|
|
939
|
-
skipPostStart: Boolean(resumeAttempt),
|
|
940
943
|
});
|
|
941
944
|
// Transport policy is committed inside startMeshDetached before delivery launch (S5+S9).
|
|
942
945
|
console.log(c.dim(`Started nats-server (${source}).`));
|
|
@@ -1023,77 +1026,43 @@ async function runUp(args, inheritedLock, onAdopt) {
|
|
|
1023
1026
|
if (!resumeAttempt)
|
|
1024
1027
|
process.exit(1);
|
|
1025
1028
|
});
|
|
1026
|
-
// The control plane is coupled to the broker: stop the
|
|
1029
|
+
// The control plane is coupled to the broker: stop the detached manager AND the delivery daemon
|
|
1027
1030
|
// (AND the space's user-auth service) when this `up` stops (Ctrl-C), so none outlives the broker
|
|
1028
1031
|
// it serves — a surviving manager would reconnect-loop invisibly against the dead (or the NEXT)
|
|
1029
1032
|
// broker (the documented orphan-supervisor failure mode). All kill by pidfile, symmetric; the
|
|
1030
1033
|
// auth service's pid is space-scoped so no other space's daemon can ever be hit.
|
|
1031
|
-
//
|
|
1032
|
-
//
|
|
1033
|
-
//
|
|
1034
|
-
//
|
|
1035
|
-
//
|
|
1036
|
-
//
|
|
1037
|
-
//
|
|
1038
|
-
//
|
|
1034
|
+
// The manager goes first, in `cotal down`'s order, through the stop `cotal down` runs (#1307): it
|
|
1035
|
+
// holds the stop reservation, asserts the exact manager can spare its seats before any signal,
|
|
1036
|
+
// reports the seats left behind, and escalates a wedged manager to SIGKILL. When that stop throws
|
|
1037
|
+
// the manager may still be serving, so NOTHING else is signalled — no delivery, no auth, no broker:
|
|
1038
|
+
// the refusal is printed with the reap route and the latch is released, so the stack keeps running
|
|
1039
|
+
// in the foreground; the operator ends it with `cotal down --with-agents` from another terminal,
|
|
1040
|
+
// and the broker-exit handler below already ends `up` when the broker goes.
|
|
1041
|
+
// stopDelivery is the stop `cotal down delivery` performs (reservation, SIGKILL escalation); the
|
|
1042
|
+
// rest of the teardown must run even if it fails — the failure is logged, never swallowed
|
|
1043
|
+
// silently. Teardown is AWAITED before the broker is signalled, so it never races a broker that is
|
|
1044
|
+
// already going away.
|
|
1039
1045
|
let stopping = false;
|
|
1046
|
+
let teardown;
|
|
1040
1047
|
const stop = () => {
|
|
1041
1048
|
if (stopping)
|
|
1042
1049
|
return; // a second Ctrl-C during teardown must not start a second teardown
|
|
1043
1050
|
stopping = true;
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
const managerPidPath = localProcessPath(MANAGER_PIDFILE, managerContext);
|
|
1050
|
-
const stopMarker = `${managerPidPath}.stopping`;
|
|
1051
|
-
let reserved = false;
|
|
1052
|
-
if (existsSync(managerPidPath)) {
|
|
1053
|
-
const pin = verifyIdentityPin(managerPidPath);
|
|
1054
|
-
if (pin.kind === "legacy")
|
|
1055
|
-
legacyManagerSpareUnverified = true;
|
|
1056
|
-
else if (pin.kind === "match") {
|
|
1057
|
-
spared = await listManagerSeatsForSpare(managerContext);
|
|
1058
|
-
try {
|
|
1059
|
-
// Taken before the capability read and held until the manager is gone, as bare `down`
|
|
1060
|
-
// holds it, so a concurrent `cotal down` can neither stop this manager nor arm a reap
|
|
1061
|
-
// while this stop is in flight.
|
|
1062
|
-
reserveStop("manager", stopMarker);
|
|
1063
|
-
reserved = true;
|
|
1064
|
-
spareSeats = assertManagerCanSpare(managerContext, undefined, pin.record);
|
|
1065
|
-
}
|
|
1066
|
-
catch (e) {
|
|
1067
|
-
// THE CAPABILITY ASSERT IS THE SIGNAL GATE, the same rule bare `down` enforces inside
|
|
1068
|
-
// its beforeSignal hook: a throw there signals nothing. Signal NOTHING here either — no
|
|
1069
|
-
// manager, no delivery, no auth, no broker — print the refusal with the reap route, and
|
|
1070
|
-
// release the latch so the stack keeps running in the foreground; the operator ends it
|
|
1071
|
-
// with `cotal down --with-agents` from another terminal, and the broker-exit handler
|
|
1072
|
-
// below already ends `up` when the broker goes.
|
|
1073
|
-
if (reserved)
|
|
1074
|
-
rmSync(stopMarker, { force: true });
|
|
1075
|
-
console.error(c.red(`! teardown: ${e.message}`));
|
|
1076
|
-
console.error(c.red(`the stack is still running; to take managed agents with it, run: cotal down --with-agents`));
|
|
1077
|
-
stopping = false;
|
|
1078
|
-
return;
|
|
1079
|
-
}
|
|
1080
|
-
}
|
|
1081
|
-
}
|
|
1082
|
-
await stopDelivery(undefined, undefined, space).catch((e) => console.error(`! delivery teardown: ${e.message}`));
|
|
1051
|
+
// Once the broker has exited, its handler below is already running this teardown and reads
|
|
1052
|
+
// `stopping` when it is done.
|
|
1053
|
+
if (child.exitCode !== null || child.signalCode !== null)
|
|
1054
|
+
return;
|
|
1055
|
+
teardown = (async () => {
|
|
1083
1056
|
try {
|
|
1084
|
-
await stopManager(
|
|
1085
|
-
if (legacyManagerSpareUnverified)
|
|
1086
|
-
printLegacyManagerSpareUncertainty();
|
|
1087
|
-
else if (spared)
|
|
1088
|
-
printSparedAgents(spared, spareSeats);
|
|
1057
|
+
await stopManager(space);
|
|
1089
1058
|
}
|
|
1090
1059
|
catch (e) {
|
|
1091
|
-
console.error(`!
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
rmSync(stopMarker, { force: true });
|
|
1060
|
+
console.error(c.red(`! teardown: ${e.message}`));
|
|
1061
|
+
console.error(c.red(`the stack is still running; to take managed agents with it, run: cotal down --with-agents`));
|
|
1062
|
+
stopping = false;
|
|
1063
|
+
return;
|
|
1096
1064
|
}
|
|
1065
|
+
await stopDelivery(space).catch((e) => console.error(`! delivery teardown: ${e.message}`));
|
|
1097
1066
|
await stopAuthService(space).catch((e) => console.error(`! auth teardown: ${e.message}`));
|
|
1098
1067
|
child.kill("SIGTERM");
|
|
1099
1068
|
})();
|
|
@@ -1104,9 +1073,11 @@ async function runUp(args, inheritedLock, onAdopt) {
|
|
|
1104
1073
|
// so a later `cotal spawn` doesn't try to join a dead mesh.
|
|
1105
1074
|
child.on("exit", async (code, signal) => {
|
|
1106
1075
|
removePidPair(cotalPath("nats.pid"), String(child.pid));
|
|
1107
|
-
//
|
|
1108
|
-
|
|
1109
|
-
await
|
|
1076
|
+
// One teardown at a time: a stop started while another is in flight is refused the reservation
|
|
1077
|
+
// that one holds, so wait for a Ctrl-C teardown already running instead of racing it.
|
|
1078
|
+
await teardown;
|
|
1079
|
+
await stopDelivery(space).catch((e) => console.error(`! delivery teardown: ${e.message}`));
|
|
1080
|
+
await stopManager(space).catch((e) => console.error(`! manager teardown: ${e.message}`));
|
|
1110
1081
|
await stopAuthService(space).catch((e) => console.error(`! auth teardown: ${e.message}`));
|
|
1111
1082
|
// Unrecording (and the exit code below) both depend on WHY the broker is gone. `stopping` is
|
|
1112
1083
|
// true only for the intended shutdown `stop()` drives (Ctrl-C, SIGTERM) — everything else here
|
|
@@ -1157,46 +1128,7 @@ async function runUp(args, inheritedLock, onAdopt) {
|
|
|
1157
1128
|
console.error(c.yellow(`! nats-server ${v.version} is below 2.14.5 and the presence bucket of "${space}" is file-backed: it can latch after a stall and close the mesh to new joins (#1356); put a nats-server 2.14.5 or newer on PATH (it wins over the bundled one)`));
|
|
1158
1129
|
if (restored)
|
|
1159
1130
|
await provePreparedRestoreListener(restored);
|
|
1160
|
-
// Listener ready — commit the transport decision (S5: not before start).
|
|
1161
|
-
commitTransportPolicy(meshRoot, transport);
|
|
1162
1131
|
{
|
|
1163
|
-
if (!resumeAttempt)
|
|
1164
|
-
await postStart(server, space, setup, seedFile);
|
|
1165
|
-
else
|
|
1166
|
-
await recreateMemoryBuckets(server, space, setup);
|
|
1167
|
-
// USER MODE: the auth service comes up FIRST among the daemons — until its callout answers,
|
|
1168
|
-
// every user-mode connect to this broker is denied, so `up` must not report a usable user mesh
|
|
1169
|
-
// (nor let agents race it) on a half-started auth plane. (Foreground `up` doesn't exit here, so
|
|
1170
|
-
// `ok` has no exit code to carry — the red consequence line above is the operator signal.)
|
|
1171
|
-
const svc = await startUserAuthService(space, server, setup, publicExchange);
|
|
1172
|
-
// Record BEFORE the control plane comes up: the manager's fail-closed mode detection requires
|
|
1173
|
-
// an authoritative registry entry (marker-without-registry is a refused start, not a guess),
|
|
1174
|
-
// so the record must exist by the time it boots. A manager/delivery failure after this leaves
|
|
1175
|
-
// a recorded-but-degraded mesh — the documented, healable posture.
|
|
1176
|
-
// Resolve exposure BEFORE recording. This path also serves a RESUME of an already-recorded mesh
|
|
1177
|
-
// (`down --preserve-state` then a bare `up`), where the operator's `--host` lives only in the
|
|
1178
|
-
// registry — and the record below is written whole, so reading it afterwards would find the
|
|
1179
|
-
// field this very call had just erased.
|
|
1180
|
-
const effectiveAttachHost = attachHostFor(space, values.host);
|
|
1181
|
-
const effectiveMaxSessions = maxSessionsFor(space, maxSessions);
|
|
1182
|
-
recordOurMesh({
|
|
1183
|
-
space, server, root: cotalRoot(),
|
|
1184
|
-
mode: setup?.prepared ? "user" : useAuth ? "auth" : "open",
|
|
1185
|
-
// Written ALWAYS, as a boolean, unlike `attachHost` below. Absence and `false` resolve
|
|
1186
|
-
// identically for clients, so an omitted field would be indistinguishable from a deliberate
|
|
1187
|
-
// plaintext mesh — and this is the one field whose whole purpose is that the answer was
|
|
1188
|
-
// stated rather than defaulted.
|
|
1189
|
-
tlsRequired: transport.kind === "tls-required",
|
|
1190
|
-
...(svc.userAuth ? { userAuth: svc.userAuth } : {}),
|
|
1191
|
-
// Only a real decision is persisted — an explicit `--host` now, or one carried forward from a
|
|
1192
|
-
// previous launch. The bare case stays absent rather than recording the loopback default as
|
|
1193
|
-
// though the operator had chosen it.
|
|
1194
|
-
...(effectiveAttachHost ? { attachHost: effectiveAttachHost } : {}),
|
|
1195
|
-
...(effectiveMaxSessions !== undefined ? { maxSessions: effectiveMaxSessions } : {}),
|
|
1196
|
-
...(maxFileStore !== undefined ? { maxFileStore } : {}),
|
|
1197
|
-
storeDir,
|
|
1198
|
-
ts: new Date().toISOString(),
|
|
1199
|
-
}, "started");
|
|
1200
1132
|
// A DAEMON THAT DIES UNDER THIS RUNNING BROKER IS RESTARTED HERE (#2469). It exits on its own once
|
|
1201
1133
|
// it cannot reach the broker, and a starved host makes a running broker look unreachable, so it can
|
|
1202
1134
|
// end while this broker serves on. Nothing else brings it back, and every static retirement then
|
|
@@ -1247,26 +1179,21 @@ async function runUp(args, inheritedLock, onAdopt) {
|
|
|
1247
1179
|
restartingDelivery = false;
|
|
1248
1180
|
})();
|
|
1249
1181
|
};
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
runtime: values.runtime,
|
|
1257
|
-
// The address the broker was bound to. This is what lets `cotal attach` reach this manager
|
|
1258
|
-
// from another machine; without it the attach face stays loopback-only, so exposing terminals
|
|
1259
|
-
// is never a side effect of anything but the operator binding the mesh somewhere reachable.
|
|
1260
|
-
attachHost: effectiveAttachHost,
|
|
1261
|
-
maxSessions: effectiveMaxSessions,
|
|
1262
|
-
noManager,
|
|
1182
|
+
const { authService, controlPlane } = await serveReadyListener({
|
|
1183
|
+
child, server, space, storeDir, setup, seedFile, transport,
|
|
1184
|
+
host: values.host,
|
|
1185
|
+
maxSessions,
|
|
1186
|
+
maxFileStore,
|
|
1187
|
+
publicExchange,
|
|
1263
1188
|
resumeAttempt,
|
|
1264
1189
|
resumeCommitToken: restored?.managerCommit?.durableCommitToken ?? ordinaryAttempt?.managerCommit?.durableCommitToken,
|
|
1265
|
-
|
|
1190
|
+
runtime: values.runtime,
|
|
1191
|
+
noManager,
|
|
1192
|
+
onDeliveryExit,
|
|
1266
1193
|
});
|
|
1267
1194
|
if (restored && process.env.COTAL_SMOKE_FAIL_AFTER_RESTORE_LISTENER_READY === "1")
|
|
1268
1195
|
throw new Error("smoke-injected failure after restore listener readiness");
|
|
1269
|
-
await completeResumeActivation(resumeAttempt, controlPlane &&
|
|
1196
|
+
await completeResumeActivation(resumeAttempt, controlPlane !== undefined && authService, !authService ? "normal listener started but the user-auth service is unavailable" : "normal listener started but the control plane is degraded", server, startupLock);
|
|
1270
1197
|
activationFinished = true;
|
|
1271
1198
|
}
|
|
1272
1199
|
await new Promise(() => { });
|
|
@@ -1644,7 +1571,7 @@ async function resumeProvenOrdinaryListener(pending, heldLock) {
|
|
|
1644
1571
|
resumeAttempt: pending.attemptId,
|
|
1645
1572
|
resumeCommitToken: pending.managerCommit?.durableCommitToken,
|
|
1646
1573
|
});
|
|
1647
|
-
await completeResumeActivation(pending.attemptId, controlPlane && svc.ok, !svc.ok ? "adopted resume listener has no user-auth service" : "adopted resume listener has a degraded control plane", pending.server, heldLock);
|
|
1574
|
+
await completeResumeActivation(pending.attemptId, controlPlane !== undefined && svc.ok, !svc.ok ? "adopted resume listener has no user-auth service" : "adopted resume listener has a degraded control plane", pending.server, heldLock);
|
|
1648
1575
|
}
|
|
1649
1576
|
async function ensureRecoveredUserAuth(prepared) {
|
|
1650
1577
|
if (prepared.mode !== "user")
|
|
@@ -1690,7 +1617,7 @@ async function resumeProvenRestoreListener(prepared, heldLock) {
|
|
|
1690
1617
|
resumeAttempt: prepared.attemptId,
|
|
1691
1618
|
resumeCommitToken: prepared.managerCommit?.durableCommitToken,
|
|
1692
1619
|
});
|
|
1693
|
-
await completeResumeActivation(prepared.attemptId, controlPlane && svc.ok, !svc.ok ? "proven restore listener has no user-auth service" : "proven restore listener has a degraded control plane", prepared.server, heldLock);
|
|
1620
|
+
await completeResumeActivation(prepared.attemptId, controlPlane !== undefined && svc.ok, !svc.ok ? "proven restore listener has no user-auth service" : "proven restore listener has a degraded control plane", prepared.server, heldLock);
|
|
1694
1621
|
}
|
|
1695
1622
|
async function completeResumeActivation(attemptId, healthy, reason, server, heldLock) {
|
|
1696
1623
|
if (!attemptId)
|
|
@@ -2054,7 +1981,7 @@ async function upManifest(file, opts) {
|
|
|
2054
1981
|
// A leftover detached manager (its broker is gone — the reachability check above proved nothing
|
|
2055
1982
|
// lives at this address) would win the fresh mesh's lease and the launch manager would refuse.
|
|
2056
1983
|
// Stop it, so the manager started below WITH the launch spec is THE manager.
|
|
2057
|
-
await stopManager(
|
|
1984
|
+
await stopManager(m.space);
|
|
2058
1985
|
let pid;
|
|
2059
1986
|
let controlPlane = false;
|
|
2060
1987
|
let authService = true;
|
|
@@ -2124,8 +2051,9 @@ function applyUpOverrides(prepared, o) {
|
|
|
2124
2051
|
* cotal_spawn find a manager without any setup side effect. Coupled to the broker by the
|
|
2125
2052
|
* daemon's watchdog + the `up`/`down` teardown. `mgr` rides through to the manager start — the
|
|
2126
2053
|
* `up -f` path hands THE manager its runtime + resolved launch spec here (one manager per space,
|
|
2127
|
-
* so the launch can never be a second supervise). Returns
|
|
2128
|
-
* caller whose output claims a manager (the `up -f` launching
|
|
2054
|
+
* so the launch can never be a second supervise). Returns the control plane it ensured, or
|
|
2055
|
+
* `undefined` when it is degraded, so a caller whose output claims a manager (the `up -f` launching
|
|
2056
|
+
* line) can tell the truth. */
|
|
2129
2057
|
async function startDeliveryWithBroker(space, server,
|
|
2130
2058
|
/**
|
|
2131
2059
|
* Whether this listener requires TLS, decided by the CALLER.
|
|
@@ -2146,11 +2074,6 @@ async function startDeliveryWithBroker(space, server,
|
|
|
2146
2074
|
tlsRequired, mgr) {
|
|
2147
2075
|
try {
|
|
2148
2076
|
const plane = await ensureControlPlane({ space, server, tls: tlsRequired, ...(mgr ?? {}) });
|
|
2149
|
-
// #883: a refresh that HEALED a missing manager must say so — the summary line otherwise reads
|
|
2150
|
-
// identically to a refresh that found everything already up, and the operator cannot tell which
|
|
2151
|
-
// world they are in without a second command.
|
|
2152
|
-
if (plane.started)
|
|
2153
|
-
console.log(c.green(`✓ restored in the background: manager (pid ${plane.pid})`));
|
|
2154
2077
|
// #1576: `up` either BINDS the responder or SAYS SO HERE. The delivery daemon is a hard
|
|
2155
2078
|
// dependency of spawn, retirement and join, and this function used to return `true` for a boot
|
|
2156
2079
|
// that started a daemon whose responder never bound — so `cotal up` printed its success banner
|
|
@@ -2160,15 +2083,100 @@ tlsRequired, mgr) {
|
|
|
2160
2083
|
if (plane.responderBound === false)
|
|
2161
2084
|
console.error(c.yellow(`! delivery responder did not bind before this boot finished - ${RESPONDER_UNBOUND_CONSEQUENCE}`) +
|
|
2162
2085
|
c.dim(`\n The daemon process is running and the wait is open-ended; boot durable joins reconcile by themselves once it binds (agents do NOT need respawning).\n Watch it with \`${displayCmd()} status --components\`.`));
|
|
2163
|
-
return
|
|
2086
|
+
return plane;
|
|
2164
2087
|
}
|
|
2165
2088
|
catch (e) {
|
|
2166
2089
|
// Non-fatal (live messaging is unaffected) — but never SILENT: without the manager,
|
|
2167
2090
|
// `spawn --detach` / cotal_spawn have no responder, and the operator must hear it here,
|
|
2168
2091
|
// not as an unexplained "no manager reachable" later.
|
|
2169
2092
|
console.error(c.dim(`! control plane degraded: ${e.message} - durable delivery/manager may be down; start one with: cotal supervise`));
|
|
2170
|
-
return
|
|
2093
|
+
return undefined;
|
|
2094
|
+
}
|
|
2095
|
+
}
|
|
2096
|
+
/**
|
|
2097
|
+
* Everything a launch runs once its listener answers: the space setup, the user-auth service, the
|
|
2098
|
+
* mesh record, the transport commit and the control plane. Foreground `up` and
|
|
2099
|
+
* {@link startMeshDetached} both call it, so the two launch modes cannot drift apart (#2496).
|
|
2100
|
+
*
|
|
2101
|
+
* A fresh boot whose setup throws stops the listener and removes `nats.pid` before rethrowing the
|
|
2102
|
+
* original error. Nothing has recorded the mesh yet, so a listener left running would hold the port
|
|
2103
|
+
* with no registry entry for `cotal down` to reach. A channel seed with an unparseable value fails
|
|
2104
|
+
* here. A resume keeps its listener: its maintenance journal is already bound to it.
|
|
2105
|
+
*/
|
|
2106
|
+
async function serveReadyListener(l) {
|
|
2107
|
+
const { child, server, space, setup, transport } = l;
|
|
2108
|
+
if (l.resumeAttempt)
|
|
2109
|
+
await recreateMemoryBuckets(server, space, setup);
|
|
2110
|
+
else {
|
|
2111
|
+
try {
|
|
2112
|
+
await postStart(server, space, setup, l.seedFile);
|
|
2113
|
+
}
|
|
2114
|
+
catch (e) {
|
|
2115
|
+
try {
|
|
2116
|
+
child.kill("SIGTERM");
|
|
2117
|
+
}
|
|
2118
|
+
catch { /* already gone */ }
|
|
2119
|
+
try {
|
|
2120
|
+
removePidPair(cotalPath("nats.pid"), String(child.pid));
|
|
2121
|
+
}
|
|
2122
|
+
catch { /* best effort */ }
|
|
2123
|
+
throw e;
|
|
2124
|
+
}
|
|
2171
2125
|
}
|
|
2126
|
+
// USER MODE: the auth service comes up FIRST among the daemons — until its callout answers,
|
|
2127
|
+
// every user-mode connect to this broker is denied, so `up` must not report a usable user mesh
|
|
2128
|
+
// (nor let agents race it) on a half-started auth plane.
|
|
2129
|
+
const svc = await startUserAuthService(space, server, setup, l.publicExchange);
|
|
2130
|
+
// Record BEFORE the control plane comes up: the manager's fail-closed mode detection requires
|
|
2131
|
+
// an authoritative registry entry (marker-without-registry is a refused start, not a guess),
|
|
2132
|
+
// so the record must exist by the time it boots. A manager/delivery failure after this leaves
|
|
2133
|
+
// a recorded-but-degraded mesh — the documented, healable posture.
|
|
2134
|
+
// Resolve exposure BEFORE recording. A RESUME of an already-recorded mesh (`down
|
|
2135
|
+
// --preserve-state` then a bare `up`) carries the operator's `--host` only in the registry, and
|
|
2136
|
+
// the record below is written whole, so reading it afterwards would find the field this very call
|
|
2137
|
+
// had just erased.
|
|
2138
|
+
const attachHost = attachHostFor(space, l.host);
|
|
2139
|
+
const maxSessions = maxSessionsFor(space, l.maxSessions);
|
|
2140
|
+
recordOurMesh({
|
|
2141
|
+
space, server, root: cotalRoot(),
|
|
2142
|
+
mode: setup?.prepared ? "user" : setup ? "auth" : "open",
|
|
2143
|
+
// Written ALWAYS, as a boolean, unlike `attachHost` below. Absence and `false` resolve
|
|
2144
|
+
// identically for clients, so an omitted field would be indistinguishable from a deliberate
|
|
2145
|
+
// plaintext mesh — and this is the one field whose whole purpose is that the answer was
|
|
2146
|
+
// stated rather than defaulted.
|
|
2147
|
+
tlsRequired: transport.kind === "tls-required",
|
|
2148
|
+
...(svc.userAuth ? { userAuth: svc.userAuth } : {}),
|
|
2149
|
+
// Only a real decision is persisted — an explicit `--host` now, or one carried forward from a
|
|
2150
|
+
// previous launch. The bare case stays absent rather than recording the loopback default as
|
|
2151
|
+
// though the operator had chosen it.
|
|
2152
|
+
...(attachHost ? { attachHost } : {}),
|
|
2153
|
+
...(maxSessions !== undefined ? { maxSessions } : {}),
|
|
2154
|
+
...(l.maxFileStore !== undefined ? { maxFileStore: l.maxFileStore } : {}),
|
|
2155
|
+
storeDir: l.storeDir,
|
|
2156
|
+
ts: new Date().toISOString(),
|
|
2157
|
+
}, "started");
|
|
2158
|
+
// Commit the policy only for a listener that is up and recorded (S5), and before the control
|
|
2159
|
+
// plane (S9). startDeliveryWithBroker is HANDED the transport decision, so it does not depend on
|
|
2160
|
+
// the file at all.
|
|
2161
|
+
commitTransportPolicy(cotalRoot(), transport);
|
|
2162
|
+
// The delivery daemon self-gates on `.cotal/auth`, so an open mesh has none. Class-2 credential
|
|
2163
|
+
// renewal is NOT wired here: the MANAGER is the renewal owner (it is resident in every mesh mode,
|
|
2164
|
+
// where a foreground `up` process is not).
|
|
2165
|
+
const controlPlane = await startDeliveryWithBroker(space, server, transport.kind === "tls-required", {
|
|
2166
|
+
onDeliveryExit: l.onDeliveryExit,
|
|
2167
|
+
runtime: l.runtime,
|
|
2168
|
+
launch: l.launch,
|
|
2169
|
+
// The address the broker was bound to. This is what lets `cotal attach` reach this manager
|
|
2170
|
+
// from another machine; without it the attach face stays loopback-only, so exposing terminals
|
|
2171
|
+
// is never a side effect of anything but the operator binding the mesh somewhere reachable.
|
|
2172
|
+
attachHost,
|
|
2173
|
+
maxSessions,
|
|
2174
|
+
noManager: l.noManager,
|
|
2175
|
+
resumeAttempt: l.resumeAttempt,
|
|
2176
|
+
resumeCommitToken: l.resumeCommitToken,
|
|
2177
|
+
wsPort: setup?.wsPort, // P2 item 6: the console session client's broker ws port
|
|
2178
|
+
});
|
|
2179
|
+
return { authService: svc.ok, controlPlane };
|
|
2172
2180
|
}
|
|
2173
2181
|
/**
|
|
2174
2182
|
* Start a background nats-server (JetStream), wait until it's reachable, pre-create the
|
|
@@ -2260,85 +2268,24 @@ opts) {
|
|
|
2260
2268
|
}
|
|
2261
2269
|
if (opts.boundListener)
|
|
2262
2270
|
await opts.boundListener.verify();
|
|
2263
|
-
|
|
2264
|
-
|
|
2265
|
-
|
|
2266
|
-
|
|
2267
|
-
|
|
2268
|
-
|
|
2269
|
-
|
|
2270
|
-
|
|
2271
|
-
// This is reachable BECAUSE of TLS and cannot happen on `main`: the post-start client verifies the
|
|
2272
|
-
// certificate, so a private CA without `NODE_EXTRA_CA_CERTS` fails here — after the listener is up.
|
|
2273
|
-
// The feature introduced the state, so the feature tears it down.
|
|
2274
|
-
//
|
|
2275
|
-
// Deliberately narrow: this is a teardown on the failure path, not a restructuring of the launch
|
|
2276
|
-
// sequence. The listener is stopped and the pid file removed, then the original error is rethrown
|
|
2277
|
-
// unchanged — the operator needs the certificate error, not a message about cleanup.
|
|
2278
|
-
if (!opts.skipPostStart) {
|
|
2279
|
-
try {
|
|
2280
|
-
await postStart(server, space, setup, seedFile);
|
|
2281
|
-
}
|
|
2282
|
-
catch (e) {
|
|
2283
|
-
try {
|
|
2284
|
-
child.kill("SIGTERM");
|
|
2285
|
-
}
|
|
2286
|
-
catch { /* already gone */ }
|
|
2287
|
-
try {
|
|
2288
|
-
removePidPair(cotalPath("nats.pid"), String(child.pid));
|
|
2289
|
-
}
|
|
2290
|
-
catch { /* best effort */ }
|
|
2291
|
-
throw e;
|
|
2292
|
-
}
|
|
2293
|
-
}
|
|
2294
|
-
else {
|
|
2295
|
-
await recreateMemoryBuckets(server, space, setup);
|
|
2296
|
-
}
|
|
2297
|
-
// USER MODE: the auth service comes up FIRST among the daemons (see the foreground path).
|
|
2298
|
-
const svc = await startUserAuthService(space, server, setup, opts.publicExchange);
|
|
2299
|
-
// Record BEFORE the control plane: the manager's fail-closed mode detection needs the
|
|
2300
|
-
// authoritative registry entry at boot (marker-without-registry refuses). Detached: the entry
|
|
2301
|
-
// outlives this process — `cotal down` removes it.
|
|
2302
|
-
// Same capture-before-record as the foreground path: this also runs for a bare `up --detach` that
|
|
2303
|
-
// RESUMES an already-recorded mesh, whose exposure decision exists only in the registry entry the
|
|
2304
|
-
// call below rewrites.
|
|
2305
|
-
const effectiveAttachHost = attachHostFor(space, opts.host);
|
|
2306
|
-
const effectiveMaxSessions = maxSessionsFor(space, opts.maxSessions);
|
|
2307
|
-
recordOurMesh({
|
|
2308
|
-
space, server, root: cotalRoot(),
|
|
2309
|
-
mode: setup?.prepared ? "user" : useAuth ? "auth" : "open",
|
|
2310
|
-
// The detached listener is started from `transport` a few lines above, so this is the same
|
|
2311
|
-
// decision that shaped the config file — not a re-derivation.
|
|
2312
|
-
tlsRequired: transport.kind === "tls-required",
|
|
2313
|
-
...(svc.userAuth ? { userAuth: svc.userAuth } : {}),
|
|
2314
|
-
// Persist only a real decision — declared now, or carried forward — never the loopback default.
|
|
2315
|
-
...(effectiveAttachHost ? { attachHost: effectiveAttachHost } : {}),
|
|
2316
|
-
...(effectiveMaxSessions !== undefined ? { maxSessions: effectiveMaxSessions } : {}),
|
|
2317
|
-
...(opts.maxFileStore !== undefined ? { maxFileStore: opts.maxFileStore } : {}),
|
|
2318
|
-
storeDir,
|
|
2319
|
-
ts: new Date().toISOString(),
|
|
2320
|
-
}, "started");
|
|
2321
|
-
// Commit policy BEFORE delivery launch (S9). Listener is proved; refuse paths never reach here.
|
|
2322
|
-
// startDeliveryWithBroker is HANDED the transport decision, so it does not depend on the file at all.
|
|
2323
|
-
commitTransportPolicy(cotalRoot(), transport);
|
|
2324
|
-
// Bring up the delivery daemon WITH the detached broker (auth mode only; `cotal down` tears both down).
|
|
2325
|
-
const controlPlane = await startDeliveryWithBroker(space, server, transport.kind === "tls-required", {
|
|
2271
|
+
const { authService, controlPlane } = await serveReadyListener({
|
|
2272
|
+
child, server, space, storeDir, setup, seedFile, transport,
|
|
2273
|
+
host: opts.host,
|
|
2274
|
+
maxSessions: opts.maxSessions,
|
|
2275
|
+
maxFileStore: opts.maxFileStore,
|
|
2276
|
+
publicExchange: opts.publicExchange,
|
|
2277
|
+
resumeAttempt: opts.resumeAttempt,
|
|
2278
|
+
resumeCommitToken: opts.resumeCommitToken,
|
|
2326
2279
|
runtime: opts.runtime,
|
|
2327
2280
|
launch: opts.launch,
|
|
2328
|
-
// See the foreground path: the broker's bind address is what makes attach reachable off-box.
|
|
2329
|
-
attachHost: effectiveAttachHost,
|
|
2330
|
-
maxSessions: effectiveMaxSessions,
|
|
2331
2281
|
noManager: opts.noManager,
|
|
2332
|
-
resumeAttempt: opts.resumeAttempt,
|
|
2333
|
-
resumeCommitToken: opts.resumeCommitToken,
|
|
2334
|
-
wsPort: setup?.wsPort, // P2 item 6: the console session client's broker ws port
|
|
2335
2282
|
});
|
|
2336
2283
|
return {
|
|
2337
2284
|
server,
|
|
2338
2285
|
pid: child.pid ?? 0,
|
|
2339
2286
|
source,
|
|
2340
|
-
controlPlane,
|
|
2341
|
-
authService
|
|
2287
|
+
controlPlane: controlPlane !== undefined,
|
|
2288
|
+
authService,
|
|
2342
2289
|
delivery: useAuth && deliveryUp(space),
|
|
2343
2290
|
manager: managerUp(space),
|
|
2344
2291
|
};
|