@cotal-ai/cli 0.16.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cotal-skills/skills/team-topology/SKILL.md +1 -1
- package/dist/command.d.ts.map +1 -1
- package/dist/command.js +13 -0
- package/dist/command.js.map +1 -1
- package/dist/commands/agents.d.ts +15 -15
- package/dist/commands/agents.d.ts.map +1 -1
- package/dist/commands/agents.js +187 -77
- package/dist/commands/agents.js.map +1 -1
- package/dist/commands/clean.d.ts.map +1 -1
- package/dist/commands/clean.js +4 -3
- package/dist/commands/clean.js.map +1 -1
- package/dist/commands/describe.d.ts +72 -0
- package/dist/commands/describe.d.ts.map +1 -0
- package/dist/commands/describe.js +148 -0
- package/dist/commands/describe.js.map +1 -0
- package/dist/commands/doctor.d.ts.map +1 -1
- package/dist/commands/doctor.js +27 -8
- package/dist/commands/doctor.js.map +1 -1
- package/dist/commands/down-manifest.d.ts.map +1 -1
- package/dist/commands/down-manifest.js +38 -18
- package/dist/commands/down-manifest.js.map +1 -1
- package/dist/commands/down.d.ts +9 -0
- package/dist/commands/down.d.ts.map +1 -1
- package/dist/commands/down.js +38 -10
- package/dist/commands/down.js.map +1 -1
- package/dist/commands/ext.d.ts.map +1 -1
- package/dist/commands/ext.js +16 -10
- package/dist/commands/ext.js.map +1 -1
- package/dist/commands/join.d.ts.map +1 -1
- package/dist/commands/join.js +2 -4
- package/dist/commands/join.js.map +1 -1
- package/dist/commands/meshes-add.d.ts +17 -0
- package/dist/commands/meshes-add.d.ts.map +1 -1
- package/dist/commands/meshes-add.js +72 -4
- package/dist/commands/meshes-add.js.map +1 -1
- package/dist/commands/meshes-wizard.d.ts +3 -0
- package/dist/commands/meshes-wizard.d.ts.map +1 -1
- package/dist/commands/meshes-wizard.js +78 -6
- package/dist/commands/meshes-wizard.js.map +1 -1
- package/dist/commands/meshes.d.ts +4 -0
- package/dist/commands/meshes.d.ts.map +1 -1
- package/dist/commands/meshes.js +14 -4
- package/dist/commands/meshes.js.map +1 -1
- package/dist/commands/models.d.ts.map +1 -1
- package/dist/commands/models.js +2 -2
- package/dist/commands/models.js.map +1 -1
- package/dist/commands/spawn-manifest.d.ts.map +1 -1
- package/dist/commands/spawn-manifest.js +22 -12
- package/dist/commands/spawn-manifest.js.map +1 -1
- package/dist/commands/spawn.d.ts +5 -0
- package/dist/commands/spawn.d.ts.map +1 -1
- package/dist/commands/spawn.js +5 -3
- package/dist/commands/spawn.js.map +1 -1
- package/dist/commands/status.js +25 -5
- package/dist/commands/status.js.map +1 -1
- package/dist/commands/up.d.ts +17 -2
- package/dist/commands/up.d.ts.map +1 -1
- package/dist/commands/up.js +685 -50
- package/dist/commands/up.js.map +1 -1
- package/dist/console/app.d.ts +1 -1
- package/dist/console/app.d.ts.map +1 -1
- package/dist/console/app.js +16 -5
- package/dist/console/app.js.map +1 -1
- package/dist/console/commands.d.ts.map +1 -1
- package/dist/console/commands.js +5 -5
- package/dist/console/commands.js.map +1 -1
- package/dist/console/root.d.ts.map +1 -1
- package/dist/console/root.js +6 -1
- package/dist/console/root.js.map +1 -1
- package/dist/ext-loader.d.ts.map +1 -1
- package/dist/ext-loader.js +1 -0
- package/dist/ext-loader.js.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +23 -0
- package/dist/index.js.map +1 -1
- package/dist/lib/attach-client.d.ts +37 -4
- package/dist/lib/attach-client.d.ts.map +1 -1
- package/dist/lib/attach-client.js +88 -18
- package/dist/lib/attach-client.js.map +1 -1
- package/dist/lib/auth-proc.d.ts +3 -1
- package/dist/lib/auth-proc.d.ts.map +1 -1
- package/dist/lib/auth-proc.js +15 -4
- package/dist/lib/auth-proc.js.map +1 -1
- package/dist/lib/connect.d.ts +1 -1
- package/dist/lib/connect.d.ts.map +1 -1
- package/dist/lib/connect.js +1 -1
- package/dist/lib/connect.js.map +1 -1
- package/dist/lib/control.d.ts +48 -12
- package/dist/lib/control.d.ts.map +1 -1
- package/dist/lib/control.js +258 -46
- package/dist/lib/control.js.map +1 -1
- package/dist/lib/delivery-proc.d.ts +19 -4
- package/dist/lib/delivery-proc.d.ts.map +1 -1
- package/dist/lib/delivery-proc.js +129 -43
- package/dist/lib/delivery-proc.js.map +1 -1
- package/dist/lib/join-target.d.ts +112 -0
- package/dist/lib/join-target.d.ts.map +1 -0
- package/dist/lib/join-target.js +130 -0
- package/dist/lib/join-target.js.map +1 -0
- package/dist/lib/manager-proc.d.ts +48 -5
- package/dist/lib/manager-proc.d.ts.map +1 -1
- package/dist/lib/manager-proc.js +131 -30
- package/dist/lib/manager-proc.js.map +1 -1
- package/dist/lib/manifest/ledger.d.ts +52 -8
- package/dist/lib/manifest/ledger.d.ts.map +1 -1
- package/dist/lib/manifest/ledger.js +77 -19
- package/dist/lib/manifest/ledger.js.map +1 -1
- package/dist/lib/manifest/live.d.ts +21 -16
- package/dist/lib/manifest/live.d.ts.map +1 -1
- package/dist/lib/manifest/live.js +33 -20
- package/dist/lib/manifest/live.js.map +1 -1
- package/dist/lib/restore.d.ts.map +1 -1
- package/dist/lib/restore.js +21 -3
- package/dist/lib/restore.js.map +1 -1
- package/dist/view/mesh-view.js +2 -2
- package/dist/view/mesh-view.js.map +1 -1
- package/package.json +4 -3
- package/dist/lib/pid.d.ts +0 -23
- package/dist/lib/pid.d.ts.map +0 -1
- package/dist/lib/pid.js +0 -39
- package/dist/lib/pid.js.map +0 -1
package/dist/commands/up.js
CHANGED
|
@@ -2,11 +2,11 @@ import { spawn } from "node:child_process";
|
|
|
2
2
|
import { randomUUID } from "node:crypto";
|
|
3
3
|
import { createConnection, createServer } from "node:net";
|
|
4
4
|
import { hostname } from "node:os";
|
|
5
|
-
import { mkdirSync, writeFileSync, readFileSync, existsSync, openSync, statSync, readSync, closeSync, lstatSync, rmSync, } from "node:fs";
|
|
5
|
+
import { mkdirSync, writeFileSync, readFileSync, existsSync, openSync, statSync, readSync, closeSync, lstatSync, rmSync, realpathSync, } from "node:fs";
|
|
6
6
|
import { join, resolve } from "node:path";
|
|
7
|
-
import { isReachable, DEFAULT_SERVER,
|
|
7
|
+
import { isReachable, DEFAULT_SERVER, createSpaceAuth, serverConfig, openServerConfig, validateTlsMaterial, mintCreds, mintLifecycleUid, DEV_OWNER, mintConnectionEvictorCreds, mintMembershipObserverCreds, newIdentity, setupSpaceStreams, reconcileSpaceTtls, standaloneConnectOpts, seedChannelRegistry, ensureDefaultDeliveryClass, mkSecretDir, writeSecretFile, } from "@cotal-ai/core";
|
|
8
8
|
import { connect } from "@nats-io/transport-node";
|
|
9
|
-
import { assertSingleSpaceBroker, assertUserAuthInfo, authDir, getSoleSpaceAuth, getSpaceAuth, hasUserAuthState, loadSoleSpaceAuth, putSpaceAuth, clearCurrent, findMesh, getCurrent, loadMeshes, MEMBERSHIP_RW_CREDS_KEY, recordMesh, removeMesh, setCurrent, userAuthStateDir, workspaceSecretStore, acquireMaintenanceLock, assertStoreIdentity, assessRestoreClaim, beginOrdinaryResume, bindOrdinaryResumeListener, consumeRetiredMaintenance, markOrdinaryResumeActive, markOrdinaryResumeDegraded, replaceDeadOrdinaryResumeListener, localProcessOwnerStatus, readMaintenanceJournal, readMaintenanceResumeDocument, readStoreIdentity, recordOrdinaryResumeManagerCommit, releaseMaintenanceLock, retireOrdinaryResume, sameStoreIdentity, } from "@cotal-ai/workspace";
|
|
9
|
+
import { assertSingleSpaceBroker, assertUserAuthInfo, authDir, getSoleSpaceAuth, getSpaceAuth, hasUserAuthState, loadSoleSpaceAuth, putSpaceAuth, clearCurrent, findMesh, getCurrent, loadMeshes, MEMBERSHIP_RW_CREDS_KEY, recordMesh, meshesForRoot, removeMesh, rotateSystemCreds, setCurrent, staleSystemCreds, SYSTEM_CREDS_FILES, userAuthStateDir, workspaceSecretStore, acquireMaintenanceLock, assertStoreIdentity, assessRestoreClaim, beginOrdinaryResume, bindOrdinaryResumeListener, consumeRetiredMaintenance, markOrdinaryResumeActive, markOrdinaryResumeDegraded, replaceDeadOrdinaryResumeListener, localProcessOwnerStatus, readMaintenanceJournal, readMaintenanceResumeDocument, readStoreIdentity, recordOrdinaryResumeManagerCommit, releaseMaintenanceLock, retireOrdinaryResume, sameStoreIdentity, readBrokerPolicy, writeBrokerPolicy, } from "@cotal-ai/workspace";
|
|
10
10
|
import { ensureAuthService, resolveAuthProvider, stopAuthService } from "../lib/auth-proc.js";
|
|
11
11
|
import { resolveSpace } from "../lib/status.js";
|
|
12
12
|
import { c } from "../ui.js";
|
|
@@ -32,12 +32,15 @@ export const upFlags = [
|
|
|
32
32
|
{ name: "space", type: "string", value: "<s>", description: "space name (default: the folder's)" },
|
|
33
33
|
{ name: "store-dir", type: "string", value: "<dir>", description: "JetStream store directory" },
|
|
34
34
|
{ name: "channels", type: "string", value: "<path>", description: "channel-registry seed file (JSON; default .cotal/channels.json)" },
|
|
35
|
+
{ name: "tls-cert", type: "string", value: "<file>", description: "serve the broker over TLS with this certificate (requires --tls-key)" },
|
|
36
|
+
{ name: "tls-key", type: "string", value: "<file>", description: "the private key for --tls-cert" },
|
|
35
37
|
{ name: "restore", type: "string", value: "<dir>", description: "restore an offline backup before exposing the normal listener" },
|
|
36
38
|
{ name: "restore-only", type: "string", value: "<registry>", description: "restore only the registry component" },
|
|
37
39
|
{ name: "accept-missing-source", type: "boolean", description: "explicit disaster consent when the inode-bound preserved source is absent" },
|
|
38
40
|
{ name: "open", type: "boolean", description: "unauthenticated dev mesh (no JWT/ACLs)" },
|
|
39
41
|
{ name: "user-auth", type: "boolean", description: "per-USER auth: login + bearer through the space's auth service" },
|
|
40
42
|
{ name: "idp", type: "string", value: "<url>", description: "with --user-auth: the IdP auth base URL to pin (first enable)" },
|
|
43
|
+
{ name: "rotate-sys", type: "boolean", description: "renew the expired/expiring $SYS creds by rotating the system account (agents, data and creds survive; needs a stopped mesh)" },
|
|
41
44
|
{ name: "detach", type: "boolean", description: "run in the background (stop with `cotal down`)" },
|
|
42
45
|
{ name: "runtime", type: "string", value: "<name>", description: "agent runtime for the mesh manager (default pty; extension runtimes are explicit-only, see `cotal runtimes`); with -f overrides the manifest's runtime" },
|
|
43
46
|
{ name: "file", type: "string", short: "f", value: "<cotal.yaml>", description: "launch a whole mesh from a manifest" },
|
|
@@ -63,6 +66,13 @@ export async function up(args) {
|
|
|
63
66
|
if (values.restore) {
|
|
64
67
|
if (values.file || values.channels)
|
|
65
68
|
throw new Error("--restore cannot be combined with --file/-f or --channels");
|
|
69
|
+
// A restore REINSTATES a trust root from an artifact; a rotation SUPERSEDES the one on disk.
|
|
70
|
+
// Together they would restore a system account and retire it in the same command, leaving the
|
|
71
|
+
// operator unable to say which authority the mesh actually came up on, and the artifact's own
|
|
72
|
+
// $SYS creds overwritten by the rotation before anyone verified the restore. Restore, verify,
|
|
73
|
+
// then rotate as its own deliberate step.
|
|
74
|
+
if (values["rotate-sys"])
|
|
75
|
+
throw new Error("--restore cannot be combined with --rotate-sys - restore the space and verify it first, then rotate: `cotal down` then `cotal up --rotate-sys`");
|
|
66
76
|
// A restore rewrites the shared store and the trust root under it, from an artifact that names
|
|
67
77
|
// one space (see `cotal backup`) - it cannot leave the root's other tenants standing.
|
|
68
78
|
assertSingleSpaceBroker(authDir(cotalRoot()), "cotal up --restore");
|
|
@@ -93,8 +103,22 @@ export async function up(args) {
|
|
|
93
103
|
}
|
|
94
104
|
if (values["restore-only"] || values["accept-missing-source"])
|
|
95
105
|
throw new Error("--restore-only and --accept-missing-source require --restore <dir>");
|
|
106
|
+
// MAINTENANCE RE-ENTRY. `--restore` is refused with `--rotate-sys` above, but that guard only sees
|
|
107
|
+
// the EXPLICIT flag: a restore/resume re-entry arrives with `restore` cleared and an `__*Attempt`
|
|
108
|
+
// set, and an auto-recovered journal reaches the same place with no restore flag ever typed. Both
|
|
109
|
+
// re-entries then hit adopt-the-live-listener paths that RETURN before `authSetup`, so the flag
|
|
110
|
+
// would be accepted, nothing would rotate, and the command would exit 0. That is the silent
|
|
111
|
+
// success this whole change exists to remove, so it is refused here, before any attempt state is
|
|
112
|
+
// read. A rotation is a stopped, fresh boot; a half-finished maintenance attempt is neither.
|
|
113
|
+
if (values["rotate-sys"] && (values.__restoreAttempt || values.__ordinaryResumeAttempt))
|
|
114
|
+
throw new Error("--rotate-sys cannot run during a restore/resume re-entry - finish or roll back the maintenance attempt, then `cotal down` and `cotal up --rotate-sys`");
|
|
96
115
|
if (!values.__restoreAttempt && !values.__ordinaryResumeAttempt && !values.file) {
|
|
97
116
|
const root = cotalRoot();
|
|
117
|
+
// Same refusal for the AUTO-recovered journal, raised before the recovery is prepared and
|
|
118
|
+
// re-entered, so the operator reads one clear error instead of one thrown from inside a nested
|
|
119
|
+
// `up` that has already begun adopting a maintenance attempt.
|
|
120
|
+
if (values["rotate-sys"] && readMaintenanceJournal(root))
|
|
121
|
+
throw new Error("--rotate-sys is refused while this root has a maintenance attempt to recover - finish or roll back that attempt (`cotal up` alone recovers it), then `cotal down` and `cotal up --rotate-sys`");
|
|
98
122
|
const lock = acquireMaintenanceLock(root);
|
|
99
123
|
let pending;
|
|
100
124
|
let recoveredRestore;
|
|
@@ -304,6 +328,77 @@ export async function up(args) {
|
|
|
304
328
|
if (values.idp && !wantUser && !values.file) {
|
|
305
329
|
throw new Error('--idp is for user-auth spaces; pair it with --user-auth, or set broker.auth: "user" in a manifest');
|
|
306
330
|
}
|
|
331
|
+
// An open mesh has no operator, no system account, and no $SYS creds, so there is nothing to
|
|
332
|
+
// rotate; the request is a misunderstanding to name, never a silent no-op that reports success.
|
|
333
|
+
if (values["rotate-sys"] && values.open) {
|
|
334
|
+
throw new Error("--rotate-sys is for auth meshes: an open mesh (--open) has no system account or $SYS credentials to rotate");
|
|
335
|
+
}
|
|
336
|
+
// THE EFFECTIVE PROJECT ROOT IS PINNED BEFORE THE TRANSPORT IS DECIDED.
|
|
337
|
+
//
|
|
338
|
+
// `ensureRootForSpace` may create `cwd/.cotal` when the nearest ancestor root already owns a
|
|
339
|
+
// different auth space — the child becomes its own mesh root. That pin used to run AFTER
|
|
340
|
+
// `resolveTransport`, so a TLS policy was written to the PARENT root while the listener and
|
|
341
|
+
// MeshEntry landed on the CHILD. First `up --tls-*` looked green (transport was in memory);
|
|
342
|
+
// `down` then bare `up` from the child found no policy and served plaintext, rewriting
|
|
343
|
+
// `tlsRequired: false`. Same class as B4/B5: operator-visible TLS success, silent cleartext on
|
|
344
|
+
// the documented retain path. Also polluted the parent's policy file.
|
|
345
|
+
//
|
|
346
|
+
// Dry-run must not pin (mkdir is a mutation). Validation still uses whatever root is visible;
|
|
347
|
+
// persistence is already suppressed below.
|
|
348
|
+
if (!values["dry-run"]) {
|
|
349
|
+
const spaceForRoot = values.space ?? resolveSpace(process.cwd());
|
|
350
|
+
if (values.file) {
|
|
351
|
+
try {
|
|
352
|
+
const prepared = loadManifest(resolve(values.file));
|
|
353
|
+
const space = values.space ?? prepared.manifest.space;
|
|
354
|
+
const open = Boolean(values.open) || prepared.manifest.broker?.auth === false;
|
|
355
|
+
ensureRootForSpace(!open, space);
|
|
356
|
+
}
|
|
357
|
+
catch {
|
|
358
|
+
// Manifest errors are reported by the -f path with the same load; do not double-print.
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
else {
|
|
362
|
+
ensureRootForSpace(!values.open, spaceForRoot);
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
// THE TRANSPORT IS DECIDED HERE, ABOVE EVERY BRANCH, AND THIS POSITION IS THE FIX.
|
|
366
|
+
//
|
|
367
|
+
// It used to be resolved inside the foreground path only, which meant three routes to a listener
|
|
368
|
+
// never saw it: `up -f` (manifest), `up --detach`, and the already-running refresh. Each accepted
|
|
369
|
+
// `--tls-cert`/`--tls-key`, validated nothing, and served PLAINTEXT while printing `✓ mesh up`.
|
|
370
|
+
// Every one of those was a separate exit from the same room, and closing them one at a time is
|
|
371
|
+
// what produced three rounds of the same defect.
|
|
372
|
+
//
|
|
373
|
+
// A guard only fences what comes AFTER it, so the decision has to dominate the branch rather than
|
|
374
|
+
// sit in one arm of it. The dial-host SAN is deliberately NOT checked here — the effective host is
|
|
375
|
+
// not known until a route picks it — so each route re-asserts it via `assertServesDialHost` once
|
|
376
|
+
// it does. Decide early so nothing can skip it; check late so it checks the right host.
|
|
377
|
+
// `persist` is false on a dry run, and that is a REGRESSION THIS HOIST CAUSED. Before the move,
|
|
378
|
+
// `resolveTransport` ran inside the launch path and a `--dry-run` never reached it. Hoisting it
|
|
379
|
+
// above the branch is what makes the transport dominate every route — and it also put a WRITE in
|
|
380
|
+
// front of a command whose entire contract is "mutate nothing", so `up -f --dry-run --tls-cert`
|
|
381
|
+
// printed "Dry run - nothing was changed" and left a broker-policy.json behind.
|
|
382
|
+
//
|
|
383
|
+
// Validation still runs: a dry run should absolutely refuse an expired or unreadable cert, and
|
|
384
|
+
// reporting that is the point of it. Only the persistence is suppressed. An instrument that
|
|
385
|
+
// modifies what it inspects is a defect even when everything it reports is true.
|
|
386
|
+
//
|
|
387
|
+
// The root argument is the PINNED root (or the walked root on dry-run). One root for policy,
|
|
388
|
+
// auth, store, and MeshEntry — never a second walk that can disagree after a later pin.
|
|
389
|
+
//
|
|
390
|
+
// PERSIST IS ALWAYS FALSE HERE (commit-after-apply). resolveTransport used to write the policy
|
|
391
|
+
// before the already-running refresh branch could refuse a transport change, so a failed
|
|
392
|
+
// `up --tls-*` against a live plaintext mesh left tls-required on disk; the next bare `up`
|
|
393
|
+
// printed "TLS: inheriting" / "TLS: serving" / green already-running over a cleartext listener
|
|
394
|
+
// (S5). Decide and validate early so every branch sees the transport; write only after a
|
|
395
|
+
// matching listener is started and proved (`commitTransportPolicy`), never on a refuse path.
|
|
396
|
+
// `quiet` suppresses the serving/inheriting lines until that commit (or a dry-run announce).
|
|
397
|
+
const meshRoot = cotalRoot();
|
|
398
|
+
const transport = resolveTransport(values, undefined, meshRoot, {
|
|
399
|
+
persist: false,
|
|
400
|
+
quiet: true,
|
|
401
|
+
});
|
|
307
402
|
// `up -f cotal.yaml` is a distinct path: bring up a FRESH mesh described by a manifest (broker +
|
|
308
403
|
// channels + booted agents). It owns the whole space; deploying onto a RUNNING mesh is `spawn -f`.
|
|
309
404
|
// CLI flags override the manifest (flag > manifest > default) so the same file runs at a different
|
|
@@ -311,6 +406,7 @@ export async function up(args) {
|
|
|
311
406
|
if (values.file) {
|
|
312
407
|
if (values["dry-run"]) {
|
|
313
408
|
await upManifest(values.file, {
|
|
409
|
+
transport,
|
|
314
410
|
dryRun: true,
|
|
315
411
|
server: values.server,
|
|
316
412
|
host: values.host,
|
|
@@ -319,6 +415,7 @@ export async function up(args) {
|
|
|
319
415
|
open: values.open,
|
|
320
416
|
userAuth: wantUser,
|
|
321
417
|
idp: values.idp,
|
|
418
|
+
rotateSys: values["rotate-sys"],
|
|
322
419
|
});
|
|
323
420
|
return;
|
|
324
421
|
}
|
|
@@ -326,6 +423,7 @@ export async function up(args) {
|
|
|
326
423
|
try {
|
|
327
424
|
assertOrdinaryUpAllowed(cotalRoot());
|
|
328
425
|
await upManifest(values.file, {
|
|
426
|
+
transport,
|
|
329
427
|
dryRun: Boolean(values["dry-run"]),
|
|
330
428
|
server: values.server,
|
|
331
429
|
host: values.host,
|
|
@@ -334,6 +432,7 @@ export async function up(args) {
|
|
|
334
432
|
open: values.open,
|
|
335
433
|
userAuth: wantUser,
|
|
336
434
|
idp: values.idp,
|
|
435
|
+
rotateSys: values["rotate-sys"],
|
|
337
436
|
});
|
|
338
437
|
}
|
|
339
438
|
finally {
|
|
@@ -417,6 +516,78 @@ export async function up(args) {
|
|
|
417
516
|
console.error(c.red(`✗ mesh "${held.space}" is already running at ${server} with ${label} - a running broker can't change auth mode; \`cotal down\` it first, then \`cotal up ${wantUser ? "--user-auth" : "--open"}\``));
|
|
418
517
|
process.exit(1);
|
|
419
518
|
}
|
|
519
|
+
// A rotation retires the system account this LIVE broker was started on, and only a broker
|
|
520
|
+
// (re)started from the rewritten config carries the successor. Rotating under a running mesh
|
|
521
|
+
// would leave the record and the creds a generation ahead of the broker: every $SYS client
|
|
522
|
+
// denied, with `doctor auth` reporting freshly-minted creds. Refuse with the two-step recipe.
|
|
523
|
+
if (values["rotate-sys"]) {
|
|
524
|
+
console.error(c.red(`✗ mesh "${held.space}" is already running at ${server} - a running broker can't rotate its system account (it would keep serving the retired one); \`cotal down\` it first, then \`cotal up --rotate-sys\``));
|
|
525
|
+
process.exit(1);
|
|
526
|
+
}
|
|
527
|
+
// A running broker cannot change its transport either, and for exactly the reason it cannot
|
|
528
|
+
// change its auth mode: the listener's TLS config was fixed when nats-server read its config
|
|
529
|
+
// file. This branch starts nothing, so `--tls-cert`/`--tls-key` here can only be a request to
|
|
530
|
+
// change something that is already decided — and answering it with `✓ mesh up` told the
|
|
531
|
+
// operator they had TLS while the live broker went on serving whatever it was started with.
|
|
532
|
+
//
|
|
533
|
+
// Refuse rather than warn. The whole feature is that a command accepting a TLS flag either
|
|
534
|
+
// encrypts or refuses to start; printing a success line over an unchanged plaintext listener
|
|
535
|
+
// is the precise outcome that must be unreachable.
|
|
536
|
+
if (values["tls-cert"] || values["tls-key"]) {
|
|
537
|
+
console.error(c.red(`✗ mesh "${held.space}" is already running at ${server} - a running broker can't change its transport; \`cotal down\` it first, then \`cotal up --tls-cert <cert> --tls-key <key>\``));
|
|
538
|
+
process.exit(1);
|
|
539
|
+
}
|
|
540
|
+
// Live INFO must agree with the recorded/requested transport. A bare refresh that greets
|
|
541
|
+
// "already running" over a plaintext listener while policy claims TLS is S5's second half —
|
|
542
|
+
// the refuse left a durable lie and this path used to reprint TLS success over cleartext.
|
|
543
|
+
const liveTls = await liveListenerRequiresTls(server);
|
|
544
|
+
const wantTls = transport.kind === "tls-required" || held.tlsRequired === true;
|
|
545
|
+
if (wantTls && liveTls === false) {
|
|
546
|
+
console.error(c.red(`✗ mesh "${held.space}" at ${server} is recorded/expected TLS-required but the live listener is plaintext - refuse rather than claim it is up; \`cotal down\` then \`cotal up --tls-cert <cert> --tls-key <key>\` (or fix the broker)`));
|
|
547
|
+
process.exit(1);
|
|
548
|
+
}
|
|
549
|
+
if (!wantTls && liveTls === true) {
|
|
550
|
+
console.error(c.red(`✗ mesh "${held.space}" at ${server} is serving TLS but this mesh is recorded plaintext - \`cotal down\` then re-up with matching flags`));
|
|
551
|
+
process.exit(1);
|
|
552
|
+
}
|
|
553
|
+
// #286: reconcile the presence/lease bucket TTLs, HERE, before the success line.
|
|
554
|
+
//
|
|
555
|
+
// This branch is the upgrade path. A mesh created before the bucket TTLs existed keeps no
|
|
556
|
+
// `max_age` forever, so dead presence records never expire and the roster reports a despawned
|
|
557
|
+
// agent as live — and that mesh is by definition ALREADY RUNNING, which is precisely the case
|
|
558
|
+
// that returns from here without ever reaching `postStart`/`setupSpaceStreams`. Reconciling
|
|
559
|
+
// only on the create path fixes the deployments that never had the bug.
|
|
560
|
+
//
|
|
561
|
+
// BEFORE the success print, not after: `✓ already running` is a claim about a healthy mesh,
|
|
562
|
+
// and printing it over an unreconciled one is the silent-drift failure this change exists to
|
|
563
|
+
// remove. A reconcile that cannot complete is a loud refusal, not a footnote under a tick.
|
|
564
|
+
//
|
|
565
|
+
// Read-first, so this stays a no-op in steady state: each bucket is skipped when its `max_age`
|
|
566
|
+
// already matches, so a repeat `cotal up` on a current mesh issues three reads and no writes.
|
|
567
|
+
// When it DOES write it says so — a bare `cotal up` now performs a config write on a running
|
|
568
|
+
// mesh, and an operator should never have to infer that from silence.
|
|
569
|
+
try {
|
|
570
|
+
// Open meshes hold no creds and need none (a bare connection has the rights) — but they DO
|
|
571
|
+
// carry the same three TTL'd buckets, so they drift identically and are reconciled too.
|
|
572
|
+
let reconcileCreds;
|
|
573
|
+
if (held.mode !== "open") {
|
|
574
|
+
const spaceAuth = await getSpaceAuth(workspaceSecretStore(root), held.space);
|
|
575
|
+
if (!spaceAuth) {
|
|
576
|
+
console.error(c.red(`✗ mesh "${held.space}" has no trust material under ${authDir(root)} - cannot reconcile its presence/lease TTLs; restore or re-provision \`.cotal/auth\``));
|
|
577
|
+
process.exit(1);
|
|
578
|
+
}
|
|
579
|
+
// Ephemeral, reconcile-only, discarded with the connection. Same enumerated `provisioner`
|
|
580
|
+
// scope the create path mints — no principal gains authority it did not already have, since
|
|
581
|
+
// a same-root caller holds the space's signing material either way.
|
|
582
|
+
reconcileCreds = await mintCreds(spaceAuth, newIdentity(), "provisioner");
|
|
583
|
+
}
|
|
584
|
+
for (const r of await reconcileSpaceTtls({ servers: server, space: held.space, creds: reconcileCreds }))
|
|
585
|
+
console.log(c.dim(` reconciled ${r.stream} TTL ${r.fromMs === 0 ? "none" : `${r.fromMs}ms`} -> ${r.toMs}ms`));
|
|
586
|
+
}
|
|
587
|
+
catch (e) {
|
|
588
|
+
console.error(c.red(`✗ mesh "${held.space}" is running at ${server} but its presence/lease TTLs could not be reconciled: ${e.message}`));
|
|
589
|
+
process.exit(1);
|
|
590
|
+
}
|
|
420
591
|
console.log(c.green(`✓ mesh "${held.space}" already running at ${server}`));
|
|
421
592
|
// USER MODE: re-upping IS the documented recovery for a dead auth service (the provider's
|
|
422
593
|
// failure copy says "restart it with `cotal up`"), so a refresh must re-ensure the service —
|
|
@@ -480,10 +651,26 @@ export async function up(args) {
|
|
|
480
651
|
const heldAttachHost = attachHostFor(held.space, values.host);
|
|
481
652
|
// A broker was already answering here — this branch starts nothing, so it must not claim the
|
|
482
653
|
// record as ours (see `Provenance`).
|
|
483
|
-
|
|
654
|
+
// `tlsRequired` is CARRIED FORWARD, not re-derived. This branch starts no listener, so it has
|
|
655
|
+
// no transport decision to record — and `recordOurMesh` writes the entry whole, so omitting
|
|
656
|
+
// the field here would erase the requirement on every bare refresh, exactly the way dropping
|
|
657
|
+
// `attachHost` would silently demote the mesh to loopback.
|
|
658
|
+
recordOurMesh({ space: held.space, server, root, mode: held.mode, ...(held.tlsRequired !== undefined ? { tlsRequired: held.tlsRequired } : {}), ...(userAuth ? { userAuth } : {}), ...(heldAttachHost ? { attachHost: heldAttachHost } : {}), ts: new Date().toISOString() }, "refresh");
|
|
484
659
|
return;
|
|
485
660
|
}
|
|
486
661
|
const who = held ? `mesh "${held.space}" (${held.root})` : "a broker not started here";
|
|
662
|
+
// Reaching here with `--rotate-sys` means something IS answering at the requested address and it
|
|
663
|
+
// is not this root's recorded mesh (that case was refused above). The ordinary response is to pick
|
|
664
|
+
// a free port and carry on, which for a rotation is the wrong instinct: the unidentified listener
|
|
665
|
+
// may be a `nats-server -c <root>/.cotal/auth/server.conf` started by hand, holding THIS root's
|
|
666
|
+
// config and JetStream store while writing neither a pidfile nor a registry row. Rotating around
|
|
667
|
+
// it retires the account it is still serving and opens its store a second time. Nothing available
|
|
668
|
+
// here can identify it (that is what unidentified means), so refuse instead of stepping past it.
|
|
669
|
+
if (values["rotate-sys"]) {
|
|
670
|
+
console.error(c.red(`✗ ${server} is answering and it is not this root's recorded mesh (${who})`) +
|
|
671
|
+
c.dim(" - `--rotate-sys` will not start on another port around an unidentified broker: it may be serving this root's own server.conf and JetStream store. Stop whatever is listening there (`cotal down` if it was started here), then rotate."));
|
|
672
|
+
process.exit(1);
|
|
673
|
+
}
|
|
487
674
|
if (values.server === undefined && (!held || held.root !== root)) {
|
|
488
675
|
const next = await serverWithFreePort(server, host);
|
|
489
676
|
console.log(c.dim(`${server} is already in use by ${who}; starting "${space}" at ${next} instead`));
|
|
@@ -497,11 +684,13 @@ export async function up(args) {
|
|
|
497
684
|
if (values.detach) {
|
|
498
685
|
const restored = resumeAttempt ? pendingRestores.get(resumeAttempt) : undefined;
|
|
499
686
|
const { pid, source, authService, controlPlane, delivery, manager } = await startMeshDetached({
|
|
687
|
+
transport,
|
|
500
688
|
server,
|
|
501
689
|
storeDir: values["store-dir"],
|
|
502
690
|
space: values.space,
|
|
503
691
|
open: values.open,
|
|
504
692
|
userAuth: wantUser ? { idpUrl: values.idp } : undefined,
|
|
693
|
+
rotateSys: values["rotate-sys"],
|
|
505
694
|
channels: values.channels,
|
|
506
695
|
// The RAW flag, not the loopback-defaulted `host` above: `startMeshDetached` applies the same
|
|
507
696
|
// default itself, and what it records must distinguish "the operator asked for this address"
|
|
@@ -527,6 +716,7 @@ export async function up(args) {
|
|
|
527
716
|
} : {}),
|
|
528
717
|
skipPostStart: Boolean(resumeAttempt),
|
|
529
718
|
});
|
|
719
|
+
// Transport policy is committed inside startMeshDetached before delivery launch (S5+S9).
|
|
530
720
|
console.log(c.dim(`Started nats-server (${source}).`));
|
|
531
721
|
console.log(c.green(renderDetachedSummary({ pid, delivery, authService: wantUser && authService, manager })));
|
|
532
722
|
if (restored && process.env.COTAL_SMOKE_FAIL_AFTER_RESTORE_LISTENER_READY === "1")
|
|
@@ -556,11 +746,19 @@ export async function up(args) {
|
|
|
556
746
|
mkdirSync(storeDir, { recursive: true });
|
|
557
747
|
await claimSpace(space, server, cotalRoot());
|
|
558
748
|
const seedFile = loadChannelsFile(values.channels);
|
|
559
|
-
|
|
749
|
+
// Decided above the branch; the dial host is only settled here (a port collision may have moved
|
|
750
|
+
// the server, and the hostname is what the certificate has to match).
|
|
751
|
+
assertServesDialHost(transport, new URL(server).hostname);
|
|
752
|
+
const setup = useAuth ? await authSetup(storeDir, server, space, host, wantUser ? { idpUrl: values.idp } : undefined, transport, values["rotate-sys"]) : undefined;
|
|
560
753
|
const port = Number(new URL(server).port) || 4222;
|
|
561
754
|
const restored = resumeAttempt ? pendingRestores.get(resumeAttempt) : undefined;
|
|
755
|
+
// Both modes go through a RENDERER, never bare CLI flags. Open mode used to start from
|
|
756
|
+
// `-js -sd … -p … -a …`, which never called a renderer at all — so the required transport union
|
|
757
|
+
// protected the auth path and was silent on the open one, and a cert/key pair passed to an
|
|
758
|
+
// open-mode `up` would have been accepted while the listener came up in cleartext.
|
|
759
|
+
const confPath = setup ? setup.confPath : writeOpenBrokerConf(storeDir, { port, host, transport });
|
|
562
760
|
const natsArgs = [
|
|
563
|
-
|
|
761
|
+
"-c", confPath,
|
|
564
762
|
...(restored ? ["--name", restored.serverName]
|
|
565
763
|
: ordinaryAttempt ? ["--name", ordinaryAttempt.serverName]
|
|
566
764
|
: []),
|
|
@@ -611,8 +809,9 @@ export async function up(args) {
|
|
|
611
809
|
void stopDelivery()
|
|
612
810
|
.catch((e) => console.error(`! delivery teardown: ${e.message}`))
|
|
613
811
|
.then(() => {
|
|
614
|
-
stopManager()
|
|
615
|
-
|
|
812
|
+
void stopManager()
|
|
813
|
+
.then(() => stopAuthService(space))
|
|
814
|
+
.catch((e) => console.error(`! teardown: ${e.message}`));
|
|
616
815
|
child.kill("SIGTERM");
|
|
617
816
|
});
|
|
618
817
|
};
|
|
@@ -624,8 +823,8 @@ export async function up(args) {
|
|
|
624
823
|
rmSync(cotalPath("nats.pid"), { force: true });
|
|
625
824
|
// Logged, never silently swallowed; the daemon kill runs in stopDelivery's finally regardless.
|
|
626
825
|
await stopDelivery().catch((e) => console.error(`! delivery teardown: ${e.message}`));
|
|
627
|
-
stopManager();
|
|
628
|
-
stopAuthService(space);
|
|
826
|
+
await stopManager().catch((e) => console.error(`! manager teardown: ${e.message}`));
|
|
827
|
+
await stopAuthService(space).catch((e) => console.error(`! auth teardown: ${e.message}`));
|
|
629
828
|
// Only unrecord if the registry still points at THIS broker. A newer broker for the same space
|
|
630
829
|
// (a concurrent `up`, or a different-port re-up that recorded after us) may have replaced our
|
|
631
830
|
// record — removing by name would clobber the live winner and hide it from the registry.
|
|
@@ -651,6 +850,8 @@ export async function up(args) {
|
|
|
651
850
|
}
|
|
652
851
|
if (restored)
|
|
653
852
|
await provePreparedRestoreListener(restored);
|
|
853
|
+
// Listener ready — commit the transport decision (S5: not before start).
|
|
854
|
+
commitTransportPolicy(meshRoot, transport);
|
|
654
855
|
{
|
|
655
856
|
if (!resumeAttempt)
|
|
656
857
|
await postStart(server, space, setup, seedFile);
|
|
@@ -671,6 +872,11 @@ export async function up(args) {
|
|
|
671
872
|
recordOurMesh({
|
|
672
873
|
space, server, root: cotalRoot(),
|
|
673
874
|
mode: setup?.prepared ? "user" : useAuth ? "auth" : "open",
|
|
875
|
+
// Written ALWAYS, as a boolean, unlike `attachHost` below. Absence and `false` resolve
|
|
876
|
+
// identically for clients, so an omitted field would be indistinguishable from a deliberate
|
|
877
|
+
// plaintext mesh — and this is the one field whose whole purpose is that the answer was
|
|
878
|
+
// stated rather than defaulted.
|
|
879
|
+
tlsRequired: transport.kind === "tls-required",
|
|
674
880
|
...(svc.userAuth ? { userAuth: svc.userAuth } : {}),
|
|
675
881
|
// Only a real decision is persisted — an explicit `--host` now, or one carried forward from a
|
|
676
882
|
// previous launch. The bare case stays absent rather than recording the loopback default as
|
|
@@ -690,6 +896,8 @@ export async function up(args) {
|
|
|
690
896
|
attachHost: effectiveAttachHost,
|
|
691
897
|
resumeAttempt,
|
|
692
898
|
resumeCommitToken: restored?.managerCommit?.durableCommitToken ?? ordinaryAttempt?.managerCommit?.durableCommitToken,
|
|
899
|
+
transport,
|
|
900
|
+
wsPort: setup?.wsPort, // P2 item 6: the console session client's broker ws port
|
|
693
901
|
});
|
|
694
902
|
if (restored && process.env.COTAL_SMOKE_FAIL_AFTER_RESTORE_LISTENER_READY === "1")
|
|
695
903
|
throw new Error("smoke-injected failure after restore listener readiness");
|
|
@@ -755,10 +963,15 @@ async function resumeControlAuth(root, mode) {
|
|
|
755
963
|
if (!auth)
|
|
756
964
|
throw new Error("same-principal resume requires the existing space trust material");
|
|
757
965
|
const identity = newIdentity();
|
|
966
|
+
// The instrument's ep caller triple (1c.2b): the admin instrument's rows are lifecycle-keyed,
|
|
967
|
+
// and the triple rides back so the resume/preservation calls take askManager's ep path.
|
|
968
|
+
const uid = mintLifecycleUid();
|
|
758
969
|
return {
|
|
759
970
|
creds: await mintCreds(auth, identity, "control-caller-admin", {
|
|
971
|
+
lifecycleUid: uid,
|
|
760
972
|
expiresAt: Math.floor((Date.now() + 30 * 60 * 1000) / 1000),
|
|
761
973
|
}),
|
|
974
|
+
epCaller: { owner: DEV_OWNER, actor: identity.id, uid },
|
|
762
975
|
};
|
|
763
976
|
}
|
|
764
977
|
function restoreListenerOwner(pid, nonce, startedAt) {
|
|
@@ -910,7 +1123,7 @@ async function provePreparedRestoreListener(prepared) {
|
|
|
910
1123
|
const auth = await resumeControlAuth(prepared.root, prepared.mode);
|
|
911
1124
|
const nc = await connect({
|
|
912
1125
|
servers: prepared.server,
|
|
913
|
-
...standaloneConnectOpts(auth),
|
|
1126
|
+
...standaloneConnectOpts({ ...auth, /* not yet wired to a recorded transport - see broker-policy/MeshEntry work */ tls: false }),
|
|
914
1127
|
maxReconnectAttempts: 0,
|
|
915
1128
|
});
|
|
916
1129
|
try {
|
|
@@ -992,7 +1205,7 @@ async function proveOrdinaryResumeListener(pending) {
|
|
|
992
1205
|
const auth = await resumeControlAuth(pending.root, pending.mode);
|
|
993
1206
|
const nc = await connect({
|
|
994
1207
|
servers: pending.server,
|
|
995
|
-
...standaloneConnectOpts(auth),
|
|
1208
|
+
...standaloneConnectOpts({ ...auth, /* not yet wired to a recorded transport - see broker-policy/MeshEntry work */ tls: false }),
|
|
996
1209
|
maxReconnectAttempts: 0,
|
|
997
1210
|
});
|
|
998
1211
|
try {
|
|
@@ -1025,6 +1238,8 @@ async function resumeProvenOrdinaryListener(pending) {
|
|
|
1025
1238
|
server: pending.server,
|
|
1026
1239
|
root: pending.root,
|
|
1027
1240
|
mode: pending.mode,
|
|
1241
|
+
// Adopted from the recorded policy: this path proves and re-adopts a listener it did not start.
|
|
1242
|
+
tlsRequired: adoptedTlsRequired(pending.root),
|
|
1028
1243
|
...(svc.userAuth ? { userAuth: svc.userAuth } : {}),
|
|
1029
1244
|
...(adoptAttachHost ? { attachHost: adoptAttachHost } : {}),
|
|
1030
1245
|
ts: new Date().toISOString(),
|
|
@@ -1064,6 +1279,9 @@ async function resumeProvenRestoreListener(prepared) {
|
|
|
1064
1279
|
server: prepared.server,
|
|
1065
1280
|
root: prepared.root,
|
|
1066
1281
|
mode: prepared.mode,
|
|
1282
|
+
// Same as the ordinary-resume path: a restore adopts an existing listener, so the requirement
|
|
1283
|
+
// comes from the policy that listener was started from, never from a default.
|
|
1284
|
+
tlsRequired: adoptedTlsRequired(prepared.root),
|
|
1067
1285
|
...(svc.userAuth ? { userAuth: svc.userAuth } : {}),
|
|
1068
1286
|
...(restoreAttachHost ? { attachHost: restoreAttachHost } : {}),
|
|
1069
1287
|
ts: new Date().toISOString(),
|
|
@@ -1129,7 +1347,7 @@ async function completeResumeActivation(attemptId, healthy, reason, server) {
|
|
|
1129
1347
|
const auth = await resumeControlAuth(pending.root, pending.mode);
|
|
1130
1348
|
const readinessDeadline = Date.now() + 20_000;
|
|
1131
1349
|
for (;;) {
|
|
1132
|
-
const ready = await askManager(pending.space, server, "ps", undefined, auth,
|
|
1350
|
+
const ready = await askManager(pending.space, server, "ps", undefined, auth, "any", 2_000);
|
|
1133
1351
|
if (!ready.error?.startsWith("no manager reachable"))
|
|
1134
1352
|
break;
|
|
1135
1353
|
if (Date.now() >= readinessDeadline) {
|
|
@@ -1143,7 +1361,7 @@ async function completeResumeActivation(attemptId, healthy, reason, server) {
|
|
|
1143
1361
|
inventory: restored?.selection === "registry"
|
|
1144
1362
|
? { ...pending.inventory, agents: [] }
|
|
1145
1363
|
: pending.inventory,
|
|
1146
|
-
}, auth,
|
|
1364
|
+
}, auth, "any", 10 * 60 * 1000);
|
|
1147
1365
|
if (!resumed.ok) {
|
|
1148
1366
|
const detail = resumed.data ? ` (${JSON.stringify(resumed.data)})` : "";
|
|
1149
1367
|
const message = `${resumed.error ?? "retained-agent resume failed"}${detail}`;
|
|
@@ -1171,7 +1389,7 @@ async function completeResumeActivation(attemptId, healthy, reason, server) {
|
|
|
1171
1389
|
}
|
|
1172
1390
|
}
|
|
1173
1391
|
if (!managerCommit) {
|
|
1174
|
-
const committed = await askManager(pending.space, server, "commitResume", { attemptId }, auth,
|
|
1392
|
+
const committed = await askManager(pending.space, server, "commitResume", { attemptId }, auth, "any", 40_000);
|
|
1175
1393
|
if (!committed.ok) {
|
|
1176
1394
|
const message = committed.error ?? "manager resume commit failed";
|
|
1177
1395
|
markPendingResumeDegraded(attemptId, message);
|
|
@@ -1201,7 +1419,7 @@ async function completeResumeActivation(attemptId, healthy, reason, server) {
|
|
|
1201
1419
|
process.exit(89);
|
|
1202
1420
|
}
|
|
1203
1421
|
else {
|
|
1204
|
-
const recommitted = await askManager(pending.space, server, "commitResume", { attemptId }, auth,
|
|
1422
|
+
const recommitted = await askManager(pending.space, server, "commitResume", { attemptId }, auth, "any", 40_000);
|
|
1205
1423
|
// A surviving manager that already finalized legitimately answers {state:"active"} with the
|
|
1206
1424
|
// exact durable token: accept both committed shapes idempotently, then reissue the token-bound
|
|
1207
1425
|
// finalize (itself idempotent) below.
|
|
@@ -1213,7 +1431,7 @@ async function completeResumeActivation(attemptId, healthy, reason, server) {
|
|
|
1213
1431
|
if (!exactToken)
|
|
1214
1432
|
throw new Error(`replacement manager did not recover the durable commit token for attempt ${attemptId}`);
|
|
1215
1433
|
}
|
|
1216
|
-
const finalized = await askManager(pending.space, server, "finalizeResume", { attemptId, durableCommitToken: managerCommit.durableCommitToken }, auth,
|
|
1434
|
+
const finalized = await askManager(pending.space, server, "finalizeResume", { attemptId, durableCommitToken: managerCommit.durableCommitToken }, auth, "any", 40_000);
|
|
1217
1435
|
if (!finalized.ok)
|
|
1218
1436
|
throw new Error(finalized.error ?? `manager resume finalize failed for attempt ${attemptId}`);
|
|
1219
1437
|
if (!isManagerFinalizeResult(finalized.data, attemptId))
|
|
@@ -1305,7 +1523,21 @@ async function upManifest(file, opts) {
|
|
|
1305
1523
|
const open = m.broker?.auth === false; // default is auth
|
|
1306
1524
|
const userAuth = m.broker?.auth === "user" ? { idpUrl: opts.idp ?? m.broker?.idp } : undefined; // flag > manifest
|
|
1307
1525
|
const runtime = m.runtime ?? "pty";
|
|
1526
|
+
// The open-mesh refusal must be RE-STATED here, not only against the CLI `--open` flag: on this
|
|
1527
|
+
// path openness comes from the MANIFEST (`broker.auth: false`), which the flag-level guard cannot
|
|
1528
|
+
// see. Without it, `cotal up -f open.yaml --rotate-sys` boots an open broker, exits 0 and rotates
|
|
1529
|
+
// nothing: a rotation ask answered with silent success, the exact failure class this change
|
|
1530
|
+
// exists to remove. Before the dry-run print and before anything boots.
|
|
1531
|
+
if (opts.rotateSys && open)
|
|
1532
|
+
throw new Error("--rotate-sys is for auth meshes: this manifest sets broker.auth: false (open), so there is no system account or $SYS credentials to rotate");
|
|
1308
1533
|
if (opts.dryRun) {
|
|
1534
|
+
// Dial-host SAN must refuse here too (S6). The real route runs assertServesDialHost after
|
|
1535
|
+
// the plan is computed; dry-run used to return before that check and green-lit a launch
|
|
1536
|
+
// the applying command refuses. Validate against the effective server; still no writes.
|
|
1537
|
+
// Announce only AFTER the dial-host check so a wrong-SAN dry-run does not print TLS: serving.
|
|
1538
|
+
assertServesDialHost(opts.transport, new URL(server).hostname);
|
|
1539
|
+
if (opts.transport.kind === "tls-required")
|
|
1540
|
+
announceTransport(opts.transport);
|
|
1309
1541
|
console.log(renderUpPlan(eff, server));
|
|
1310
1542
|
return;
|
|
1311
1543
|
}
|
|
@@ -1332,7 +1564,10 @@ async function upManifest(file, opts) {
|
|
|
1332
1564
|
if (userAuth) {
|
|
1333
1565
|
try {
|
|
1334
1566
|
mkdirSync(cotalPath("nats"), { recursive: true });
|
|
1335
|
-
|
|
1567
|
+
// NO `rotateSys` here on purpose: this is an owner-derivation pre-resolve that runs BEFORE the
|
|
1568
|
+
// real boot, and the boot below carries the flag. Rotating in both would burn two generations
|
|
1569
|
+
// per `up -f --rotate-sys` and retire the account the first pass just minted creds against.
|
|
1570
|
+
const setup = await authSetup(cotalPath("nats"), server, m.space, host, userAuth, assertServesDialHost(opts.transport, new URL(server).hostname));
|
|
1336
1571
|
owner = await resolveAuthProvider().ownerForLogin({ store: workspaceSecretStore(cotalRoot()), dir: setup.stateDir, space: m.space });
|
|
1337
1572
|
}
|
|
1338
1573
|
catch (e) {
|
|
@@ -1350,19 +1585,20 @@ async function upManifest(file, opts) {
|
|
|
1350
1585
|
// A leftover detached manager (its broker is gone — the reachability check above proved nothing
|
|
1351
1586
|
// lives at this address) would win the fresh mesh's lease and the launch manager would refuse.
|
|
1352
1587
|
// Stop it, so the manager started below WITH the launch spec is THE manager.
|
|
1353
|
-
stopManager();
|
|
1588
|
+
await stopManager();
|
|
1354
1589
|
let pid;
|
|
1355
1590
|
let controlPlane = false;
|
|
1356
1591
|
let authService = true;
|
|
1357
1592
|
try {
|
|
1358
1593
|
// `m.broker?.host`, not the defaulted `host`: same reason as the flag path — only a host the
|
|
1359
1594
|
// manifest actually declared is an exposure decision worth persisting.
|
|
1360
|
-
({ pid, controlPlane, authService } = await startMeshDetached({ server, space: m.space, open, userAuth, host: m.broker?.host, seed: manifestToChannels(eff), runtime, launch: specPath }));
|
|
1595
|
+
({ pid, controlPlane, authService } = await startMeshDetached({ transport: opts.transport, server, space: m.space, open, userAuth, rotateSys: opts.rotateSys, host: m.broker?.host, seed: manifestToChannels(eff), runtime, launch: specPath }));
|
|
1361
1596
|
}
|
|
1362
1597
|
catch (e) {
|
|
1363
1598
|
console.error(c.red(`✗ ${e.message}`));
|
|
1364
1599
|
process.exit(1);
|
|
1365
1600
|
}
|
|
1601
|
+
// Transport committed inside startMeshDetached before delivery (S5+S9).
|
|
1366
1602
|
console.log(c.green(`✓ mesh "${m.space}" up at ${server}`) + c.dim(` (broker pid ${pid})`));
|
|
1367
1603
|
console.log(c.dim(` seeded ${m.channels.length} channel(s): ${m.channels.map((ch) => "#" + ch.name).join(", ")}`));
|
|
1368
1604
|
// Never claim a launch the control plane can't deliver: the manager carries the launch spec, so a
|
|
@@ -1418,7 +1654,12 @@ function applyUpOverrides(prepared, o) {
|
|
|
1418
1654
|
* caller whose output claims a manager (the `up -f` launching line) can tell the truth. */
|
|
1419
1655
|
async function startDeliveryWithBroker(space, server, mgr) {
|
|
1420
1656
|
try {
|
|
1421
|
-
|
|
1657
|
+
// Prefer the in-memory decision for a fresh apply (policy may not be on disk yet — S9).
|
|
1658
|
+
// Fall back to the recorded policy for resume/restore/refresh paths that start no new listener.
|
|
1659
|
+
const tls = mgr?.transport?.kind === "tls-required" ||
|
|
1660
|
+
readBrokerPolicy(cotalRoot())?.transport.kind === "tls-required";
|
|
1661
|
+
const { transport: _t, ...mgrRest } = mgr ?? {};
|
|
1662
|
+
await ensureControlPlane({ space, server, tls, ...mgrRest });
|
|
1422
1663
|
return true;
|
|
1423
1664
|
}
|
|
1424
1665
|
catch (e) {
|
|
@@ -1436,9 +1677,17 @@ async function startDeliveryWithBroker(space, server, mgr) {
|
|
|
1436
1677
|
* log file and forwarded — the child writes to the file (not a pipe), so it survives the
|
|
1437
1678
|
* parent exiting.
|
|
1438
1679
|
*/
|
|
1439
|
-
export async function startMeshDetached(
|
|
1680
|
+
export async function startMeshDetached(
|
|
1681
|
+
// The `= {}` default is gone with the optional transport: an options object that can be omitted
|
|
1682
|
+
// entirely cannot carry a mandatory decision.
|
|
1683
|
+
opts) {
|
|
1440
1684
|
const server = opts.server ?? DEFAULT_SERVER;
|
|
1441
1685
|
const useAuth = !opts.open;
|
|
1686
|
+
// Belt on the one boot that both renders the broker config and can be reached by any caller: an
|
|
1687
|
+
// open boot skips `authSetup` entirely, so a `rotateSys` arriving here would be dropped in silence
|
|
1688
|
+
// rather than refused. Callers guard this too; this is the seam that cannot be bypassed.
|
|
1689
|
+
if (opts.rotateSys && !useAuth)
|
|
1690
|
+
throw new Error("startMeshDetached: --rotate-sys is for auth meshes; an open mesh has no system account or $SYS credentials to rotate");
|
|
1442
1691
|
const space = opts.space ?? resolveSpace(process.cwd());
|
|
1443
1692
|
ensureRootForSpace(useAuth, space); // may pin the cwd as this space's root — before any cotalPath use
|
|
1444
1693
|
refuseOpenOverUserState(Boolean(opts.open), space);
|
|
@@ -1447,10 +1696,17 @@ export async function startMeshDetached(opts = {}) {
|
|
|
1447
1696
|
await claimSpace(space, server, cotalRoot());
|
|
1448
1697
|
const seedFile = opts.seed ?? loadChannelsFile(opts.channels);
|
|
1449
1698
|
const host = opts.host ?? "127.0.0.1";
|
|
1450
|
-
|
|
1699
|
+
// The transport arrives decided. The dial-host SAN is re-checked here because THIS is where the
|
|
1700
|
+
// effective server is finally known — a manifest may name a different host than the flags did, and
|
|
1701
|
+
// a certificate that is valid for one is not thereby valid for the other.
|
|
1702
|
+
const transport = assertServesDialHost(opts.transport, new URL(server).hostname);
|
|
1703
|
+
const setup = useAuth ? await authSetup(storeDir, server, space, host, opts.userAuth, transport, opts.rotateSys) : undefined;
|
|
1451
1704
|
const port = Number(new URL(server).port) || 4222;
|
|
1705
|
+
// Same rule as the foreground path: every route to a listener names its transport (see
|
|
1706
|
+
// `writeOpenBrokerConf`). Detach must not be the mode where the fence quietly does not apply.
|
|
1707
|
+
const confPath = setup ? setup.confPath : writeOpenBrokerConf(storeDir, { port, host, transport });
|
|
1452
1708
|
const args = [
|
|
1453
|
-
|
|
1709
|
+
"-c", confPath,
|
|
1454
1710
|
...(opts.boundListener ? ["--name", opts.boundListener.serverName] : []),
|
|
1455
1711
|
];
|
|
1456
1712
|
const { bin, source } = await resolveNatsServer();
|
|
@@ -1489,8 +1745,37 @@ export async function startMeshDetached(opts = {}) {
|
|
|
1489
1745
|
writeFileSync(cotalPath("nats.pid"), String(child.pid));
|
|
1490
1746
|
if (opts.boundListener)
|
|
1491
1747
|
await opts.boundListener.verify();
|
|
1492
|
-
|
|
1493
|
-
|
|
1748
|
+
// POST-START MUST NOT LEAVE AN ORPHAN LISTENER.
|
|
1749
|
+
//
|
|
1750
|
+
// Everything above has already bound the port and written `nats.pid`, but NOTHING has recorded the
|
|
1751
|
+
// mesh yet — `recordOurMesh` is below. So a throw between here and there used to exit non-zero
|
|
1752
|
+
// while leaving a live broker holding the port with no registry entry, which `cotal down` cannot
|
|
1753
|
+
// reach because `down` works from the registry. A third state between "started" and "refused",
|
|
1754
|
+
// and the operator's only recourse is to hunt a pid.
|
|
1755
|
+
//
|
|
1756
|
+
// This is reachable BECAUSE of TLS and cannot happen on `main`: the post-start client verifies the
|
|
1757
|
+
// certificate, so a private CA without `NODE_EXTRA_CA_CERTS` fails here — after the listener is up.
|
|
1758
|
+
// The feature introduced the state, so the feature tears it down.
|
|
1759
|
+
//
|
|
1760
|
+
// Deliberately narrow: this is a teardown on the failure path, not a restructuring of the launch
|
|
1761
|
+
// sequence. The listener is stopped and the pid file removed, then the original error is rethrown
|
|
1762
|
+
// unchanged — the operator needs the certificate error, not a message about cleanup.
|
|
1763
|
+
if (!opts.skipPostStart) {
|
|
1764
|
+
try {
|
|
1765
|
+
await postStart(server, space, setup, seedFile);
|
|
1766
|
+
}
|
|
1767
|
+
catch (e) {
|
|
1768
|
+
try {
|
|
1769
|
+
child.kill("SIGTERM");
|
|
1770
|
+
}
|
|
1771
|
+
catch { /* already gone */ }
|
|
1772
|
+
try {
|
|
1773
|
+
rmSync(cotalPath("nats.pid"), { force: true });
|
|
1774
|
+
}
|
|
1775
|
+
catch { /* best effort */ }
|
|
1776
|
+
throw e;
|
|
1777
|
+
}
|
|
1778
|
+
}
|
|
1494
1779
|
// USER MODE: the auth service comes up FIRST among the daemons (see the foreground path).
|
|
1495
1780
|
const svc = await startUserAuthService(space, server, setup);
|
|
1496
1781
|
// Record BEFORE the control plane: the manager's fail-closed mode detection needs the
|
|
@@ -1503,11 +1788,17 @@ export async function startMeshDetached(opts = {}) {
|
|
|
1503
1788
|
recordOurMesh({
|
|
1504
1789
|
space, server, root: cotalRoot(),
|
|
1505
1790
|
mode: setup?.prepared ? "user" : useAuth ? "auth" : "open",
|
|
1791
|
+
// The detached listener is started from `transport` a few lines above, so this is the same
|
|
1792
|
+
// decision that shaped the config file — not a re-derivation.
|
|
1793
|
+
tlsRequired: transport.kind === "tls-required",
|
|
1506
1794
|
...(svc.userAuth ? { userAuth: svc.userAuth } : {}),
|
|
1507
1795
|
// Persist only a real decision — declared now, or carried forward — never the loopback default.
|
|
1508
1796
|
...(effectiveAttachHost ? { attachHost: effectiveAttachHost } : {}),
|
|
1509
1797
|
ts: new Date().toISOString(),
|
|
1510
1798
|
}, "started");
|
|
1799
|
+
// Commit policy BEFORE delivery launch (S9). Listener is proved; refuse paths never reach here.
|
|
1800
|
+
// startDeliveryWithBroker also receives `transport` so it does not depend on the file alone.
|
|
1801
|
+
commitTransportPolicy(cotalRoot(), transport);
|
|
1511
1802
|
// Bring up the delivery daemon WITH the detached broker (auth mode only; `cotal down` tears both down).
|
|
1512
1803
|
const controlPlane = await startDeliveryWithBroker(space, server, {
|
|
1513
1804
|
runtime: opts.runtime,
|
|
@@ -1516,6 +1807,8 @@ export async function startMeshDetached(opts = {}) {
|
|
|
1516
1807
|
attachHost: effectiveAttachHost,
|
|
1517
1808
|
resumeAttempt: opts.resumeAttempt,
|
|
1518
1809
|
resumeCommitToken: opts.resumeCommitToken,
|
|
1810
|
+
transport,
|
|
1811
|
+
wsPort: setup?.wsPort, // P2 item 6: the console session client's broker ws port
|
|
1519
1812
|
});
|
|
1520
1813
|
return {
|
|
1521
1814
|
server,
|
|
@@ -1539,28 +1832,42 @@ function refuseOpenOverUserState(open, space) {
|
|
|
1539
1832
|
const stateDir = userAuthStateDir(cotalRoot(), space);
|
|
1540
1833
|
throw new Error(`space "${space}" has user auth enabled (state under ${stateDir}) - \`--open\` would serve its streams without auth. Start it with \`cotal up --user-auth\`, or remove that directory deliberately to disable user auth (existing logins/grants die with it)`);
|
|
1541
1834
|
}
|
|
1542
|
-
/**
|
|
1543
|
-
*
|
|
1544
|
-
*
|
|
1545
|
-
*
|
|
1546
|
-
*
|
|
1547
|
-
*
|
|
1548
|
-
*
|
|
1549
|
-
*
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1835
|
+
/** Pin cwd as its own mesh root when the nearest ancestor root already belongs to a DIFFERENT
|
|
1836
|
+
* space — mode-independent.
|
|
1837
|
+
*
|
|
1838
|
+
* Auth path: a root's `.cotal/auth` is space-bound, so an explicit `--space` naming another space
|
|
1839
|
+
* cannot share that trust material. Open path: the same ancestor still owns policy, store, pid
|
|
1840
|
+
* and MeshEntry; reusing it for a second space collides those (S4 open arm — child `--open --tls-*`
|
|
1841
|
+
* wrote the parent's broker-policy and nats.pid while a second broker shared the JetStream store).
|
|
1842
|
+
*
|
|
1843
|
+
* A mismatch only exists when the resolved space differs from what the ancestor already hosts.
|
|
1844
|
+
* When the root was merely inherited (cwd has no `.cotal`), honor the new-space intent by making
|
|
1845
|
+
* cwd its own root. Only a folder that itself holds the other space's material refuses.
|
|
1846
|
+
* (When multi-space-per-root lands, that refusal becomes provision-the-new-space instead.) */
|
|
1847
|
+
function ensureRootForSpace(_useAuth, space) {
|
|
1553
1848
|
const root = cotalRoot();
|
|
1554
|
-
const existing = loadSoleSpaceAuth(authDir(root));
|
|
1555
|
-
if (!existing || existing.space === space)
|
|
1556
|
-
return;
|
|
1557
1849
|
const cwd = process.cwd();
|
|
1850
|
+
// What space does this root already host? Prefer auth material; fall back to a live MeshEntry
|
|
1851
|
+
// whose root is this directory (open meshes have no auth dir).
|
|
1852
|
+
const existingAuth = loadSoleSpaceAuth(authDir(root));
|
|
1853
|
+
const existingSpace = existingAuth?.space ??
|
|
1854
|
+
loadMeshes().find((m) => m.root === root || realpathSafe(m.root) === realpathSafe(root))?.space;
|
|
1855
|
+
if (!existingSpace || existingSpace === space)
|
|
1856
|
+
return;
|
|
1558
1857
|
if (root !== cwd) {
|
|
1559
1858
|
mkdirSync(join(cwd, ".cotal"), { recursive: true });
|
|
1560
|
-
console.log(c.dim(`nearest mesh root ${root} is space "${
|
|
1859
|
+
console.log(c.dim(`nearest mesh root ${root} is space "${existingSpace}" - making this folder its own root for "${space}"`));
|
|
1561
1860
|
return;
|
|
1562
1861
|
}
|
|
1563
|
-
throw new Error(`this folder is the root of space "${
|
|
1862
|
+
throw new Error(`this folder is the root of space "${existingSpace}" (${root}/.cotal), so it can't also run "${space}" - drop \`--space\` to run "${existingSpace}", or start "${space}" from a different folder (it becomes that mesh's own root)`);
|
|
1863
|
+
}
|
|
1864
|
+
function realpathSafe(p) {
|
|
1865
|
+
try {
|
|
1866
|
+
return realpathSync(p);
|
|
1867
|
+
}
|
|
1868
|
+
catch {
|
|
1869
|
+
return p;
|
|
1870
|
+
}
|
|
1564
1871
|
}
|
|
1565
1872
|
/** A space name maps to one mesh in the registry (the key `--space`/`use`/`down` act on). Before
|
|
1566
1873
|
* starting a broker, refuse to reuse a space already claimed by a DIFFERENT live mesh — a stale/dead
|
|
@@ -1608,7 +1915,13 @@ function recordOurMesh(m, provenance) {
|
|
|
1608
1915
|
const usableCurrent = cur && findMesh(cur) ? cur : undefined; // compute before recording m
|
|
1609
1916
|
const prior = findMesh(m.space);
|
|
1610
1917
|
const origin = provenance === "refresh" && prior?.origin === "manual" ? "manual" : "up";
|
|
1611
|
-
|
|
1918
|
+
// A REFRESH starts nothing: it concluded the mesh is up from reachability alone, and rebuilds `m`
|
|
1919
|
+
// from what THIS launch knows, which is never the operator's past decisions. `origin` was already
|
|
1920
|
+
// carried across for that reason; the overlay acceptance is the same class and was not, so a
|
|
1921
|
+
// no-op refresh silently erased a consent the operator had given. A `started` takeover may
|
|
1922
|
+
// replace it (that launch really is the mesh now); a refresh may not quietly drop it.
|
|
1923
|
+
const unencryptedOverlay = provenance === "refresh" && m.unencryptedOverlay === undefined ? prior?.unencryptedOverlay : m.unencryptedOverlay;
|
|
1924
|
+
recordMesh({ ...m, origin, ...(unencryptedOverlay !== undefined ? { unencryptedOverlay } : {}) });
|
|
1612
1925
|
if (!usableCurrent) {
|
|
1613
1926
|
setCurrent(m.space);
|
|
1614
1927
|
return;
|
|
@@ -1745,8 +2058,219 @@ function loadChannelsFile(explicit) {
|
|
|
1745
2058
|
* SPACE-SCOPED state dir (`.cotal/auth/<space>/` — the multi-space-ready layout; nothing user-auth
|
|
1746
2059
|
* lives flat) and preload its extra account(s) into the broker config. The inverse is fail-closed:
|
|
1747
2060
|
* a space whose user-auth state exists MUST keep being started with --user-auth — regenerating the
|
|
1748
|
-
* config without the callout account would silently break every sentinel connect.
|
|
1749
|
-
|
|
2061
|
+
* config without the callout account would silently break every sentinel connect.
|
|
2062
|
+
*
|
|
2063
|
+
* `rotateSys` (`cotal up --rotate-sys`) is the class-3 renewal for an EXISTING space: rotate the
|
|
2064
|
+
* system account, re-mint `membership-observer.creds` + `connection-evictor.creds` against the
|
|
2065
|
+
* successor, and render the config from the ROTATED record so the broker this `up` starts is the one
|
|
2066
|
+
* that trusts them. It belongs here because this is the single site that both owns the trust record
|
|
2067
|
+
* and renders `server.conf`; anywhere else would publish creds the live broker cannot honor. */
|
|
2068
|
+
/**
|
|
2069
|
+
* Turn the `--tls-cert`/`--tls-key` pair into a validated {@link BrokerTransport}, or `plaintext`
|
|
2070
|
+
* when neither was given.
|
|
2071
|
+
*
|
|
2072
|
+
* REFUSES BEFORE LAUNCH, never after. Everything it rejects would otherwise become a running broker
|
|
2073
|
+
* in some wrong state, and the two failure modes are not symmetrical: a broker that will not start
|
|
2074
|
+
* is an operator reading an error, while a broker that starts wrong is an operator believing they
|
|
2075
|
+
* have TLS. So the validation is deliberately ours and deliberately early.
|
|
2076
|
+
*
|
|
2077
|
+
* `validateTlsMaterial` does the substantive checks — readability, private-key mode, PEM pair
|
|
2078
|
+
* match, validity window, dial-host SAN — because nats-server's are not sufficient. Missing,
|
|
2079
|
+
* unreadable and mismatched pairs do stop it before it opens a listener, but an EXPIRED certificate
|
|
2080
|
+
* does not: it reports the config valid, starts, logs "Server is ready" and "TLS required", and
|
|
2081
|
+
* only the client then fails. An expired cert must never yield "mesh up".
|
|
2082
|
+
*
|
|
2083
|
+
* @param dialHost the hostname clients will verify against, which is NOT the bind host — a broker
|
|
2084
|
+
* may bind `0.0.0.0` while clients dial `broker.example`.
|
|
2085
|
+
*/
|
|
2086
|
+
function resolveTransport(values, dialHost, root,
|
|
2087
|
+
/** `persist` defaults false (commit-after-apply). `quiet` suppresses serving/inheriting lines
|
|
2088
|
+
* until the caller announces after a successful apply. */
|
|
2089
|
+
opts = {}) {
|
|
2090
|
+
const certFile = values["tls-cert"], keyFile = values["tls-key"];
|
|
2091
|
+
const quiet = opts.quiet === true;
|
|
2092
|
+
// NO FLAGS: inherit the RECORDED decision rather than defaulting to plaintext. This is the half
|
|
2093
|
+
// that makes a bare `cotal up` after a `cotal down` keep serving TLS. Without it the decision
|
|
2094
|
+
// lives only in the argv of whichever invocation first made it, and the most ordinary operator
|
|
2095
|
+
// gesture there is - stop the mesh, start it again - silently downgrades a broker that was
|
|
2096
|
+
// deliberately put on TLS. `readBrokerPolicy` refuses rather than degrading if the recorded
|
|
2097
|
+
// material has gone missing, so a moved cert fails loud here instead of coming up in cleartext.
|
|
2098
|
+
if (!certFile && !keyFile) {
|
|
2099
|
+
let recorded;
|
|
2100
|
+
try {
|
|
2101
|
+
recorded = readBrokerPolicy(root);
|
|
2102
|
+
}
|
|
2103
|
+
catch (e) {
|
|
2104
|
+
console.error(c.red(`✗ ${e instanceof Error ? e.message : String(e)}`));
|
|
2105
|
+
process.exit(1);
|
|
2106
|
+
}
|
|
2107
|
+
if (recorded?.transport.kind === "tls-required") {
|
|
2108
|
+
if (!quiet)
|
|
2109
|
+
console.log(c.dim("TLS: inheriting the recorded broker policy (no --tls-cert/--tls-key given)"));
|
|
2110
|
+
return validateRecorded(recorded.transport, dialHost, { quiet });
|
|
2111
|
+
}
|
|
2112
|
+
return { kind: "plaintext" };
|
|
2113
|
+
}
|
|
2114
|
+
// Half a pair is a mistake, never a configuration. Refusing names the missing half rather than
|
|
2115
|
+
// failing later inside nats-server with a message about a file the operator did not mention.
|
|
2116
|
+
if (!certFile || !keyFile) {
|
|
2117
|
+
console.error(c.red(`✗ --tls-cert and --tls-key must be given together (missing ${certFile ? "--tls-key" : "--tls-cert"})`));
|
|
2118
|
+
process.exit(1);
|
|
2119
|
+
}
|
|
2120
|
+
const transport = { kind: "tls-required", certFile: resolve(certFile), keyFile: resolve(keyFile) };
|
|
2121
|
+
const validated = validateRecorded(transport, dialHost, { quiet });
|
|
2122
|
+
// RECORD only when the caller has already applied the transport (started a matching listener).
|
|
2123
|
+
// Eager write here is S5: a refuse path cannot unwrite, and the next bare up inherits fiction.
|
|
2124
|
+
if (opts.persist === true)
|
|
2125
|
+
writeBrokerPolicy(root, { version: 1, transport: validated });
|
|
2126
|
+
return validated;
|
|
2127
|
+
}
|
|
2128
|
+
/** Persist + announce a transport that has been APPLIED (listener started and proved). */
|
|
2129
|
+
function commitTransportPolicy(root, transport) {
|
|
2130
|
+
if (transport.kind !== "tls-required")
|
|
2131
|
+
return;
|
|
2132
|
+
writeBrokerPolicy(root, { version: 1, transport });
|
|
2133
|
+
announceTransport(transport);
|
|
2134
|
+
}
|
|
2135
|
+
function announceTransport(transport) {
|
|
2136
|
+
if (transport.kind !== "tls-required")
|
|
2137
|
+
return;
|
|
2138
|
+
try {
|
|
2139
|
+
const material = validateTlsMaterial(transport, {});
|
|
2140
|
+
console.log(c.dim(`TLS: serving ${material.subject}, valid until ${material.notAfter.toISOString()}`));
|
|
2141
|
+
}
|
|
2142
|
+
catch {
|
|
2143
|
+
// Already validated at decide-time; announce is best-effort display.
|
|
2144
|
+
}
|
|
2145
|
+
}
|
|
2146
|
+
/** Read the live INFO greeting and report whether the listener requires TLS.
|
|
2147
|
+
* `undefined` means no usable INFO (timeout / closed) — caller treats as unknown. */
|
|
2148
|
+
async function liveListenerRequiresTls(server, timeoutMs = 2000) {
|
|
2149
|
+
let host;
|
|
2150
|
+
let port;
|
|
2151
|
+
try {
|
|
2152
|
+
const u = new URL(server);
|
|
2153
|
+
host = u.hostname || "127.0.0.1";
|
|
2154
|
+
port = Number(u.port) || 4222;
|
|
2155
|
+
}
|
|
2156
|
+
catch {
|
|
2157
|
+
return undefined;
|
|
2158
|
+
}
|
|
2159
|
+
return await new Promise((res) => {
|
|
2160
|
+
const sock = createConnection({ host, port });
|
|
2161
|
+
let buf = "";
|
|
2162
|
+
const done = (v) => {
|
|
2163
|
+
try {
|
|
2164
|
+
sock.destroy();
|
|
2165
|
+
}
|
|
2166
|
+
catch { /* */ }
|
|
2167
|
+
res(v);
|
|
2168
|
+
};
|
|
2169
|
+
sock.setTimeout(timeoutMs, () => done(undefined));
|
|
2170
|
+
sock.on("error", () => done(undefined));
|
|
2171
|
+
sock.on("close", () => done(undefined));
|
|
2172
|
+
sock.on("data", (d) => {
|
|
2173
|
+
buf += d.toString("utf8");
|
|
2174
|
+
const nl = buf.indexOf("\r\n");
|
|
2175
|
+
if (nl < 0)
|
|
2176
|
+
return;
|
|
2177
|
+
const line = buf.slice(0, nl);
|
|
2178
|
+
try {
|
|
2179
|
+
const info = JSON.parse(line.replace(/^INFO\s+/, ""));
|
|
2180
|
+
done(info.tls_required === true);
|
|
2181
|
+
}
|
|
2182
|
+
catch {
|
|
2183
|
+
done(undefined);
|
|
2184
|
+
}
|
|
2185
|
+
});
|
|
2186
|
+
});
|
|
2187
|
+
}
|
|
2188
|
+
/**
|
|
2189
|
+
* Re-assert that `transport` is valid for the host clients will actually dial, and exit loudly if
|
|
2190
|
+
* not. A no-op for plaintext.
|
|
2191
|
+
*
|
|
2192
|
+
* This exists because the transport is decided ONCE, early, before the route is known, so that no
|
|
2193
|
+
* route can be reached without one — but the effective dial host is decided LATE and differs per
|
|
2194
|
+
* route: a manifest may set `broker.host`, and a certificate valid for the flag-derived host is not
|
|
2195
|
+
* thereby valid for that one. Deciding early and checking late is the only ordering that satisfies
|
|
2196
|
+
* both, so the check is deliberately separate from the decision rather than folded into it.
|
|
2197
|
+
*
|
|
2198
|
+
* Quiet on success: `resolveTransport` already printed what is being served, and a second identical
|
|
2199
|
+
* line reads like a second broker.
|
|
2200
|
+
*/
|
|
2201
|
+
function assertServesDialHost(transport, dialHost) {
|
|
2202
|
+
if (transport.kind !== "tls-required" || dialHost === undefined)
|
|
2203
|
+
return transport;
|
|
2204
|
+
try {
|
|
2205
|
+
validateTlsMaterial(transport, { dialHost });
|
|
2206
|
+
}
|
|
2207
|
+
catch (e) {
|
|
2208
|
+
console.error(c.red(`✗ ${e instanceof Error ? e.message : String(e)}`));
|
|
2209
|
+
process.exit(1);
|
|
2210
|
+
}
|
|
2211
|
+
return transport;
|
|
2212
|
+
}
|
|
2213
|
+
/**
|
|
2214
|
+
* The client-side TLS requirement to persist for a listener THIS PROCESS DID NOT START — the
|
|
2215
|
+
* restore and ordinary-resume paths, which adopt a broker that is already listening.
|
|
2216
|
+
*
|
|
2217
|
+
* They have no `transport` in scope because they never decided one, so the honest source is the
|
|
2218
|
+
* recorded policy: the listener they are adopting was started from it. Reading it here is what stops
|
|
2219
|
+
* a resume from writing a registry entry that says "plaintext" over a broker serving TLS — clients
|
|
2220
|
+
* resolved from that entry would then connect without requiring it, which is the downgrade this
|
|
2221
|
+
* feature exists to prevent, arriving by way of a recovery path rather than a launch.
|
|
2222
|
+
*
|
|
2223
|
+
* Fails loud on an unusable policy for the same reason `resolveTransport` does: a resume that cannot
|
|
2224
|
+
* tell what it is adopting must not guess.
|
|
2225
|
+
*/
|
|
2226
|
+
function adoptedTlsRequired(root) {
|
|
2227
|
+
try {
|
|
2228
|
+
return readBrokerPolicy(root)?.transport.kind === "tls-required";
|
|
2229
|
+
}
|
|
2230
|
+
catch (e) {
|
|
2231
|
+
console.error(c.red(`✗ ${e instanceof Error ? e.message : String(e)}`));
|
|
2232
|
+
process.exit(1);
|
|
2233
|
+
}
|
|
2234
|
+
}
|
|
2235
|
+
/** Validate a TLS pair and exit loudly on anything wrong. Shared by the flag path and the
|
|
2236
|
+
* recorded-policy path so a cert that rots on disk is caught on the NEXT `up`, not only on the
|
|
2237
|
+
* one that first configured it. */
|
|
2238
|
+
function validateRecorded(transport, dialHost, opts = {}) {
|
|
2239
|
+
try {
|
|
2240
|
+
const material = validateTlsMaterial(transport, dialHost !== undefined ? { dialHost } : {});
|
|
2241
|
+
if (!opts.quiet)
|
|
2242
|
+
console.log(c.dim(`TLS: serving ${material.subject}, valid until ${material.notAfter.toISOString()}`));
|
|
2243
|
+
}
|
|
2244
|
+
catch (e) {
|
|
2245
|
+
// Surface the CERTIFICATE cause. A TLS failure reported as a generic startup error invites the
|
|
2246
|
+
// wrong remedy - operators go looking at ports and firewalls when the answer is the cert.
|
|
2247
|
+
console.error(c.red(`✗ ${e instanceof Error ? e.message : String(e)}`));
|
|
2248
|
+
process.exit(1);
|
|
2249
|
+
}
|
|
2250
|
+
return transport;
|
|
2251
|
+
}
|
|
2252
|
+
/**
|
|
2253
|
+
* Render and write the config for an OPEN (no-auth) broker, returning its path.
|
|
2254
|
+
*
|
|
2255
|
+
* The `authSetup` sibling for the no-auth mode. Open mode previously launched nats-server from
|
|
2256
|
+
* bare CLI flags and so never passed through any renderer — meaning the REQUIRED transport union
|
|
2257
|
+
* on the config renderers covered the auth path only, and a `--tls-cert`/`--tls-key` pair given to
|
|
2258
|
+
* an open-mode `up` would have been accepted while the broker came up in cleartext. Routing open
|
|
2259
|
+
* mode through a renderer is what makes the union total: there is now no route to a listener that
|
|
2260
|
+
* does not state its transport.
|
|
2261
|
+
*/
|
|
2262
|
+
function writeOpenBrokerConf(storeDir, opts) {
|
|
2263
|
+
const confPath = resolve(storeDir, "..", "server-open.conf");
|
|
2264
|
+
writeFileSync(confPath, openServerConfig({ port: opts.port, host: opts.host, storeDir, transport: opts.transport }));
|
|
2265
|
+
return confPath;
|
|
2266
|
+
}
|
|
2267
|
+
async function authSetup(storeDir, server, space, host = "127.0.0.1", user,
|
|
2268
|
+
// NON-OPTIONAL, and it used to default to `{ kind: "plaintext" }`. That default was the hole: the
|
|
2269
|
+
// manifest path called this with five arguments and silently got a cleartext listener while the
|
|
2270
|
+
// operator's `--tls-cert` sat validated and unused. A default here cannot be right, because the
|
|
2271
|
+
// safe value and the common value are different values — every caller knows its transport, and
|
|
2272
|
+
// the one that does not is the one that must not compile.
|
|
2273
|
+
transport, rotateSys = false) {
|
|
1750
2274
|
const dir = authDir(cotalRoot()); // the broker config (server.conf) still lands under the FS auth dir
|
|
1751
2275
|
const store = workspaceSecretStore(cotalRoot());
|
|
1752
2276
|
let auth = await getSpaceAuth(store, space);
|
|
@@ -1754,7 +2278,58 @@ async function authSetup(storeDir, server, space, host = "127.0.0.1", user) {
|
|
|
1754
2278
|
auth = await createSpaceAuth(space);
|
|
1755
2279
|
await putSpaceAuth(store, auth); // strips the $SYS seed at rest, but leaves the in-memory `auth` intact …
|
|
1756
2280
|
await provisionMembershipCreds(auth, cotalRoot()); // … so the observer can still be minted here (fresh-space only)
|
|
1757
|
-
|
|
2281
|
+
// A fresh space's $SYS material was just minted from the seed that only exists in this branch, so
|
|
2282
|
+
// the ASK is already satisfied, so say so rather than rotating a one-second-old account, and never
|
|
2283
|
+
// report a rotation that did not happen.
|
|
2284
|
+
if (rotateSys)
|
|
2285
|
+
console.log(c.dim("• --rotate-sys: this space is new, so its $SYS creds were just minted - nothing to rotate"));
|
|
2286
|
+
}
|
|
2287
|
+
else if (rotateSys) {
|
|
2288
|
+
// Third and last of the stopped-broker checks, at the one point every rotation path converges on.
|
|
2289
|
+
// The other two live in `up`'s reachable-listener branch: this root's recorded mesh at the
|
|
2290
|
+
// requested address, and any UNIDENTIFIED listener there (which refuses rather than free-porting
|
|
2291
|
+
// around a broker that may be serving this root's own server.conf and store). This one reads the
|
|
2292
|
+
// root's ownership records for a broker at an address nobody probed. It reports what those
|
|
2293
|
+
// records say, not what the process table says (see its own comment for the residual), and fails
|
|
2294
|
+
// CLOSED, so an ambiguous record refuses exactly like a live one.
|
|
2295
|
+
await assertRootBrokerStopped(cotalRoot());
|
|
2296
|
+
// Fails loud: a caller that swallowed this would boot the broker on the RETIRED system account
|
|
2297
|
+
// while `doctor auth` reported the rotation as done.
|
|
2298
|
+
const rot = await rotateSystemCreds(cotalRoot(), space);
|
|
2299
|
+
auth = rot.auth; // the config below MUST be rendered from the successor, never the pre-rotation copy
|
|
2300
|
+
console.log(c.green(`✓ rotated the system account for "${space}"`) +
|
|
2301
|
+
c.dim(` (generation ${rot.gen}) - re-minted ${SYSTEM_CREDS_FILES.join(" + ")}${rot.expiresAt ? `, valid to ${new Date(rot.expiresAt * 1000).toISOString().slice(0, 10)}` : ""}`));
|
|
2302
|
+
// Say exactly what is true. The retirement is CONFIG-LOAD-BOUND: an old cred dies against any
|
|
2303
|
+
// broker that loads the successor config, which is every broker started from this root from here
|
|
2304
|
+
// on, but a stale nats-server still holding the pre-rotation config in memory would keep
|
|
2305
|
+
// honoring it, so "now dead" without that qualifier oversells the guarantee.
|
|
2306
|
+
console.log(c.dim(" the data account, every agent cred and the JetStream store are untouched. The OLD $SYS creds are refused by any broker that loads this config; a stale broker still running the previous config would still honor them, so stop those first."));
|
|
2307
|
+
// A full backup binds to the trust chain it was taken against: `rootChainCommitment` hashes the
|
|
2308
|
+
// operator JWT and the system account, both of which just changed, so `cotal up --restore`
|
|
2309
|
+
// refuses every full artifact taken before this moment. That is correct (it is a different trust
|
|
2310
|
+
// root), but it is only obvious to someone who has read the fingerprint code, and rotation is
|
|
2311
|
+
// now a routine 30-day act rather than a once-per-space event. Say it at the moment it becomes
|
|
2312
|
+
// true, not in a doc the operator reads after the restore has already failed.
|
|
2313
|
+
console.log(c.dim(" NOTE: full backups taken before this rotation can no longer be restored (they are bound to the retired trust chain) - take a fresh `cotal backup` once the mesh is up."));
|
|
2314
|
+
}
|
|
2315
|
+
// The $SYS creds must be signed by the system account THIS boot is about to put in `server.conf`.
|
|
2316
|
+
// A rotation that committed the trust record and then died leaves them stale, unexpired, and
|
|
2317
|
+
// broker-dead and, crash-before-either-write, stale in a way that no comparison between the two
|
|
2318
|
+
// FILES can see (they agree with each other; they just disagree with the record). This is the one
|
|
2319
|
+
// place holding both, on the one path that renders the config, so it is where the split is caught.
|
|
2320
|
+
//
|
|
2321
|
+
// REFUSE, do not warn. A warning is what the `--detach` path turns into an unread log line under a
|
|
2322
|
+
// green "✓ running in the background", which is the false success this whole change exists to
|
|
2323
|
+
// remove. And what stays broken is not only the display feed: live connection EVICTION rides the
|
|
2324
|
+
// same pair, so booting here would silently downgrade revocation to deny-new for the life of the
|
|
2325
|
+
// mesh. The repo's posture is to throw rather than degrade, and the recovery is one command that
|
|
2326
|
+
// this message names.
|
|
2327
|
+
const stale = staleSystemCreds(cotalRoot(), auth.sys.pub);
|
|
2328
|
+
if (stale.length)
|
|
2329
|
+
throw new Error(`${stale.map((x) => `${x.file} (signed by ${x.iss ? `${x.iss.slice(0, 12)}…` : "an unreadable issuer"})`).join(", ")} ` +
|
|
2330
|
+
`${stale.length === 1 ? "is" : "are"} not signed by this space's system account (${auth.sys.pub.slice(0, 12)}…) - ` +
|
|
2331
|
+
"an interrupted rotation left the $SYS creds behind the trust record, so the broker would deny them and live eviction + the membership feed would stay down. " +
|
|
2332
|
+
"Re-run the rotation to land a complete generation: `cotal up --rotate-sys`");
|
|
1758
2333
|
const stateDir = userAuthStateDir(cotalRoot(), space); // the provider's space-scoped state dir
|
|
1759
2334
|
if (!user && hasUserAuthState(cotalRoot(), space)) {
|
|
1760
2335
|
throw new Error(`space "${space}" has user auth enabled (state under ${stateDir}) - start it with \`cotal up --user-auth\`, or remove that directory deliberately to disable user auth (existing logins/grants die with it)`);
|
|
@@ -1780,13 +2355,73 @@ async function authSetup(storeDir, server, space, host = "127.0.0.1", user) {
|
|
|
1780
2355
|
}
|
|
1781
2356
|
}
|
|
1782
2357
|
const port = Number(new URL(server).port) || 4222;
|
|
2358
|
+
// P2 item 6: allocate the broker's loopback WebSocket listener port — the console page becomes a
|
|
2359
|
+
// mesh §13.6 session client over it (a NEW same-host attack surface, localhost-bound, no TLS). The
|
|
2360
|
+
// manager's establisher builds its wsUrl from this; threaded to `supervise --ws-port`.
|
|
2361
|
+
const wsPort = await freePort(host);
|
|
1783
2362
|
const confPath = resolve(dir, "server.conf");
|
|
1784
|
-
writeFileSync(confPath, serverConfig(auth, [auth], { port, storeDir, host, ...(prepared ? { extraAccounts: prepared.extraAccounts } : {}) }));
|
|
2363
|
+
writeFileSync(confPath, serverConfig(auth, [auth], { transport, port, storeDir, host, wsPort, wsHost: host, ...(prepared ? { extraAccounts: prepared.extraAccounts } : {}) }));
|
|
1785
2364
|
// Ephemeral setup cred: used only to probe reachability, pre-create the space streams/buckets
|
|
1786
2365
|
// (setupSpaceStreams) and seed the channel registry (seedChannelRegistry) — all within the
|
|
1787
2366
|
// enumerated `provisioner` scope. No broad `manager` residual for the up path.
|
|
1788
2367
|
const creds = await mintCreds(auth, newIdentity(), "provisioner");
|
|
1789
|
-
return { confPath, creds, ...(prepared ? { prepared, stateDir } : {}) };
|
|
2368
|
+
return { confPath, creds, wsPort, ...(prepared ? { prepared, stateDir } : {}) };
|
|
2369
|
+
}
|
|
2370
|
+
/**
|
|
2371
|
+
* Refuse the rotation if this root's own bookkeeping still shows a broker running. This is a
|
|
2372
|
+
* BEST-EFFORT check over Cotal-managed ownership records, NOT a proof that no process is serving this
|
|
2373
|
+
* root, and the difference matters: a survivor keeps honoring the retired account from memory, and a
|
|
2374
|
+
* second broker on the successor opens that survivor's JetStream store underneath it.
|
|
2375
|
+
*
|
|
2376
|
+
* Two records, because either alone has a blind spot. The PID FILE catches a broker this root started
|
|
2377
|
+
* whose registry row was lost. The REGISTRY sweep catches one recorded for this root at some OTHER
|
|
2378
|
+
* address, which no probe of the requested URL would reach. Both fail CLOSED: an unreadable or
|
|
2379
|
+
* malformed pid file refuses exactly like a live one, since "cannot tell" and "still running" have
|
|
2380
|
+
* the same consequence.
|
|
2381
|
+
*
|
|
2382
|
+
* WHAT IT CANNOT SEE, stated because an earlier version of this comment claimed otherwise: both
|
|
2383
|
+
* records are mutable, and neither is written by a broker started outside `cotal up`. Delete both
|
|
2384
|
+
* while the process lives, or run `nats-server -c <root>/.cotal/auth/server.conf` by hand, and this
|
|
2385
|
+
* returns success having probed nothing. The requested address is covered separately (an unidentified
|
|
2386
|
+
* listener there refuses the rotation rather than moving to a free port), which leaves a hand-started
|
|
2387
|
+
* broker on a DIFFERENT port as the honest residual. Closing that needs something a survivor holds
|
|
2388
|
+
* and cannot delete, an exclusive store lock, which does not exist today; until it does, do not run
|
|
2389
|
+
* `nats-server` against this root's config outside `cotal up`.
|
|
2390
|
+
*
|
|
2391
|
+
* Not a general `up` guard: an ordinary boot adopting or replacing a listener is a supported flow with
|
|
2392
|
+
* its own claim machinery. This is specifically the precondition for retiring an authority.
|
|
2393
|
+
*/
|
|
2394
|
+
async function assertRootBrokerStopped(root) {
|
|
2395
|
+
const pidPath = join(root, ".cotal", "nats.pid");
|
|
2396
|
+
if (existsSync(pidPath)) {
|
|
2397
|
+
let raw;
|
|
2398
|
+
try {
|
|
2399
|
+
raw = readFileSync(pidPath, "utf8").trim();
|
|
2400
|
+
}
|
|
2401
|
+
catch (e) {
|
|
2402
|
+
throw new Error(`--rotate-sys: ${pidPath} exists but cannot be read (${e.message}) - refusing to rotate while a broker for this root may still be running; \`cotal down\` first`);
|
|
2403
|
+
}
|
|
2404
|
+
const pid = Number(raw);
|
|
2405
|
+
if (!Number.isInteger(pid) || pid <= 0)
|
|
2406
|
+
throw new Error(`--rotate-sys: ${pidPath} does not hold a pid (${JSON.stringify(raw)}) - refusing to rotate while a broker for this root may still be running; \`cotal down\` first`);
|
|
2407
|
+
let live;
|
|
2408
|
+
try {
|
|
2409
|
+
process.kill(pid, 0); // signal 0 is a liveness probe, it signals nothing
|
|
2410
|
+
live = true;
|
|
2411
|
+
}
|
|
2412
|
+
catch (e) {
|
|
2413
|
+
// EPERM means the process EXISTS and is not ours to signal, i.e. alive for this purpose.
|
|
2414
|
+
live = e.code === "EPERM";
|
|
2415
|
+
}
|
|
2416
|
+
if (live)
|
|
2417
|
+
throw new Error(`--rotate-sys: this root's broker is still running (pid ${pid}) - it would keep serving the retired system account, and a second broker would share its JetStream store. Stop it first: \`cotal down\``);
|
|
2418
|
+
}
|
|
2419
|
+
// A broker recorded for this root at ANY address, still answering. The rotate refusal earlier in
|
|
2420
|
+
// `up` only sees the URL this invocation asked for; this sees the ones it did not.
|
|
2421
|
+
for (const m of meshesForRoot(root)) {
|
|
2422
|
+
if (await isReachable(m.server))
|
|
2423
|
+
throw new Error(`--rotate-sys: mesh "${m.space}" for this root is still reachable at ${m.server} - stop it first (\`cotal down\`), then rotate`);
|
|
2424
|
+
}
|
|
1790
2425
|
}
|
|
1791
2426
|
/** Mint the two scoped creds the delivery daemon's membership feed loads (broker-sourced graph
|
|
1792
2427
|
* membership), at the FRESH `cotal up` while the in-memory `$SYS` signing seed still exists:
|
|
@@ -1815,9 +2450,9 @@ async function provisionMembershipCreds(auth, root) {
|
|
|
1815
2450
|
// deny-new-only (durable reauth) — surfaced loudly by the removal path, never silent.
|
|
1816
2451
|
const evictor = await mintConnectionEvictorCreds(auth, newIdentity());
|
|
1817
2452
|
mkSecretDir(cotalPath()); // harden .cotal/ before the creds land (born under a private ACL, no race)
|
|
1818
|
-
writeSecretFile(cotalPath(
|
|
2453
|
+
writeSecretFile(cotalPath(SYSTEM_CREDS_FILES[0]), observer);
|
|
1819
2454
|
await workspaceSecretStore(root).put(MEMBERSHIP_RW_CREDS_KEY, rw); // migrated kind: through the seam (0600 FS put)
|
|
1820
|
-
writeSecretFile(cotalPath(
|
|
2455
|
+
writeSecretFile(cotalPath(SYSTEM_CREDS_FILES[1]), evictor);
|
|
1821
2456
|
writeSecretFile(cotalPath("membership.json"), JSON.stringify({ accountId: auth.account.pub }));
|
|
1822
2457
|
}
|
|
1823
2458
|
catch (e) {
|