@cotal-ai/cli 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/command.d.ts.map +1 -1
- package/dist/command.js +13 -0
- package/dist/command.js.map +1 -1
- package/dist/commands/agents.d.ts +5 -15
- package/dist/commands/agents.d.ts.map +1 -1
- package/dist/commands/agents.js +133 -72
- package/dist/commands/agents.js.map +1 -1
- package/dist/commands/clean.d.ts.map +1 -1
- package/dist/commands/clean.js +4 -3
- package/dist/commands/clean.js.map +1 -1
- package/dist/commands/describe.d.ts +72 -0
- package/dist/commands/describe.d.ts.map +1 -0
- package/dist/commands/describe.js +148 -0
- package/dist/commands/describe.js.map +1 -0
- package/dist/commands/doctor.d.ts.map +1 -1
- package/dist/commands/doctor.js +27 -8
- package/dist/commands/doctor.js.map +1 -1
- package/dist/commands/down-manifest.d.ts.map +1 -1
- package/dist/commands/down-manifest.js +38 -18
- package/dist/commands/down-manifest.js.map +1 -1
- package/dist/commands/down.d.ts +9 -0
- package/dist/commands/down.d.ts.map +1 -1
- package/dist/commands/down.js +38 -10
- package/dist/commands/down.js.map +1 -1
- package/dist/commands/ext.d.ts.map +1 -1
- package/dist/commands/ext.js +16 -10
- package/dist/commands/ext.js.map +1 -1
- package/dist/commands/join.d.ts.map +1 -1
- package/dist/commands/join.js +2 -4
- package/dist/commands/join.js.map +1 -1
- package/dist/commands/meshes-add.d.ts.map +1 -1
- package/dist/commands/meshes-add.js +23 -0
- package/dist/commands/meshes-add.js.map +1 -1
- package/dist/commands/models.d.ts.map +1 -1
- package/dist/commands/models.js +2 -2
- package/dist/commands/models.js.map +1 -1
- package/dist/commands/spawn-manifest.d.ts.map +1 -1
- package/dist/commands/spawn-manifest.js +22 -12
- package/dist/commands/spawn-manifest.js.map +1 -1
- package/dist/commands/spawn.d.ts +5 -0
- package/dist/commands/spawn.d.ts.map +1 -1
- package/dist/commands/spawn.js +4 -2
- package/dist/commands/spawn.js.map +1 -1
- package/dist/commands/status.js +25 -5
- package/dist/commands/status.js.map +1 -1
- package/dist/commands/up.d.ts +17 -2
- package/dist/commands/up.d.ts.map +1 -1
- package/dist/commands/up.js +640 -49
- package/dist/commands/up.js.map +1 -1
- package/dist/console/app.d.ts +1 -1
- package/dist/console/app.d.ts.map +1 -1
- package/dist/console/app.js +16 -5
- package/dist/console/app.js.map +1 -1
- package/dist/console/commands.d.ts.map +1 -1
- package/dist/console/commands.js +5 -5
- package/dist/console/commands.js.map +1 -1
- package/dist/console/root.d.ts.map +1 -1
- package/dist/console/root.js +6 -1
- package/dist/console/root.js.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +23 -0
- package/dist/index.js.map +1 -1
- package/dist/lib/attach-client.d.ts +37 -4
- package/dist/lib/attach-client.d.ts.map +1 -1
- package/dist/lib/attach-client.js +88 -18
- package/dist/lib/attach-client.js.map +1 -1
- package/dist/lib/auth-proc.d.ts +3 -1
- package/dist/lib/auth-proc.d.ts.map +1 -1
- package/dist/lib/auth-proc.js +15 -4
- package/dist/lib/auth-proc.js.map +1 -1
- package/dist/lib/connect.d.ts +1 -1
- package/dist/lib/connect.d.ts.map +1 -1
- package/dist/lib/connect.js +1 -1
- package/dist/lib/connect.js.map +1 -1
- package/dist/lib/control.d.ts +42 -11
- package/dist/lib/control.d.ts.map +1 -1
- package/dist/lib/control.js +251 -44
- package/dist/lib/control.js.map +1 -1
- package/dist/lib/delivery-proc.d.ts +19 -4
- package/dist/lib/delivery-proc.d.ts.map +1 -1
- package/dist/lib/delivery-proc.js +129 -43
- package/dist/lib/delivery-proc.js.map +1 -1
- package/dist/lib/manager-proc.d.ts +48 -5
- package/dist/lib/manager-proc.d.ts.map +1 -1
- package/dist/lib/manager-proc.js +131 -30
- package/dist/lib/manager-proc.js.map +1 -1
- package/dist/lib/manifest/ledger.d.ts +52 -8
- package/dist/lib/manifest/ledger.d.ts.map +1 -1
- package/dist/lib/manifest/ledger.js +77 -19
- package/dist/lib/manifest/ledger.js.map +1 -1
- package/dist/lib/manifest/live.d.ts +21 -16
- package/dist/lib/manifest/live.d.ts.map +1 -1
- package/dist/lib/manifest/live.js +33 -20
- package/dist/lib/manifest/live.js.map +1 -1
- package/dist/lib/restore.d.ts.map +1 -1
- package/dist/lib/restore.js +21 -3
- package/dist/lib/restore.js.map +1 -1
- package/dist/view/mesh-view.js +2 -2
- package/dist/view/mesh-view.js.map +1 -1
- package/package.json +4 -3
- package/dist/lib/pid.d.ts +0 -23
- package/dist/lib/pid.d.ts.map +0 -1
- package/dist/lib/pid.js +0 -39
- package/dist/lib/pid.js.map +0 -1
package/dist/commands/up.js
CHANGED
|
@@ -2,11 +2,11 @@ import { spawn } from "node:child_process";
|
|
|
2
2
|
import { randomUUID } from "node:crypto";
|
|
3
3
|
import { createConnection, createServer } from "node:net";
|
|
4
4
|
import { hostname } from "node:os";
|
|
5
|
-
import { mkdirSync, writeFileSync, readFileSync, existsSync, openSync, statSync, readSync, closeSync, lstatSync, rmSync, } from "node:fs";
|
|
5
|
+
import { mkdirSync, writeFileSync, readFileSync, existsSync, openSync, statSync, readSync, closeSync, lstatSync, rmSync, realpathSync, } from "node:fs";
|
|
6
6
|
import { join, resolve } from "node:path";
|
|
7
|
-
import { isReachable, DEFAULT_SERVER,
|
|
7
|
+
import { isReachable, DEFAULT_SERVER, createSpaceAuth, serverConfig, openServerConfig, validateTlsMaterial, mintCreds, mintLifecycleUid, DEV_OWNER, mintConnectionEvictorCreds, mintMembershipObserverCreds, newIdentity, setupSpaceStreams, standaloneConnectOpts, seedChannelRegistry, ensureDefaultDeliveryClass, mkSecretDir, writeSecretFile, } from "@cotal-ai/core";
|
|
8
8
|
import { connect } from "@nats-io/transport-node";
|
|
9
|
-
import { assertSingleSpaceBroker, assertUserAuthInfo, authDir, getSoleSpaceAuth, getSpaceAuth, hasUserAuthState, loadSoleSpaceAuth, putSpaceAuth, clearCurrent, findMesh, getCurrent, loadMeshes, MEMBERSHIP_RW_CREDS_KEY, recordMesh, removeMesh, setCurrent, userAuthStateDir, workspaceSecretStore, acquireMaintenanceLock, assertStoreIdentity, assessRestoreClaim, beginOrdinaryResume, bindOrdinaryResumeListener, consumeRetiredMaintenance, markOrdinaryResumeActive, markOrdinaryResumeDegraded, replaceDeadOrdinaryResumeListener, localProcessOwnerStatus, readMaintenanceJournal, readMaintenanceResumeDocument, readStoreIdentity, recordOrdinaryResumeManagerCommit, releaseMaintenanceLock, retireOrdinaryResume, sameStoreIdentity, } from "@cotal-ai/workspace";
|
|
9
|
+
import { assertSingleSpaceBroker, assertUserAuthInfo, authDir, getSoleSpaceAuth, getSpaceAuth, hasUserAuthState, loadSoleSpaceAuth, putSpaceAuth, clearCurrent, findMesh, getCurrent, loadMeshes, MEMBERSHIP_RW_CREDS_KEY, recordMesh, meshesForRoot, removeMesh, rotateSystemCreds, setCurrent, staleSystemCreds, SYSTEM_CREDS_FILES, userAuthStateDir, workspaceSecretStore, acquireMaintenanceLock, assertStoreIdentity, assessRestoreClaim, beginOrdinaryResume, bindOrdinaryResumeListener, consumeRetiredMaintenance, markOrdinaryResumeActive, markOrdinaryResumeDegraded, replaceDeadOrdinaryResumeListener, localProcessOwnerStatus, readMaintenanceJournal, readMaintenanceResumeDocument, readStoreIdentity, recordOrdinaryResumeManagerCommit, releaseMaintenanceLock, retireOrdinaryResume, sameStoreIdentity, readBrokerPolicy, writeBrokerPolicy, } from "@cotal-ai/workspace";
|
|
10
10
|
import { ensureAuthService, resolveAuthProvider, stopAuthService } from "../lib/auth-proc.js";
|
|
11
11
|
import { resolveSpace } from "../lib/status.js";
|
|
12
12
|
import { c } from "../ui.js";
|
|
@@ -32,12 +32,15 @@ export const upFlags = [
|
|
|
32
32
|
{ name: "space", type: "string", value: "<s>", description: "space name (default: the folder's)" },
|
|
33
33
|
{ name: "store-dir", type: "string", value: "<dir>", description: "JetStream store directory" },
|
|
34
34
|
{ name: "channels", type: "string", value: "<path>", description: "channel-registry seed file (JSON; default .cotal/channels.json)" },
|
|
35
|
+
{ name: "tls-cert", type: "string", value: "<file>", description: "serve the broker over TLS with this certificate (requires --tls-key)" },
|
|
36
|
+
{ name: "tls-key", type: "string", value: "<file>", description: "the private key for --tls-cert" },
|
|
35
37
|
{ name: "restore", type: "string", value: "<dir>", description: "restore an offline backup before exposing the normal listener" },
|
|
36
38
|
{ name: "restore-only", type: "string", value: "<registry>", description: "restore only the registry component" },
|
|
37
39
|
{ name: "accept-missing-source", type: "boolean", description: "explicit disaster consent when the inode-bound preserved source is absent" },
|
|
38
40
|
{ name: "open", type: "boolean", description: "unauthenticated dev mesh (no JWT/ACLs)" },
|
|
39
41
|
{ name: "user-auth", type: "boolean", description: "per-USER auth: login + bearer through the space's auth service" },
|
|
40
42
|
{ name: "idp", type: "string", value: "<url>", description: "with --user-auth: the IdP auth base URL to pin (first enable)" },
|
|
43
|
+
{ name: "rotate-sys", type: "boolean", description: "renew the expired/expiring $SYS creds by rotating the system account (agents, data and creds survive; needs a stopped mesh)" },
|
|
41
44
|
{ name: "detach", type: "boolean", description: "run in the background (stop with `cotal down`)" },
|
|
42
45
|
{ name: "runtime", type: "string", value: "<name>", description: "agent runtime for the mesh manager (default pty; extension runtimes are explicit-only, see `cotal runtimes`); with -f overrides the manifest's runtime" },
|
|
43
46
|
{ name: "file", type: "string", short: "f", value: "<cotal.yaml>", description: "launch a whole mesh from a manifest" },
|
|
@@ -63,6 +66,13 @@ export async function up(args) {
|
|
|
63
66
|
if (values.restore) {
|
|
64
67
|
if (values.file || values.channels)
|
|
65
68
|
throw new Error("--restore cannot be combined with --file/-f or --channels");
|
|
69
|
+
// A restore REINSTATES a trust root from an artifact; a rotation SUPERSEDES the one on disk.
|
|
70
|
+
// Together they would restore a system account and retire it in the same command, leaving the
|
|
71
|
+
// operator unable to say which authority the mesh actually came up on, and the artifact's own
|
|
72
|
+
// $SYS creds overwritten by the rotation before anyone verified the restore. Restore, verify,
|
|
73
|
+
// then rotate as its own deliberate step.
|
|
74
|
+
if (values["rotate-sys"])
|
|
75
|
+
throw new Error("--restore cannot be combined with --rotate-sys - restore the space and verify it first, then rotate: `cotal down` then `cotal up --rotate-sys`");
|
|
66
76
|
// A restore rewrites the shared store and the trust root under it, from an artifact that names
|
|
67
77
|
// one space (see `cotal backup`) - it cannot leave the root's other tenants standing.
|
|
68
78
|
assertSingleSpaceBroker(authDir(cotalRoot()), "cotal up --restore");
|
|
@@ -93,8 +103,22 @@ export async function up(args) {
|
|
|
93
103
|
}
|
|
94
104
|
if (values["restore-only"] || values["accept-missing-source"])
|
|
95
105
|
throw new Error("--restore-only and --accept-missing-source require --restore <dir>");
|
|
106
|
+
// MAINTENANCE RE-ENTRY. `--restore` is refused with `--rotate-sys` above, but that guard only sees
|
|
107
|
+
// the EXPLICIT flag: a restore/resume re-entry arrives with `restore` cleared and an `__*Attempt`
|
|
108
|
+
// set, and an auto-recovered journal reaches the same place with no restore flag ever typed. Both
|
|
109
|
+
// re-entries then hit adopt-the-live-listener paths that RETURN before `authSetup`, so the flag
|
|
110
|
+
// would be accepted, nothing would rotate, and the command would exit 0. That is the silent
|
|
111
|
+
// success this whole change exists to remove, so it is refused here, before any attempt state is
|
|
112
|
+
// read. A rotation is a stopped, fresh boot; a half-finished maintenance attempt is neither.
|
|
113
|
+
if (values["rotate-sys"] && (values.__restoreAttempt || values.__ordinaryResumeAttempt))
|
|
114
|
+
throw new Error("--rotate-sys cannot run during a restore/resume re-entry - finish or roll back the maintenance attempt, then `cotal down` and `cotal up --rotate-sys`");
|
|
96
115
|
if (!values.__restoreAttempt && !values.__ordinaryResumeAttempt && !values.file) {
|
|
97
116
|
const root = cotalRoot();
|
|
117
|
+
// Same refusal for the AUTO-recovered journal, raised before the recovery is prepared and
|
|
118
|
+
// re-entered, so the operator reads one clear error instead of one thrown from inside a nested
|
|
119
|
+
// `up` that has already begun adopting a maintenance attempt.
|
|
120
|
+
if (values["rotate-sys"] && readMaintenanceJournal(root))
|
|
121
|
+
throw new Error("--rotate-sys is refused while this root has a maintenance attempt to recover - finish or roll back that attempt (`cotal up` alone recovers it), then `cotal down` and `cotal up --rotate-sys`");
|
|
98
122
|
const lock = acquireMaintenanceLock(root);
|
|
99
123
|
let pending;
|
|
100
124
|
let recoveredRestore;
|
|
@@ -304,6 +328,77 @@ export async function up(args) {
|
|
|
304
328
|
if (values.idp && !wantUser && !values.file) {
|
|
305
329
|
throw new Error('--idp is for user-auth spaces; pair it with --user-auth, or set broker.auth: "user" in a manifest');
|
|
306
330
|
}
|
|
331
|
+
// An open mesh has no operator, no system account, and no $SYS creds, so there is nothing to
|
|
332
|
+
// rotate; the request is a misunderstanding to name, never a silent no-op that reports success.
|
|
333
|
+
if (values["rotate-sys"] && values.open) {
|
|
334
|
+
throw new Error("--rotate-sys is for auth meshes: an open mesh (--open) has no system account or $SYS credentials to rotate");
|
|
335
|
+
}
|
|
336
|
+
// THE EFFECTIVE PROJECT ROOT IS PINNED BEFORE THE TRANSPORT IS DECIDED.
|
|
337
|
+
//
|
|
338
|
+
// `ensureRootForSpace` may create `cwd/.cotal` when the nearest ancestor root already owns a
|
|
339
|
+
// different auth space — the child becomes its own mesh root. That pin used to run AFTER
|
|
340
|
+
// `resolveTransport`, so a TLS policy was written to the PARENT root while the listener and
|
|
341
|
+
// MeshEntry landed on the CHILD. First `up --tls-*` looked green (transport was in memory);
|
|
342
|
+
// `down` then bare `up` from the child found no policy and served plaintext, rewriting
|
|
343
|
+
// `tlsRequired: false`. Same class as B4/B5: operator-visible TLS success, silent cleartext on
|
|
344
|
+
// the documented retain path. Also polluted the parent's policy file.
|
|
345
|
+
//
|
|
346
|
+
// Dry-run must not pin (mkdir is a mutation). Validation still uses whatever root is visible;
|
|
347
|
+
// persistence is already suppressed below.
|
|
348
|
+
if (!values["dry-run"]) {
|
|
349
|
+
const spaceForRoot = values.space ?? resolveSpace(process.cwd());
|
|
350
|
+
if (values.file) {
|
|
351
|
+
try {
|
|
352
|
+
const prepared = loadManifest(resolve(values.file));
|
|
353
|
+
const space = values.space ?? prepared.manifest.space;
|
|
354
|
+
const open = Boolean(values.open) || prepared.manifest.broker?.auth === false;
|
|
355
|
+
ensureRootForSpace(!open, space);
|
|
356
|
+
}
|
|
357
|
+
catch {
|
|
358
|
+
// Manifest errors are reported by the -f path with the same load; do not double-print.
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
else {
|
|
362
|
+
ensureRootForSpace(!values.open, spaceForRoot);
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
// THE TRANSPORT IS DECIDED HERE, ABOVE EVERY BRANCH, AND THIS POSITION IS THE FIX.
|
|
366
|
+
//
|
|
367
|
+
// It used to be resolved inside the foreground path only, which meant three routes to a listener
|
|
368
|
+
// never saw it: `up -f` (manifest), `up --detach`, and the already-running refresh. Each accepted
|
|
369
|
+
// `--tls-cert`/`--tls-key`, validated nothing, and served PLAINTEXT while printing `✓ mesh up`.
|
|
370
|
+
// Every one of those was a separate exit from the same room, and closing them one at a time is
|
|
371
|
+
// what produced three rounds of the same defect.
|
|
372
|
+
//
|
|
373
|
+
// A guard only fences what comes AFTER it, so the decision has to dominate the branch rather than
|
|
374
|
+
// sit in one arm of it. The dial-host SAN is deliberately NOT checked here — the effective host is
|
|
375
|
+
// not known until a route picks it — so each route re-asserts it via `assertServesDialHost` once
|
|
376
|
+
// it does. Decide early so nothing can skip it; check late so it checks the right host.
|
|
377
|
+
// `persist` is false on a dry run, and that is a REGRESSION THIS HOIST CAUSED. Before the move,
|
|
378
|
+
// `resolveTransport` ran inside the launch path and a `--dry-run` never reached it. Hoisting it
|
|
379
|
+
// above the branch is what makes the transport dominate every route — and it also put a WRITE in
|
|
380
|
+
// front of a command whose entire contract is "mutate nothing", so `up -f --dry-run --tls-cert`
|
|
381
|
+
// printed "Dry run - nothing was changed" and left a broker-policy.json behind.
|
|
382
|
+
//
|
|
383
|
+
// Validation still runs: a dry run should absolutely refuse an expired or unreadable cert, and
|
|
384
|
+
// reporting that is the point of it. Only the persistence is suppressed. An instrument that
|
|
385
|
+
// modifies what it inspects is a defect even when everything it reports is true.
|
|
386
|
+
//
|
|
387
|
+
// The root argument is the PINNED root (or the walked root on dry-run). One root for policy,
|
|
388
|
+
// auth, store, and MeshEntry — never a second walk that can disagree after a later pin.
|
|
389
|
+
//
|
|
390
|
+
// PERSIST IS ALWAYS FALSE HERE (commit-after-apply). resolveTransport used to write the policy
|
|
391
|
+
// before the already-running refresh branch could refuse a transport change, so a failed
|
|
392
|
+
// `up --tls-*` against a live plaintext mesh left tls-required on disk; the next bare `up`
|
|
393
|
+
// printed "TLS: inheriting" / "TLS: serving" / green already-running over a cleartext listener
|
|
394
|
+
// (S5). Decide and validate early so every branch sees the transport; write only after a
|
|
395
|
+
// matching listener is started and proved (`commitTransportPolicy`), never on a refuse path.
|
|
396
|
+
// `quiet` suppresses the serving/inheriting lines until that commit (or a dry-run announce).
|
|
397
|
+
const meshRoot = cotalRoot();
|
|
398
|
+
const transport = resolveTransport(values, undefined, meshRoot, {
|
|
399
|
+
persist: false,
|
|
400
|
+
quiet: true,
|
|
401
|
+
});
|
|
307
402
|
// `up -f cotal.yaml` is a distinct path: bring up a FRESH mesh described by a manifest (broker +
|
|
308
403
|
// channels + booted agents). It owns the whole space; deploying onto a RUNNING mesh is `spawn -f`.
|
|
309
404
|
// CLI flags override the manifest (flag > manifest > default) so the same file runs at a different
|
|
@@ -311,6 +406,7 @@ export async function up(args) {
|
|
|
311
406
|
if (values.file) {
|
|
312
407
|
if (values["dry-run"]) {
|
|
313
408
|
await upManifest(values.file, {
|
|
409
|
+
transport,
|
|
314
410
|
dryRun: true,
|
|
315
411
|
server: values.server,
|
|
316
412
|
host: values.host,
|
|
@@ -319,6 +415,7 @@ export async function up(args) {
|
|
|
319
415
|
open: values.open,
|
|
320
416
|
userAuth: wantUser,
|
|
321
417
|
idp: values.idp,
|
|
418
|
+
rotateSys: values["rotate-sys"],
|
|
322
419
|
});
|
|
323
420
|
return;
|
|
324
421
|
}
|
|
@@ -326,6 +423,7 @@ export async function up(args) {
|
|
|
326
423
|
try {
|
|
327
424
|
assertOrdinaryUpAllowed(cotalRoot());
|
|
328
425
|
await upManifest(values.file, {
|
|
426
|
+
transport,
|
|
329
427
|
dryRun: Boolean(values["dry-run"]),
|
|
330
428
|
server: values.server,
|
|
331
429
|
host: values.host,
|
|
@@ -334,6 +432,7 @@ export async function up(args) {
|
|
|
334
432
|
open: values.open,
|
|
335
433
|
userAuth: wantUser,
|
|
336
434
|
idp: values.idp,
|
|
435
|
+
rotateSys: values["rotate-sys"],
|
|
337
436
|
});
|
|
338
437
|
}
|
|
339
438
|
finally {
|
|
@@ -417,6 +516,40 @@ export async function up(args) {
|
|
|
417
516
|
console.error(c.red(`✗ mesh "${held.space}" is already running at ${server} with ${label} - a running broker can't change auth mode; \`cotal down\` it first, then \`cotal up ${wantUser ? "--user-auth" : "--open"}\``));
|
|
418
517
|
process.exit(1);
|
|
419
518
|
}
|
|
519
|
+
// A rotation retires the system account this LIVE broker was started on, and only a broker
|
|
520
|
+
// (re)started from the rewritten config carries the successor. Rotating under a running mesh
|
|
521
|
+
// would leave the record and the creds a generation ahead of the broker: every $SYS client
|
|
522
|
+
// denied, with `doctor auth` reporting freshly-minted creds. Refuse with the two-step recipe.
|
|
523
|
+
if (values["rotate-sys"]) {
|
|
524
|
+
console.error(c.red(`✗ mesh "${held.space}" is already running at ${server} - a running broker can't rotate its system account (it would keep serving the retired one); \`cotal down\` it first, then \`cotal up --rotate-sys\``));
|
|
525
|
+
process.exit(1);
|
|
526
|
+
}
|
|
527
|
+
// A running broker cannot change its transport either, and for exactly the reason it cannot
|
|
528
|
+
// change its auth mode: the listener's TLS config was fixed when nats-server read its config
|
|
529
|
+
// file. This branch starts nothing, so `--tls-cert`/`--tls-key` here can only be a request to
|
|
530
|
+
// change something that is already decided — and answering it with `✓ mesh up` told the
|
|
531
|
+
// operator they had TLS while the live broker went on serving whatever it was started with.
|
|
532
|
+
//
|
|
533
|
+
// Refuse rather than warn. The whole feature is that a command accepting a TLS flag either
|
|
534
|
+
// encrypts or refuses to start; printing a success line over an unchanged plaintext listener
|
|
535
|
+
// is the precise outcome that must be unreachable.
|
|
536
|
+
if (values["tls-cert"] || values["tls-key"]) {
|
|
537
|
+
console.error(c.red(`✗ mesh "${held.space}" is already running at ${server} - a running broker can't change its transport; \`cotal down\` it first, then \`cotal up --tls-cert <cert> --tls-key <key>\``));
|
|
538
|
+
process.exit(1);
|
|
539
|
+
}
|
|
540
|
+
// Live INFO must agree with the recorded/requested transport. A bare refresh that greets
|
|
541
|
+
// "already running" over a plaintext listener while policy claims TLS is S5's second half —
|
|
542
|
+
// the refuse left a durable lie and this path used to reprint TLS success over cleartext.
|
|
543
|
+
const liveTls = await liveListenerRequiresTls(server);
|
|
544
|
+
const wantTls = transport.kind === "tls-required" || held.tlsRequired === true;
|
|
545
|
+
if (wantTls && liveTls === false) {
|
|
546
|
+
console.error(c.red(`✗ mesh "${held.space}" at ${server} is recorded/expected TLS-required but the live listener is plaintext - refuse rather than claim it is up; \`cotal down\` then \`cotal up --tls-cert <cert> --tls-key <key>\` (or fix the broker)`));
|
|
547
|
+
process.exit(1);
|
|
548
|
+
}
|
|
549
|
+
if (!wantTls && liveTls === true) {
|
|
550
|
+
console.error(c.red(`✗ mesh "${held.space}" at ${server} is serving TLS but this mesh is recorded plaintext - \`cotal down\` then re-up with matching flags`));
|
|
551
|
+
process.exit(1);
|
|
552
|
+
}
|
|
420
553
|
console.log(c.green(`✓ mesh "${held.space}" already running at ${server}`));
|
|
421
554
|
// USER MODE: re-upping IS the documented recovery for a dead auth service (the provider's
|
|
422
555
|
// failure copy says "restart it with `cotal up`"), so a refresh must re-ensure the service —
|
|
@@ -480,10 +613,26 @@ export async function up(args) {
|
|
|
480
613
|
const heldAttachHost = attachHostFor(held.space, values.host);
|
|
481
614
|
// A broker was already answering here — this branch starts nothing, so it must not claim the
|
|
482
615
|
// record as ours (see `Provenance`).
|
|
483
|
-
|
|
616
|
+
// `tlsRequired` is CARRIED FORWARD, not re-derived. This branch starts no listener, so it has
|
|
617
|
+
// no transport decision to record — and `recordOurMesh` writes the entry whole, so omitting
|
|
618
|
+
// the field here would erase the requirement on every bare refresh, exactly the way dropping
|
|
619
|
+
// `attachHost` would silently demote the mesh to loopback.
|
|
620
|
+
recordOurMesh({ space: held.space, server, root, mode: held.mode, ...(held.tlsRequired !== undefined ? { tlsRequired: held.tlsRequired } : {}), ...(userAuth ? { userAuth } : {}), ...(heldAttachHost ? { attachHost: heldAttachHost } : {}), ts: new Date().toISOString() }, "refresh");
|
|
484
621
|
return;
|
|
485
622
|
}
|
|
486
623
|
const who = held ? `mesh "${held.space}" (${held.root})` : "a broker not started here";
|
|
624
|
+
// Reaching here with `--rotate-sys` means something IS answering at the requested address and it
|
|
625
|
+
// is not this root's recorded mesh (that case was refused above). The ordinary response is to pick
|
|
626
|
+
// a free port and carry on, which for a rotation is the wrong instinct: the unidentified listener
|
|
627
|
+
// may be a `nats-server -c <root>/.cotal/auth/server.conf` started by hand, holding THIS root's
|
|
628
|
+
// config and JetStream store while writing neither a pidfile nor a registry row. Rotating around
|
|
629
|
+
// it retires the account it is still serving and opens its store a second time. Nothing available
|
|
630
|
+
// here can identify it (that is what unidentified means), so refuse instead of stepping past it.
|
|
631
|
+
if (values["rotate-sys"]) {
|
|
632
|
+
console.error(c.red(`✗ ${server} is answering and it is not this root's recorded mesh (${who})`) +
|
|
633
|
+
c.dim(" - `--rotate-sys` will not start on another port around an unidentified broker: it may be serving this root's own server.conf and JetStream store. Stop whatever is listening there (`cotal down` if it was started here), then rotate."));
|
|
634
|
+
process.exit(1);
|
|
635
|
+
}
|
|
487
636
|
if (values.server === undefined && (!held || held.root !== root)) {
|
|
488
637
|
const next = await serverWithFreePort(server, host);
|
|
489
638
|
console.log(c.dim(`${server} is already in use by ${who}; starting "${space}" at ${next} instead`));
|
|
@@ -497,11 +646,13 @@ export async function up(args) {
|
|
|
497
646
|
if (values.detach) {
|
|
498
647
|
const restored = resumeAttempt ? pendingRestores.get(resumeAttempt) : undefined;
|
|
499
648
|
const { pid, source, authService, controlPlane, delivery, manager } = await startMeshDetached({
|
|
649
|
+
transport,
|
|
500
650
|
server,
|
|
501
651
|
storeDir: values["store-dir"],
|
|
502
652
|
space: values.space,
|
|
503
653
|
open: values.open,
|
|
504
654
|
userAuth: wantUser ? { idpUrl: values.idp } : undefined,
|
|
655
|
+
rotateSys: values["rotate-sys"],
|
|
505
656
|
channels: values.channels,
|
|
506
657
|
// The RAW flag, not the loopback-defaulted `host` above: `startMeshDetached` applies the same
|
|
507
658
|
// default itself, and what it records must distinguish "the operator asked for this address"
|
|
@@ -527,6 +678,7 @@ export async function up(args) {
|
|
|
527
678
|
} : {}),
|
|
528
679
|
skipPostStart: Boolean(resumeAttempt),
|
|
529
680
|
});
|
|
681
|
+
// Transport policy is committed inside startMeshDetached before delivery launch (S5+S9).
|
|
530
682
|
console.log(c.dim(`Started nats-server (${source}).`));
|
|
531
683
|
console.log(c.green(renderDetachedSummary({ pid, delivery, authService: wantUser && authService, manager })));
|
|
532
684
|
if (restored && process.env.COTAL_SMOKE_FAIL_AFTER_RESTORE_LISTENER_READY === "1")
|
|
@@ -556,11 +708,19 @@ export async function up(args) {
|
|
|
556
708
|
mkdirSync(storeDir, { recursive: true });
|
|
557
709
|
await claimSpace(space, server, cotalRoot());
|
|
558
710
|
const seedFile = loadChannelsFile(values.channels);
|
|
559
|
-
|
|
711
|
+
// Decided above the branch; the dial host is only settled here (a port collision may have moved
|
|
712
|
+
// the server, and the hostname is what the certificate has to match).
|
|
713
|
+
assertServesDialHost(transport, new URL(server).hostname);
|
|
714
|
+
const setup = useAuth ? await authSetup(storeDir, server, space, host, wantUser ? { idpUrl: values.idp } : undefined, transport, values["rotate-sys"]) : undefined;
|
|
560
715
|
const port = Number(new URL(server).port) || 4222;
|
|
561
716
|
const restored = resumeAttempt ? pendingRestores.get(resumeAttempt) : undefined;
|
|
717
|
+
// Both modes go through a RENDERER, never bare CLI flags. Open mode used to start from
|
|
718
|
+
// `-js -sd … -p … -a …`, which never called a renderer at all — so the required transport union
|
|
719
|
+
// protected the auth path and was silent on the open one, and a cert/key pair passed to an
|
|
720
|
+
// open-mode `up` would have been accepted while the listener came up in cleartext.
|
|
721
|
+
const confPath = setup ? setup.confPath : writeOpenBrokerConf(storeDir, { port, host, transport });
|
|
562
722
|
const natsArgs = [
|
|
563
|
-
|
|
723
|
+
"-c", confPath,
|
|
564
724
|
...(restored ? ["--name", restored.serverName]
|
|
565
725
|
: ordinaryAttempt ? ["--name", ordinaryAttempt.serverName]
|
|
566
726
|
: []),
|
|
@@ -611,8 +771,9 @@ export async function up(args) {
|
|
|
611
771
|
void stopDelivery()
|
|
612
772
|
.catch((e) => console.error(`! delivery teardown: ${e.message}`))
|
|
613
773
|
.then(() => {
|
|
614
|
-
stopManager()
|
|
615
|
-
|
|
774
|
+
void stopManager()
|
|
775
|
+
.then(() => stopAuthService(space))
|
|
776
|
+
.catch((e) => console.error(`! teardown: ${e.message}`));
|
|
616
777
|
child.kill("SIGTERM");
|
|
617
778
|
});
|
|
618
779
|
};
|
|
@@ -624,8 +785,8 @@ export async function up(args) {
|
|
|
624
785
|
rmSync(cotalPath("nats.pid"), { force: true });
|
|
625
786
|
// Logged, never silently swallowed; the daemon kill runs in stopDelivery's finally regardless.
|
|
626
787
|
await stopDelivery().catch((e) => console.error(`! delivery teardown: ${e.message}`));
|
|
627
|
-
stopManager();
|
|
628
|
-
stopAuthService(space);
|
|
788
|
+
await stopManager().catch((e) => console.error(`! manager teardown: ${e.message}`));
|
|
789
|
+
await stopAuthService(space).catch((e) => console.error(`! auth teardown: ${e.message}`));
|
|
629
790
|
// Only unrecord if the registry still points at THIS broker. A newer broker for the same space
|
|
630
791
|
// (a concurrent `up`, or a different-port re-up that recorded after us) may have replaced our
|
|
631
792
|
// record — removing by name would clobber the live winner and hide it from the registry.
|
|
@@ -651,6 +812,8 @@ export async function up(args) {
|
|
|
651
812
|
}
|
|
652
813
|
if (restored)
|
|
653
814
|
await provePreparedRestoreListener(restored);
|
|
815
|
+
// Listener ready — commit the transport decision (S5: not before start).
|
|
816
|
+
commitTransportPolicy(meshRoot, transport);
|
|
654
817
|
{
|
|
655
818
|
if (!resumeAttempt)
|
|
656
819
|
await postStart(server, space, setup, seedFile);
|
|
@@ -671,6 +834,11 @@ export async function up(args) {
|
|
|
671
834
|
recordOurMesh({
|
|
672
835
|
space, server, root: cotalRoot(),
|
|
673
836
|
mode: setup?.prepared ? "user" : useAuth ? "auth" : "open",
|
|
837
|
+
// Written ALWAYS, as a boolean, unlike `attachHost` below. Absence and `false` resolve
|
|
838
|
+
// identically for clients, so an omitted field would be indistinguishable from a deliberate
|
|
839
|
+
// plaintext mesh — and this is the one field whose whole purpose is that the answer was
|
|
840
|
+
// stated rather than defaulted.
|
|
841
|
+
tlsRequired: transport.kind === "tls-required",
|
|
674
842
|
...(svc.userAuth ? { userAuth: svc.userAuth } : {}),
|
|
675
843
|
// Only a real decision is persisted — an explicit `--host` now, or one carried forward from a
|
|
676
844
|
// previous launch. The bare case stays absent rather than recording the loopback default as
|
|
@@ -690,6 +858,8 @@ export async function up(args) {
|
|
|
690
858
|
attachHost: effectiveAttachHost,
|
|
691
859
|
resumeAttempt,
|
|
692
860
|
resumeCommitToken: restored?.managerCommit?.durableCommitToken ?? ordinaryAttempt?.managerCommit?.durableCommitToken,
|
|
861
|
+
transport,
|
|
862
|
+
wsPort: setup?.wsPort, // P2 item 6: the console session client's broker ws port
|
|
693
863
|
});
|
|
694
864
|
if (restored && process.env.COTAL_SMOKE_FAIL_AFTER_RESTORE_LISTENER_READY === "1")
|
|
695
865
|
throw new Error("smoke-injected failure after restore listener readiness");
|
|
@@ -755,10 +925,15 @@ async function resumeControlAuth(root, mode) {
|
|
|
755
925
|
if (!auth)
|
|
756
926
|
throw new Error("same-principal resume requires the existing space trust material");
|
|
757
927
|
const identity = newIdentity();
|
|
928
|
+
// The instrument's ep caller triple (1c.2b): the admin instrument's rows are lifecycle-keyed,
|
|
929
|
+
// and the triple rides back so the resume/preservation calls take askManager's ep path.
|
|
930
|
+
const uid = mintLifecycleUid();
|
|
758
931
|
return {
|
|
759
932
|
creds: await mintCreds(auth, identity, "control-caller-admin", {
|
|
933
|
+
lifecycleUid: uid,
|
|
760
934
|
expiresAt: Math.floor((Date.now() + 30 * 60 * 1000) / 1000),
|
|
761
935
|
}),
|
|
936
|
+
epCaller: { owner: DEV_OWNER, actor: identity.id, uid },
|
|
762
937
|
};
|
|
763
938
|
}
|
|
764
939
|
function restoreListenerOwner(pid, nonce, startedAt) {
|
|
@@ -910,7 +1085,7 @@ async function provePreparedRestoreListener(prepared) {
|
|
|
910
1085
|
const auth = await resumeControlAuth(prepared.root, prepared.mode);
|
|
911
1086
|
const nc = await connect({
|
|
912
1087
|
servers: prepared.server,
|
|
913
|
-
...standaloneConnectOpts(auth),
|
|
1088
|
+
...standaloneConnectOpts({ ...auth, /* not yet wired to a recorded transport - see broker-policy/MeshEntry work */ tls: false }),
|
|
914
1089
|
maxReconnectAttempts: 0,
|
|
915
1090
|
});
|
|
916
1091
|
try {
|
|
@@ -992,7 +1167,7 @@ async function proveOrdinaryResumeListener(pending) {
|
|
|
992
1167
|
const auth = await resumeControlAuth(pending.root, pending.mode);
|
|
993
1168
|
const nc = await connect({
|
|
994
1169
|
servers: pending.server,
|
|
995
|
-
...standaloneConnectOpts(auth),
|
|
1170
|
+
...standaloneConnectOpts({ ...auth, /* not yet wired to a recorded transport - see broker-policy/MeshEntry work */ tls: false }),
|
|
996
1171
|
maxReconnectAttempts: 0,
|
|
997
1172
|
});
|
|
998
1173
|
try {
|
|
@@ -1025,6 +1200,8 @@ async function resumeProvenOrdinaryListener(pending) {
|
|
|
1025
1200
|
server: pending.server,
|
|
1026
1201
|
root: pending.root,
|
|
1027
1202
|
mode: pending.mode,
|
|
1203
|
+
// Adopted from the recorded policy: this path proves and re-adopts a listener it did not start.
|
|
1204
|
+
tlsRequired: adoptedTlsRequired(pending.root),
|
|
1028
1205
|
...(svc.userAuth ? { userAuth: svc.userAuth } : {}),
|
|
1029
1206
|
...(adoptAttachHost ? { attachHost: adoptAttachHost } : {}),
|
|
1030
1207
|
ts: new Date().toISOString(),
|
|
@@ -1064,6 +1241,9 @@ async function resumeProvenRestoreListener(prepared) {
|
|
|
1064
1241
|
server: prepared.server,
|
|
1065
1242
|
root: prepared.root,
|
|
1066
1243
|
mode: prepared.mode,
|
|
1244
|
+
// Same as the ordinary-resume path: a restore adopts an existing listener, so the requirement
|
|
1245
|
+
// comes from the policy that listener was started from, never from a default.
|
|
1246
|
+
tlsRequired: adoptedTlsRequired(prepared.root),
|
|
1067
1247
|
...(svc.userAuth ? { userAuth: svc.userAuth } : {}),
|
|
1068
1248
|
...(restoreAttachHost ? { attachHost: restoreAttachHost } : {}),
|
|
1069
1249
|
ts: new Date().toISOString(),
|
|
@@ -1129,7 +1309,7 @@ async function completeResumeActivation(attemptId, healthy, reason, server) {
|
|
|
1129
1309
|
const auth = await resumeControlAuth(pending.root, pending.mode);
|
|
1130
1310
|
const readinessDeadline = Date.now() + 20_000;
|
|
1131
1311
|
for (;;) {
|
|
1132
|
-
const ready = await askManager(pending.space, server, "ps", undefined, auth,
|
|
1312
|
+
const ready = await askManager(pending.space, server, "ps", undefined, auth, "any", 2_000);
|
|
1133
1313
|
if (!ready.error?.startsWith("no manager reachable"))
|
|
1134
1314
|
break;
|
|
1135
1315
|
if (Date.now() >= readinessDeadline) {
|
|
@@ -1143,7 +1323,7 @@ async function completeResumeActivation(attemptId, healthy, reason, server) {
|
|
|
1143
1323
|
inventory: restored?.selection === "registry"
|
|
1144
1324
|
? { ...pending.inventory, agents: [] }
|
|
1145
1325
|
: pending.inventory,
|
|
1146
|
-
}, auth,
|
|
1326
|
+
}, auth, "any", 10 * 60 * 1000);
|
|
1147
1327
|
if (!resumed.ok) {
|
|
1148
1328
|
const detail = resumed.data ? ` (${JSON.stringify(resumed.data)})` : "";
|
|
1149
1329
|
const message = `${resumed.error ?? "retained-agent resume failed"}${detail}`;
|
|
@@ -1171,7 +1351,7 @@ async function completeResumeActivation(attemptId, healthy, reason, server) {
|
|
|
1171
1351
|
}
|
|
1172
1352
|
}
|
|
1173
1353
|
if (!managerCommit) {
|
|
1174
|
-
const committed = await askManager(pending.space, server, "commitResume", { attemptId }, auth,
|
|
1354
|
+
const committed = await askManager(pending.space, server, "commitResume", { attemptId }, auth, "any", 40_000);
|
|
1175
1355
|
if (!committed.ok) {
|
|
1176
1356
|
const message = committed.error ?? "manager resume commit failed";
|
|
1177
1357
|
markPendingResumeDegraded(attemptId, message);
|
|
@@ -1201,7 +1381,7 @@ async function completeResumeActivation(attemptId, healthy, reason, server) {
|
|
|
1201
1381
|
process.exit(89);
|
|
1202
1382
|
}
|
|
1203
1383
|
else {
|
|
1204
|
-
const recommitted = await askManager(pending.space, server, "commitResume", { attemptId }, auth,
|
|
1384
|
+
const recommitted = await askManager(pending.space, server, "commitResume", { attemptId }, auth, "any", 40_000);
|
|
1205
1385
|
// A surviving manager that already finalized legitimately answers {state:"active"} with the
|
|
1206
1386
|
// exact durable token: accept both committed shapes idempotently, then reissue the token-bound
|
|
1207
1387
|
// finalize (itself idempotent) below.
|
|
@@ -1213,7 +1393,7 @@ async function completeResumeActivation(attemptId, healthy, reason, server) {
|
|
|
1213
1393
|
if (!exactToken)
|
|
1214
1394
|
throw new Error(`replacement manager did not recover the durable commit token for attempt ${attemptId}`);
|
|
1215
1395
|
}
|
|
1216
|
-
const finalized = await askManager(pending.space, server, "finalizeResume", { attemptId, durableCommitToken: managerCommit.durableCommitToken }, auth,
|
|
1396
|
+
const finalized = await askManager(pending.space, server, "finalizeResume", { attemptId, durableCommitToken: managerCommit.durableCommitToken }, auth, "any", 40_000);
|
|
1217
1397
|
if (!finalized.ok)
|
|
1218
1398
|
throw new Error(finalized.error ?? `manager resume finalize failed for attempt ${attemptId}`);
|
|
1219
1399
|
if (!isManagerFinalizeResult(finalized.data, attemptId))
|
|
@@ -1305,7 +1485,21 @@ async function upManifest(file, opts) {
|
|
|
1305
1485
|
const open = m.broker?.auth === false; // default is auth
|
|
1306
1486
|
const userAuth = m.broker?.auth === "user" ? { idpUrl: opts.idp ?? m.broker?.idp } : undefined; // flag > manifest
|
|
1307
1487
|
const runtime = m.runtime ?? "pty";
|
|
1488
|
+
// The open-mesh refusal must be RE-STATED here, not only against the CLI `--open` flag: on this
|
|
1489
|
+
// path openness comes from the MANIFEST (`broker.auth: false`), which the flag-level guard cannot
|
|
1490
|
+
// see. Without it, `cotal up -f open.yaml --rotate-sys` boots an open broker, exits 0 and rotates
|
|
1491
|
+
// nothing: a rotation ask answered with silent success, the exact failure class this change
|
|
1492
|
+
// exists to remove. Before the dry-run print and before anything boots.
|
|
1493
|
+
if (opts.rotateSys && open)
|
|
1494
|
+
throw new Error("--rotate-sys is for auth meshes: this manifest sets broker.auth: false (open), so there is no system account or $SYS credentials to rotate");
|
|
1308
1495
|
if (opts.dryRun) {
|
|
1496
|
+
// Dial-host SAN must refuse here too (S6). The real route runs assertServesDialHost after
|
|
1497
|
+
// the plan is computed; dry-run used to return before that check and green-lit a launch
|
|
1498
|
+
// the applying command refuses. Validate against the effective server; still no writes.
|
|
1499
|
+
// Announce only AFTER the dial-host check so a wrong-SAN dry-run does not print TLS: serving.
|
|
1500
|
+
assertServesDialHost(opts.transport, new URL(server).hostname);
|
|
1501
|
+
if (opts.transport.kind === "tls-required")
|
|
1502
|
+
announceTransport(opts.transport);
|
|
1309
1503
|
console.log(renderUpPlan(eff, server));
|
|
1310
1504
|
return;
|
|
1311
1505
|
}
|
|
@@ -1332,7 +1526,10 @@ async function upManifest(file, opts) {
|
|
|
1332
1526
|
if (userAuth) {
|
|
1333
1527
|
try {
|
|
1334
1528
|
mkdirSync(cotalPath("nats"), { recursive: true });
|
|
1335
|
-
|
|
1529
|
+
// NO `rotateSys` here on purpose: this is an owner-derivation pre-resolve that runs BEFORE the
|
|
1530
|
+
// real boot, and the boot below carries the flag. Rotating in both would burn two generations
|
|
1531
|
+
// per `up -f --rotate-sys` and retire the account the first pass just minted creds against.
|
|
1532
|
+
const setup = await authSetup(cotalPath("nats"), server, m.space, host, userAuth, assertServesDialHost(opts.transport, new URL(server).hostname));
|
|
1336
1533
|
owner = await resolveAuthProvider().ownerForLogin({ store: workspaceSecretStore(cotalRoot()), dir: setup.stateDir, space: m.space });
|
|
1337
1534
|
}
|
|
1338
1535
|
catch (e) {
|
|
@@ -1350,19 +1547,20 @@ async function upManifest(file, opts) {
|
|
|
1350
1547
|
// A leftover detached manager (its broker is gone — the reachability check above proved nothing
|
|
1351
1548
|
// lives at this address) would win the fresh mesh's lease and the launch manager would refuse.
|
|
1352
1549
|
// Stop it, so the manager started below WITH the launch spec is THE manager.
|
|
1353
|
-
stopManager();
|
|
1550
|
+
await stopManager();
|
|
1354
1551
|
let pid;
|
|
1355
1552
|
let controlPlane = false;
|
|
1356
1553
|
let authService = true;
|
|
1357
1554
|
try {
|
|
1358
1555
|
// `m.broker?.host`, not the defaulted `host`: same reason as the flag path — only a host the
|
|
1359
1556
|
// manifest actually declared is an exposure decision worth persisting.
|
|
1360
|
-
({ pid, controlPlane, authService } = await startMeshDetached({ server, space: m.space, open, userAuth, host: m.broker?.host, seed: manifestToChannels(eff), runtime, launch: specPath }));
|
|
1557
|
+
({ pid, controlPlane, authService } = await startMeshDetached({ transport: opts.transport, server, space: m.space, open, userAuth, rotateSys: opts.rotateSys, host: m.broker?.host, seed: manifestToChannels(eff), runtime, launch: specPath }));
|
|
1361
1558
|
}
|
|
1362
1559
|
catch (e) {
|
|
1363
1560
|
console.error(c.red(`✗ ${e.message}`));
|
|
1364
1561
|
process.exit(1);
|
|
1365
1562
|
}
|
|
1563
|
+
// Transport committed inside startMeshDetached before delivery (S5+S9).
|
|
1366
1564
|
console.log(c.green(`✓ mesh "${m.space}" up at ${server}`) + c.dim(` (broker pid ${pid})`));
|
|
1367
1565
|
console.log(c.dim(` seeded ${m.channels.length} channel(s): ${m.channels.map((ch) => "#" + ch.name).join(", ")}`));
|
|
1368
1566
|
// Never claim a launch the control plane can't deliver: the manager carries the launch spec, so a
|
|
@@ -1418,7 +1616,12 @@ function applyUpOverrides(prepared, o) {
|
|
|
1418
1616
|
* caller whose output claims a manager (the `up -f` launching line) can tell the truth. */
|
|
1419
1617
|
async function startDeliveryWithBroker(space, server, mgr) {
|
|
1420
1618
|
try {
|
|
1421
|
-
|
|
1619
|
+
// Prefer the in-memory decision for a fresh apply (policy may not be on disk yet — S9).
|
|
1620
|
+
// Fall back to the recorded policy for resume/restore/refresh paths that start no new listener.
|
|
1621
|
+
const tls = mgr?.transport?.kind === "tls-required" ||
|
|
1622
|
+
readBrokerPolicy(cotalRoot())?.transport.kind === "tls-required";
|
|
1623
|
+
const { transport: _t, ...mgrRest } = mgr ?? {};
|
|
1624
|
+
await ensureControlPlane({ space, server, tls, ...mgrRest });
|
|
1422
1625
|
return true;
|
|
1423
1626
|
}
|
|
1424
1627
|
catch (e) {
|
|
@@ -1436,9 +1639,17 @@ async function startDeliveryWithBroker(space, server, mgr) {
|
|
|
1436
1639
|
* log file and forwarded — the child writes to the file (not a pipe), so it survives the
|
|
1437
1640
|
* parent exiting.
|
|
1438
1641
|
*/
|
|
1439
|
-
export async function startMeshDetached(
|
|
1642
|
+
export async function startMeshDetached(
|
|
1643
|
+
// The `= {}` default is gone with the optional transport: an options object that can be omitted
|
|
1644
|
+
// entirely cannot carry a mandatory decision.
|
|
1645
|
+
opts) {
|
|
1440
1646
|
const server = opts.server ?? DEFAULT_SERVER;
|
|
1441
1647
|
const useAuth = !opts.open;
|
|
1648
|
+
// Belt on the one boot that both renders the broker config and can be reached by any caller: an
|
|
1649
|
+
// open boot skips `authSetup` entirely, so a `rotateSys` arriving here would be dropped in silence
|
|
1650
|
+
// rather than refused. Callers guard this too; this is the seam that cannot be bypassed.
|
|
1651
|
+
if (opts.rotateSys && !useAuth)
|
|
1652
|
+
throw new Error("startMeshDetached: --rotate-sys is for auth meshes; an open mesh has no system account or $SYS credentials to rotate");
|
|
1442
1653
|
const space = opts.space ?? resolveSpace(process.cwd());
|
|
1443
1654
|
ensureRootForSpace(useAuth, space); // may pin the cwd as this space's root — before any cotalPath use
|
|
1444
1655
|
refuseOpenOverUserState(Boolean(opts.open), space);
|
|
@@ -1447,10 +1658,17 @@ export async function startMeshDetached(opts = {}) {
|
|
|
1447
1658
|
await claimSpace(space, server, cotalRoot());
|
|
1448
1659
|
const seedFile = opts.seed ?? loadChannelsFile(opts.channels);
|
|
1449
1660
|
const host = opts.host ?? "127.0.0.1";
|
|
1450
|
-
|
|
1661
|
+
// The transport arrives decided. The dial-host SAN is re-checked here because THIS is where the
|
|
1662
|
+
// effective server is finally known — a manifest may name a different host than the flags did, and
|
|
1663
|
+
// a certificate that is valid for one is not thereby valid for the other.
|
|
1664
|
+
const transport = assertServesDialHost(opts.transport, new URL(server).hostname);
|
|
1665
|
+
const setup = useAuth ? await authSetup(storeDir, server, space, host, opts.userAuth, transport, opts.rotateSys) : undefined;
|
|
1451
1666
|
const port = Number(new URL(server).port) || 4222;
|
|
1667
|
+
// Same rule as the foreground path: every route to a listener names its transport (see
|
|
1668
|
+
// `writeOpenBrokerConf`). Detach must not be the mode where the fence quietly does not apply.
|
|
1669
|
+
const confPath = setup ? setup.confPath : writeOpenBrokerConf(storeDir, { port, host, transport });
|
|
1452
1670
|
const args = [
|
|
1453
|
-
|
|
1671
|
+
"-c", confPath,
|
|
1454
1672
|
...(opts.boundListener ? ["--name", opts.boundListener.serverName] : []),
|
|
1455
1673
|
];
|
|
1456
1674
|
const { bin, source } = await resolveNatsServer();
|
|
@@ -1489,8 +1707,37 @@ export async function startMeshDetached(opts = {}) {
|
|
|
1489
1707
|
writeFileSync(cotalPath("nats.pid"), String(child.pid));
|
|
1490
1708
|
if (opts.boundListener)
|
|
1491
1709
|
await opts.boundListener.verify();
|
|
1492
|
-
|
|
1493
|
-
|
|
1710
|
+
// POST-START MUST NOT LEAVE AN ORPHAN LISTENER.
|
|
1711
|
+
//
|
|
1712
|
+
// Everything above has already bound the port and written `nats.pid`, but NOTHING has recorded the
|
|
1713
|
+
// mesh yet — `recordOurMesh` is below. So a throw between here and there used to exit non-zero
|
|
1714
|
+
// while leaving a live broker holding the port with no registry entry, which `cotal down` cannot
|
|
1715
|
+
// reach because `down` works from the registry. A third state between "started" and "refused",
|
|
1716
|
+
// and the operator's only recourse is to hunt a pid.
|
|
1717
|
+
//
|
|
1718
|
+
// This is reachable BECAUSE of TLS and cannot happen on `main`: the post-start client verifies the
|
|
1719
|
+
// certificate, so a private CA without `NODE_EXTRA_CA_CERTS` fails here — after the listener is up.
|
|
1720
|
+
// The feature introduced the state, so the feature tears it down.
|
|
1721
|
+
//
|
|
1722
|
+
// Deliberately narrow: this is a teardown on the failure path, not a restructuring of the launch
|
|
1723
|
+
// sequence. The listener is stopped and the pid file removed, then the original error is rethrown
|
|
1724
|
+
// unchanged — the operator needs the certificate error, not a message about cleanup.
|
|
1725
|
+
if (!opts.skipPostStart) {
|
|
1726
|
+
try {
|
|
1727
|
+
await postStart(server, space, setup, seedFile);
|
|
1728
|
+
}
|
|
1729
|
+
catch (e) {
|
|
1730
|
+
try {
|
|
1731
|
+
child.kill("SIGTERM");
|
|
1732
|
+
}
|
|
1733
|
+
catch { /* already gone */ }
|
|
1734
|
+
try {
|
|
1735
|
+
rmSync(cotalPath("nats.pid"), { force: true });
|
|
1736
|
+
}
|
|
1737
|
+
catch { /* best effort */ }
|
|
1738
|
+
throw e;
|
|
1739
|
+
}
|
|
1740
|
+
}
|
|
1494
1741
|
// USER MODE: the auth service comes up FIRST among the daemons (see the foreground path).
|
|
1495
1742
|
const svc = await startUserAuthService(space, server, setup);
|
|
1496
1743
|
// Record BEFORE the control plane: the manager's fail-closed mode detection needs the
|
|
@@ -1503,11 +1750,17 @@ export async function startMeshDetached(opts = {}) {
|
|
|
1503
1750
|
recordOurMesh({
|
|
1504
1751
|
space, server, root: cotalRoot(),
|
|
1505
1752
|
mode: setup?.prepared ? "user" : useAuth ? "auth" : "open",
|
|
1753
|
+
// The detached listener is started from `transport` a few lines above, so this is the same
|
|
1754
|
+
// decision that shaped the config file — not a re-derivation.
|
|
1755
|
+
tlsRequired: transport.kind === "tls-required",
|
|
1506
1756
|
...(svc.userAuth ? { userAuth: svc.userAuth } : {}),
|
|
1507
1757
|
// Persist only a real decision — declared now, or carried forward — never the loopback default.
|
|
1508
1758
|
...(effectiveAttachHost ? { attachHost: effectiveAttachHost } : {}),
|
|
1509
1759
|
ts: new Date().toISOString(),
|
|
1510
1760
|
}, "started");
|
|
1761
|
+
// Commit policy BEFORE delivery launch (S9). Listener is proved; refuse paths never reach here.
|
|
1762
|
+
// startDeliveryWithBroker also receives `transport` so it does not depend on the file alone.
|
|
1763
|
+
commitTransportPolicy(cotalRoot(), transport);
|
|
1511
1764
|
// Bring up the delivery daemon WITH the detached broker (auth mode only; `cotal down` tears both down).
|
|
1512
1765
|
const controlPlane = await startDeliveryWithBroker(space, server, {
|
|
1513
1766
|
runtime: opts.runtime,
|
|
@@ -1516,6 +1769,8 @@ export async function startMeshDetached(opts = {}) {
|
|
|
1516
1769
|
attachHost: effectiveAttachHost,
|
|
1517
1770
|
resumeAttempt: opts.resumeAttempt,
|
|
1518
1771
|
resumeCommitToken: opts.resumeCommitToken,
|
|
1772
|
+
transport,
|
|
1773
|
+
wsPort: setup?.wsPort, // P2 item 6: the console session client's broker ws port
|
|
1519
1774
|
});
|
|
1520
1775
|
return {
|
|
1521
1776
|
server,
|
|
@@ -1539,28 +1794,42 @@ function refuseOpenOverUserState(open, space) {
|
|
|
1539
1794
|
const stateDir = userAuthStateDir(cotalRoot(), space);
|
|
1540
1795
|
throw new Error(`space "${space}" has user auth enabled (state under ${stateDir}) - \`--open\` would serve its streams without auth. Start it with \`cotal up --user-auth\`, or remove that directory deliberately to disable user auth (existing logins/grants die with it)`);
|
|
1541
1796
|
}
|
|
1542
|
-
/**
|
|
1543
|
-
*
|
|
1544
|
-
*
|
|
1545
|
-
*
|
|
1546
|
-
*
|
|
1547
|
-
*
|
|
1548
|
-
*
|
|
1549
|
-
*
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1797
|
+
/** Pin cwd as its own mesh root when the nearest ancestor root already belongs to a DIFFERENT
|
|
1798
|
+
* space — mode-independent.
|
|
1799
|
+
*
|
|
1800
|
+
* Auth path: a root's `.cotal/auth` is space-bound, so an explicit `--space` naming another space
|
|
1801
|
+
* cannot share that trust material. Open path: the same ancestor still owns policy, store, pid
|
|
1802
|
+
* and MeshEntry; reusing it for a second space collides those (S4 open arm — child `--open --tls-*`
|
|
1803
|
+
* wrote the parent's broker-policy and nats.pid while a second broker shared the JetStream store).
|
|
1804
|
+
*
|
|
1805
|
+
* A mismatch only exists when the resolved space differs from what the ancestor already hosts.
|
|
1806
|
+
* When the root was merely inherited (cwd has no `.cotal`), honor the new-space intent by making
|
|
1807
|
+
* cwd its own root. Only a folder that itself holds the other space's material refuses.
|
|
1808
|
+
* (When multi-space-per-root lands, that refusal becomes provision-the-new-space instead.) */
|
|
1809
|
+
function ensureRootForSpace(_useAuth, space) {
|
|
1553
1810
|
const root = cotalRoot();
|
|
1554
|
-
const existing = loadSoleSpaceAuth(authDir(root));
|
|
1555
|
-
if (!existing || existing.space === space)
|
|
1556
|
-
return;
|
|
1557
1811
|
const cwd = process.cwd();
|
|
1812
|
+
// What space does this root already host? Prefer auth material; fall back to a live MeshEntry
|
|
1813
|
+
// whose root is this directory (open meshes have no auth dir).
|
|
1814
|
+
const existingAuth = loadSoleSpaceAuth(authDir(root));
|
|
1815
|
+
const existingSpace = existingAuth?.space ??
|
|
1816
|
+
loadMeshes().find((m) => m.root === root || realpathSafe(m.root) === realpathSafe(root))?.space;
|
|
1817
|
+
if (!existingSpace || existingSpace === space)
|
|
1818
|
+
return;
|
|
1558
1819
|
if (root !== cwd) {
|
|
1559
1820
|
mkdirSync(join(cwd, ".cotal"), { recursive: true });
|
|
1560
|
-
console.log(c.dim(`nearest mesh root ${root} is space "${
|
|
1821
|
+
console.log(c.dim(`nearest mesh root ${root} is space "${existingSpace}" - making this folder its own root for "${space}"`));
|
|
1561
1822
|
return;
|
|
1562
1823
|
}
|
|
1563
|
-
throw new Error(`this folder is the root of space "${
|
|
1824
|
+
throw new Error(`this folder is the root of space "${existingSpace}" (${root}/.cotal), so it can't also run "${space}" - drop \`--space\` to run "${existingSpace}", or start "${space}" from a different folder (it becomes that mesh's own root)`);
|
|
1825
|
+
}
|
|
1826
|
+
function realpathSafe(p) {
|
|
1827
|
+
try {
|
|
1828
|
+
return realpathSync(p);
|
|
1829
|
+
}
|
|
1830
|
+
catch {
|
|
1831
|
+
return p;
|
|
1832
|
+
}
|
|
1564
1833
|
}
|
|
1565
1834
|
/** A space name maps to one mesh in the registry (the key `--space`/`use`/`down` act on). Before
|
|
1566
1835
|
* starting a broker, refuse to reuse a space already claimed by a DIFFERENT live mesh — a stale/dead
|
|
@@ -1745,8 +2014,219 @@ function loadChannelsFile(explicit) {
|
|
|
1745
2014
|
* SPACE-SCOPED state dir (`.cotal/auth/<space>/` — the multi-space-ready layout; nothing user-auth
|
|
1746
2015
|
* lives flat) and preload its extra account(s) into the broker config. The inverse is fail-closed:
|
|
1747
2016
|
* a space whose user-auth state exists MUST keep being started with --user-auth — regenerating the
|
|
1748
|
-
* config without the callout account would silently break every sentinel connect.
|
|
1749
|
-
|
|
2017
|
+
* config without the callout account would silently break every sentinel connect.
|
|
2018
|
+
*
|
|
2019
|
+
* `rotateSys` (`cotal up --rotate-sys`) is the class-3 renewal for an EXISTING space: rotate the
|
|
2020
|
+
* system account, re-mint `membership-observer.creds` + `connection-evictor.creds` against the
|
|
2021
|
+
* successor, and render the config from the ROTATED record so the broker this `up` starts is the one
|
|
2022
|
+
* that trusts them. It belongs here because this is the single site that both owns the trust record
|
|
2023
|
+
* and renders `server.conf`; anywhere else would publish creds the live broker cannot honor. */
|
|
2024
|
+
/**
|
|
2025
|
+
* Turn the `--tls-cert`/`--tls-key` pair into a validated {@link BrokerTransport}, or `plaintext`
|
|
2026
|
+
* when neither was given.
|
|
2027
|
+
*
|
|
2028
|
+
* REFUSES BEFORE LAUNCH, never after. Everything it rejects would otherwise become a running broker
|
|
2029
|
+
* in some wrong state, and the two failure modes are not symmetrical: a broker that will not start
|
|
2030
|
+
* is an operator reading an error, while a broker that starts wrong is an operator believing they
|
|
2031
|
+
* have TLS. So the validation is deliberately ours and deliberately early.
|
|
2032
|
+
*
|
|
2033
|
+
* `validateTlsMaterial` does the substantive checks — readability, private-key mode, PEM pair
|
|
2034
|
+
* match, validity window, dial-host SAN — because nats-server's are not sufficient. Missing,
|
|
2035
|
+
* unreadable and mismatched pairs do stop it before it opens a listener, but an EXPIRED certificate
|
|
2036
|
+
* does not: it reports the config valid, starts, logs "Server is ready" and "TLS required", and
|
|
2037
|
+
* only the client then fails. An expired cert must never yield "mesh up".
|
|
2038
|
+
*
|
|
2039
|
+
* @param dialHost the hostname clients will verify against, which is NOT the bind host — a broker
|
|
2040
|
+
* may bind `0.0.0.0` while clients dial `broker.example`.
|
|
2041
|
+
*/
|
|
2042
|
+
function resolveTransport(values, dialHost, root,
|
|
2043
|
+
/** `persist` defaults false (commit-after-apply). `quiet` suppresses serving/inheriting lines
|
|
2044
|
+
* until the caller announces after a successful apply. */
|
|
2045
|
+
opts = {}) {
|
|
2046
|
+
const certFile = values["tls-cert"], keyFile = values["tls-key"];
|
|
2047
|
+
const quiet = opts.quiet === true;
|
|
2048
|
+
// NO FLAGS: inherit the RECORDED decision rather than defaulting to plaintext. This is the half
|
|
2049
|
+
// that makes a bare `cotal up` after a `cotal down` keep serving TLS. Without it the decision
|
|
2050
|
+
// lives only in the argv of whichever invocation first made it, and the most ordinary operator
|
|
2051
|
+
// gesture there is - stop the mesh, start it again - silently downgrades a broker that was
|
|
2052
|
+
// deliberately put on TLS. `readBrokerPolicy` refuses rather than degrading if the recorded
|
|
2053
|
+
// material has gone missing, so a moved cert fails loud here instead of coming up in cleartext.
|
|
2054
|
+
if (!certFile && !keyFile) {
|
|
2055
|
+
let recorded;
|
|
2056
|
+
try {
|
|
2057
|
+
recorded = readBrokerPolicy(root);
|
|
2058
|
+
}
|
|
2059
|
+
catch (e) {
|
|
2060
|
+
console.error(c.red(`✗ ${e instanceof Error ? e.message : String(e)}`));
|
|
2061
|
+
process.exit(1);
|
|
2062
|
+
}
|
|
2063
|
+
if (recorded?.transport.kind === "tls-required") {
|
|
2064
|
+
if (!quiet)
|
|
2065
|
+
console.log(c.dim("TLS: inheriting the recorded broker policy (no --tls-cert/--tls-key given)"));
|
|
2066
|
+
return validateRecorded(recorded.transport, dialHost, { quiet });
|
|
2067
|
+
}
|
|
2068
|
+
return { kind: "plaintext" };
|
|
2069
|
+
}
|
|
2070
|
+
// Half a pair is a mistake, never a configuration. Refusing names the missing half rather than
|
|
2071
|
+
// failing later inside nats-server with a message about a file the operator did not mention.
|
|
2072
|
+
if (!certFile || !keyFile) {
|
|
2073
|
+
console.error(c.red(`✗ --tls-cert and --tls-key must be given together (missing ${certFile ? "--tls-key" : "--tls-cert"})`));
|
|
2074
|
+
process.exit(1);
|
|
2075
|
+
}
|
|
2076
|
+
const transport = { kind: "tls-required", certFile: resolve(certFile), keyFile: resolve(keyFile) };
|
|
2077
|
+
const validated = validateRecorded(transport, dialHost, { quiet });
|
|
2078
|
+
// RECORD only when the caller has already applied the transport (started a matching listener).
|
|
2079
|
+
// Eager write here is S5: a refuse path cannot unwrite, and the next bare up inherits fiction.
|
|
2080
|
+
if (opts.persist === true)
|
|
2081
|
+
writeBrokerPolicy(root, { version: 1, transport: validated });
|
|
2082
|
+
return validated;
|
|
2083
|
+
}
|
|
2084
|
+
/** Persist + announce a transport that has been APPLIED (listener started and proved). */
|
|
2085
|
+
function commitTransportPolicy(root, transport) {
|
|
2086
|
+
if (transport.kind !== "tls-required")
|
|
2087
|
+
return;
|
|
2088
|
+
writeBrokerPolicy(root, { version: 1, transport });
|
|
2089
|
+
announceTransport(transport);
|
|
2090
|
+
}
|
|
2091
|
+
function announceTransport(transport) {
|
|
2092
|
+
if (transport.kind !== "tls-required")
|
|
2093
|
+
return;
|
|
2094
|
+
try {
|
|
2095
|
+
const material = validateTlsMaterial(transport, {});
|
|
2096
|
+
console.log(c.dim(`TLS: serving ${material.subject}, valid until ${material.notAfter.toISOString()}`));
|
|
2097
|
+
}
|
|
2098
|
+
catch {
|
|
2099
|
+
// Already validated at decide-time; announce is best-effort display.
|
|
2100
|
+
}
|
|
2101
|
+
}
|
|
2102
|
+
/** Read the live INFO greeting and report whether the listener requires TLS.
|
|
2103
|
+
* `undefined` means no usable INFO (timeout / closed) — caller treats as unknown. */
|
|
2104
|
+
async function liveListenerRequiresTls(server, timeoutMs = 2000) {
|
|
2105
|
+
let host;
|
|
2106
|
+
let port;
|
|
2107
|
+
try {
|
|
2108
|
+
const u = new URL(server);
|
|
2109
|
+
host = u.hostname || "127.0.0.1";
|
|
2110
|
+
port = Number(u.port) || 4222;
|
|
2111
|
+
}
|
|
2112
|
+
catch {
|
|
2113
|
+
return undefined;
|
|
2114
|
+
}
|
|
2115
|
+
return await new Promise((res) => {
|
|
2116
|
+
const sock = createConnection({ host, port });
|
|
2117
|
+
let buf = "";
|
|
2118
|
+
const done = (v) => {
|
|
2119
|
+
try {
|
|
2120
|
+
sock.destroy();
|
|
2121
|
+
}
|
|
2122
|
+
catch { /* */ }
|
|
2123
|
+
res(v);
|
|
2124
|
+
};
|
|
2125
|
+
sock.setTimeout(timeoutMs, () => done(undefined));
|
|
2126
|
+
sock.on("error", () => done(undefined));
|
|
2127
|
+
sock.on("close", () => done(undefined));
|
|
2128
|
+
sock.on("data", (d) => {
|
|
2129
|
+
buf += d.toString("utf8");
|
|
2130
|
+
const nl = buf.indexOf("\r\n");
|
|
2131
|
+
if (nl < 0)
|
|
2132
|
+
return;
|
|
2133
|
+
const line = buf.slice(0, nl);
|
|
2134
|
+
try {
|
|
2135
|
+
const info = JSON.parse(line.replace(/^INFO\s+/, ""));
|
|
2136
|
+
done(info.tls_required === true);
|
|
2137
|
+
}
|
|
2138
|
+
catch {
|
|
2139
|
+
done(undefined);
|
|
2140
|
+
}
|
|
2141
|
+
});
|
|
2142
|
+
});
|
|
2143
|
+
}
|
|
2144
|
+
/**
|
|
2145
|
+
* Re-assert that `transport` is valid for the host clients will actually dial, and exit loudly if
|
|
2146
|
+
* not. A no-op for plaintext.
|
|
2147
|
+
*
|
|
2148
|
+
* This exists because the transport is decided ONCE, early, before the route is known, so that no
|
|
2149
|
+
* route can be reached without one — but the effective dial host is decided LATE and differs per
|
|
2150
|
+
* route: a manifest may set `broker.host`, and a certificate valid for the flag-derived host is not
|
|
2151
|
+
* thereby valid for that one. Deciding early and checking late is the only ordering that satisfies
|
|
2152
|
+
* both, so the check is deliberately separate from the decision rather than folded into it.
|
|
2153
|
+
*
|
|
2154
|
+
* Quiet on success: `resolveTransport` already printed what is being served, and a second identical
|
|
2155
|
+
* line reads like a second broker.
|
|
2156
|
+
*/
|
|
2157
|
+
function assertServesDialHost(transport, dialHost) {
|
|
2158
|
+
if (transport.kind !== "tls-required" || dialHost === undefined)
|
|
2159
|
+
return transport;
|
|
2160
|
+
try {
|
|
2161
|
+
validateTlsMaterial(transport, { dialHost });
|
|
2162
|
+
}
|
|
2163
|
+
catch (e) {
|
|
2164
|
+
console.error(c.red(`✗ ${e instanceof Error ? e.message : String(e)}`));
|
|
2165
|
+
process.exit(1);
|
|
2166
|
+
}
|
|
2167
|
+
return transport;
|
|
2168
|
+
}
|
|
2169
|
+
/**
|
|
2170
|
+
* The client-side TLS requirement to persist for a listener THIS PROCESS DID NOT START — the
|
|
2171
|
+
* restore and ordinary-resume paths, which adopt a broker that is already listening.
|
|
2172
|
+
*
|
|
2173
|
+
* They have no `transport` in scope because they never decided one, so the honest source is the
|
|
2174
|
+
* recorded policy: the listener they are adopting was started from it. Reading it here is what stops
|
|
2175
|
+
* a resume from writing a registry entry that says "plaintext" over a broker serving TLS — clients
|
|
2176
|
+
* resolved from that entry would then connect without requiring it, which is the downgrade this
|
|
2177
|
+
* feature exists to prevent, arriving by way of a recovery path rather than a launch.
|
|
2178
|
+
*
|
|
2179
|
+
* Fails loud on an unusable policy for the same reason `resolveTransport` does: a resume that cannot
|
|
2180
|
+
* tell what it is adopting must not guess.
|
|
2181
|
+
*/
|
|
2182
|
+
function adoptedTlsRequired(root) {
|
|
2183
|
+
try {
|
|
2184
|
+
return readBrokerPolicy(root)?.transport.kind === "tls-required";
|
|
2185
|
+
}
|
|
2186
|
+
catch (e) {
|
|
2187
|
+
console.error(c.red(`✗ ${e instanceof Error ? e.message : String(e)}`));
|
|
2188
|
+
process.exit(1);
|
|
2189
|
+
}
|
|
2190
|
+
}
|
|
2191
|
+
/** Validate a TLS pair and exit loudly on anything wrong. Shared by the flag path and the
|
|
2192
|
+
* recorded-policy path so a cert that rots on disk is caught on the NEXT `up`, not only on the
|
|
2193
|
+
* one that first configured it. */
|
|
2194
|
+
function validateRecorded(transport, dialHost, opts = {}) {
|
|
2195
|
+
try {
|
|
2196
|
+
const material = validateTlsMaterial(transport, dialHost !== undefined ? { dialHost } : {});
|
|
2197
|
+
if (!opts.quiet)
|
|
2198
|
+
console.log(c.dim(`TLS: serving ${material.subject}, valid until ${material.notAfter.toISOString()}`));
|
|
2199
|
+
}
|
|
2200
|
+
catch (e) {
|
|
2201
|
+
// Surface the CERTIFICATE cause. A TLS failure reported as a generic startup error invites the
|
|
2202
|
+
// wrong remedy - operators go looking at ports and firewalls when the answer is the cert.
|
|
2203
|
+
console.error(c.red(`✗ ${e instanceof Error ? e.message : String(e)}`));
|
|
2204
|
+
process.exit(1);
|
|
2205
|
+
}
|
|
2206
|
+
return transport;
|
|
2207
|
+
}
|
|
2208
|
+
/**
|
|
2209
|
+
* Render and write the config for an OPEN (no-auth) broker, returning its path.
|
|
2210
|
+
*
|
|
2211
|
+
* The `authSetup` sibling for the no-auth mode. Open mode previously launched nats-server from
|
|
2212
|
+
* bare CLI flags and so never passed through any renderer — meaning the REQUIRED transport union
|
|
2213
|
+
* on the config renderers covered the auth path only, and a `--tls-cert`/`--tls-key` pair given to
|
|
2214
|
+
* an open-mode `up` would have been accepted while the broker came up in cleartext. Routing open
|
|
2215
|
+
* mode through a renderer is what makes the union total: there is now no route to a listener that
|
|
2216
|
+
* does not state its transport.
|
|
2217
|
+
*/
|
|
2218
|
+
function writeOpenBrokerConf(storeDir, opts) {
|
|
2219
|
+
const confPath = resolve(storeDir, "..", "server-open.conf");
|
|
2220
|
+
writeFileSync(confPath, openServerConfig({ port: opts.port, host: opts.host, storeDir, transport: opts.transport }));
|
|
2221
|
+
return confPath;
|
|
2222
|
+
}
|
|
2223
|
+
async function authSetup(storeDir, server, space, host = "127.0.0.1", user,
|
|
2224
|
+
// NON-OPTIONAL, and it used to default to `{ kind: "plaintext" }`. That default was the hole: the
|
|
2225
|
+
// manifest path called this with five arguments and silently got a cleartext listener while the
|
|
2226
|
+
// operator's `--tls-cert` sat validated and unused. A default here cannot be right, because the
|
|
2227
|
+
// safe value and the common value are different values — every caller knows its transport, and
|
|
2228
|
+
// the one that does not is the one that must not compile.
|
|
2229
|
+
transport, rotateSys = false) {
|
|
1750
2230
|
const dir = authDir(cotalRoot()); // the broker config (server.conf) still lands under the FS auth dir
|
|
1751
2231
|
const store = workspaceSecretStore(cotalRoot());
|
|
1752
2232
|
let auth = await getSpaceAuth(store, space);
|
|
@@ -1754,7 +2234,58 @@ async function authSetup(storeDir, server, space, host = "127.0.0.1", user) {
|
|
|
1754
2234
|
auth = await createSpaceAuth(space);
|
|
1755
2235
|
await putSpaceAuth(store, auth); // strips the $SYS seed at rest, but leaves the in-memory `auth` intact …
|
|
1756
2236
|
await provisionMembershipCreds(auth, cotalRoot()); // … so the observer can still be minted here (fresh-space only)
|
|
1757
|
-
|
|
2237
|
+
// A fresh space's $SYS material was just minted from the seed that only exists in this branch, so
|
|
2238
|
+
// the ASK is already satisfied, so say so rather than rotating a one-second-old account, and never
|
|
2239
|
+
// report a rotation that did not happen.
|
|
2240
|
+
if (rotateSys)
|
|
2241
|
+
console.log(c.dim("• --rotate-sys: this space is new, so its $SYS creds were just minted - nothing to rotate"));
|
|
2242
|
+
}
|
|
2243
|
+
else if (rotateSys) {
|
|
2244
|
+
// Third and last of the stopped-broker checks, at the one point every rotation path converges on.
|
|
2245
|
+
// The other two live in `up`'s reachable-listener branch: this root's recorded mesh at the
|
|
2246
|
+
// requested address, and any UNIDENTIFIED listener there (which refuses rather than free-porting
|
|
2247
|
+
// around a broker that may be serving this root's own server.conf and store). This one reads the
|
|
2248
|
+
// root's ownership records for a broker at an address nobody probed. It reports what those
|
|
2249
|
+
// records say, not what the process table says (see its own comment for the residual), and fails
|
|
2250
|
+
// CLOSED, so an ambiguous record refuses exactly like a live one.
|
|
2251
|
+
await assertRootBrokerStopped(cotalRoot());
|
|
2252
|
+
// Fails loud: a caller that swallowed this would boot the broker on the RETIRED system account
|
|
2253
|
+
// while `doctor auth` reported the rotation as done.
|
|
2254
|
+
const rot = await rotateSystemCreds(cotalRoot(), space);
|
|
2255
|
+
auth = rot.auth; // the config below MUST be rendered from the successor, never the pre-rotation copy
|
|
2256
|
+
console.log(c.green(`✓ rotated the system account for "${space}"`) +
|
|
2257
|
+
c.dim(` (generation ${rot.gen}) - re-minted ${SYSTEM_CREDS_FILES.join(" + ")}${rot.expiresAt ? `, valid to ${new Date(rot.expiresAt * 1000).toISOString().slice(0, 10)}` : ""}`));
|
|
2258
|
+
// Say exactly what is true. The retirement is CONFIG-LOAD-BOUND: an old cred dies against any
|
|
2259
|
+
// broker that loads the successor config, which is every broker started from this root from here
|
|
2260
|
+
// on, but a stale nats-server still holding the pre-rotation config in memory would keep
|
|
2261
|
+
// honoring it, so "now dead" without that qualifier oversells the guarantee.
|
|
2262
|
+
console.log(c.dim(" the data account, every agent cred and the JetStream store are untouched. The OLD $SYS creds are refused by any broker that loads this config; a stale broker still running the previous config would still honor them, so stop those first."));
|
|
2263
|
+
// A full backup binds to the trust chain it was taken against: `rootChainCommitment` hashes the
|
|
2264
|
+
// operator JWT and the system account, both of which just changed, so `cotal up --restore`
|
|
2265
|
+
// refuses every full artifact taken before this moment. That is correct (it is a different trust
|
|
2266
|
+
// root), but it is only obvious to someone who has read the fingerprint code, and rotation is
|
|
2267
|
+
// now a routine 30-day act rather than a once-per-space event. Say it at the moment it becomes
|
|
2268
|
+
// true, not in a doc the operator reads after the restore has already failed.
|
|
2269
|
+
console.log(c.dim(" NOTE: full backups taken before this rotation can no longer be restored (they are bound to the retired trust chain) - take a fresh `cotal backup` once the mesh is up."));
|
|
2270
|
+
}
|
|
2271
|
+
// The $SYS creds must be signed by the system account THIS boot is about to put in `server.conf`.
|
|
2272
|
+
// A rotation that committed the trust record and then died leaves them stale, unexpired, and
|
|
2273
|
+
// broker-dead and, crash-before-either-write, stale in a way that no comparison between the two
|
|
2274
|
+
// FILES can see (they agree with each other; they just disagree with the record). This is the one
|
|
2275
|
+
// place holding both, on the one path that renders the config, so it is where the split is caught.
|
|
2276
|
+
//
|
|
2277
|
+
// REFUSE, do not warn. A warning is what the `--detach` path turns into an unread log line under a
|
|
2278
|
+
// green "✓ running in the background", which is the false success this whole change exists to
|
|
2279
|
+
// remove. And what stays broken is not only the display feed: live connection EVICTION rides the
|
|
2280
|
+
// same pair, so booting here would silently downgrade revocation to deny-new for the life of the
|
|
2281
|
+
// mesh. The repo's posture is to throw rather than degrade, and the recovery is one command that
|
|
2282
|
+
// this message names.
|
|
2283
|
+
const stale = staleSystemCreds(cotalRoot(), auth.sys.pub);
|
|
2284
|
+
if (stale.length)
|
|
2285
|
+
throw new Error(`${stale.map((x) => `${x.file} (signed by ${x.iss ? `${x.iss.slice(0, 12)}…` : "an unreadable issuer"})`).join(", ")} ` +
|
|
2286
|
+
`${stale.length === 1 ? "is" : "are"} not signed by this space's system account (${auth.sys.pub.slice(0, 12)}…) - ` +
|
|
2287
|
+
"an interrupted rotation left the $SYS creds behind the trust record, so the broker would deny them and live eviction + the membership feed would stay down. " +
|
|
2288
|
+
"Re-run the rotation to land a complete generation: `cotal up --rotate-sys`");
|
|
1758
2289
|
const stateDir = userAuthStateDir(cotalRoot(), space); // the provider's space-scoped state dir
|
|
1759
2290
|
if (!user && hasUserAuthState(cotalRoot(), space)) {
|
|
1760
2291
|
throw new Error(`space "${space}" has user auth enabled (state under ${stateDir}) - start it with \`cotal up --user-auth\`, or remove that directory deliberately to disable user auth (existing logins/grants die with it)`);
|
|
@@ -1780,13 +2311,73 @@ async function authSetup(storeDir, server, space, host = "127.0.0.1", user) {
|
|
|
1780
2311
|
}
|
|
1781
2312
|
}
|
|
1782
2313
|
const port = Number(new URL(server).port) || 4222;
|
|
2314
|
+
// P2 item 6: allocate the broker's loopback WebSocket listener port — the console page becomes a
|
|
2315
|
+
// mesh §13.6 session client over it (a NEW same-host attack surface, localhost-bound, no TLS). The
|
|
2316
|
+
// manager's establisher builds its wsUrl from this; threaded to `supervise --ws-port`.
|
|
2317
|
+
const wsPort = await freePort(host);
|
|
1783
2318
|
const confPath = resolve(dir, "server.conf");
|
|
1784
|
-
writeFileSync(confPath, serverConfig(auth, [auth], { port, storeDir, host, ...(prepared ? { extraAccounts: prepared.extraAccounts } : {}) }));
|
|
2319
|
+
writeFileSync(confPath, serverConfig(auth, [auth], { transport, port, storeDir, host, wsPort, wsHost: host, ...(prepared ? { extraAccounts: prepared.extraAccounts } : {}) }));
|
|
1785
2320
|
// Ephemeral setup cred: used only to probe reachability, pre-create the space streams/buckets
|
|
1786
2321
|
// (setupSpaceStreams) and seed the channel registry (seedChannelRegistry) — all within the
|
|
1787
2322
|
// enumerated `provisioner` scope. No broad `manager` residual for the up path.
|
|
1788
2323
|
const creds = await mintCreds(auth, newIdentity(), "provisioner");
|
|
1789
|
-
return { confPath, creds, ...(prepared ? { prepared, stateDir } : {}) };
|
|
2324
|
+
return { confPath, creds, wsPort, ...(prepared ? { prepared, stateDir } : {}) };
|
|
2325
|
+
}
|
|
2326
|
+
/**
|
|
2327
|
+
* Refuse the rotation if this root's own bookkeeping still shows a broker running. This is a
|
|
2328
|
+
* BEST-EFFORT check over Cotal-managed ownership records, NOT a proof that no process is serving this
|
|
2329
|
+
* root, and the difference matters: a survivor keeps honoring the retired account from memory, and a
|
|
2330
|
+
* second broker on the successor opens that survivor's JetStream store underneath it.
|
|
2331
|
+
*
|
|
2332
|
+
* Two records, because either alone has a blind spot. The PID FILE catches a broker this root started
|
|
2333
|
+
* whose registry row was lost. The REGISTRY sweep catches one recorded for this root at some OTHER
|
|
2334
|
+
* address, which no probe of the requested URL would reach. Both fail CLOSED: an unreadable or
|
|
2335
|
+
* malformed pid file refuses exactly like a live one, since "cannot tell" and "still running" have
|
|
2336
|
+
* the same consequence.
|
|
2337
|
+
*
|
|
2338
|
+
* WHAT IT CANNOT SEE, stated because an earlier version of this comment claimed otherwise: both
|
|
2339
|
+
* records are mutable, and neither is written by a broker started outside `cotal up`. Delete both
|
|
2340
|
+
* while the process lives, or run `nats-server -c <root>/.cotal/auth/server.conf` by hand, and this
|
|
2341
|
+
* returns success having probed nothing. The requested address is covered separately (an unidentified
|
|
2342
|
+
* listener there refuses the rotation rather than moving to a free port), which leaves a hand-started
|
|
2343
|
+
* broker on a DIFFERENT port as the honest residual. Closing that needs something a survivor holds
|
|
2344
|
+
* and cannot delete, an exclusive store lock, which does not exist today; until it does, do not run
|
|
2345
|
+
* `nats-server` against this root's config outside `cotal up`.
|
|
2346
|
+
*
|
|
2347
|
+
* Not a general `up` guard: an ordinary boot adopting or replacing a listener is a supported flow with
|
|
2348
|
+
* its own claim machinery. This is specifically the precondition for retiring an authority.
|
|
2349
|
+
*/
|
|
2350
|
+
async function assertRootBrokerStopped(root) {
|
|
2351
|
+
const pidPath = join(root, ".cotal", "nats.pid");
|
|
2352
|
+
if (existsSync(pidPath)) {
|
|
2353
|
+
let raw;
|
|
2354
|
+
try {
|
|
2355
|
+
raw = readFileSync(pidPath, "utf8").trim();
|
|
2356
|
+
}
|
|
2357
|
+
catch (e) {
|
|
2358
|
+
throw new Error(`--rotate-sys: ${pidPath} exists but cannot be read (${e.message}) - refusing to rotate while a broker for this root may still be running; \`cotal down\` first`);
|
|
2359
|
+
}
|
|
2360
|
+
const pid = Number(raw);
|
|
2361
|
+
if (!Number.isInteger(pid) || pid <= 0)
|
|
2362
|
+
throw new Error(`--rotate-sys: ${pidPath} does not hold a pid (${JSON.stringify(raw)}) - refusing to rotate while a broker for this root may still be running; \`cotal down\` first`);
|
|
2363
|
+
let live;
|
|
2364
|
+
try {
|
|
2365
|
+
process.kill(pid, 0); // signal 0 is a liveness probe, it signals nothing
|
|
2366
|
+
live = true;
|
|
2367
|
+
}
|
|
2368
|
+
catch (e) {
|
|
2369
|
+
// EPERM means the process EXISTS and is not ours to signal, i.e. alive for this purpose.
|
|
2370
|
+
live = e.code === "EPERM";
|
|
2371
|
+
}
|
|
2372
|
+
if (live)
|
|
2373
|
+
throw new Error(`--rotate-sys: this root's broker is still running (pid ${pid}) - it would keep serving the retired system account, and a second broker would share its JetStream store. Stop it first: \`cotal down\``);
|
|
2374
|
+
}
|
|
2375
|
+
// A broker recorded for this root at ANY address, still answering. The rotate refusal earlier in
|
|
2376
|
+
// `up` only sees the URL this invocation asked for; this sees the ones it did not.
|
|
2377
|
+
for (const m of meshesForRoot(root)) {
|
|
2378
|
+
if (await isReachable(m.server))
|
|
2379
|
+
throw new Error(`--rotate-sys: mesh "${m.space}" for this root is still reachable at ${m.server} - stop it first (\`cotal down\`), then rotate`);
|
|
2380
|
+
}
|
|
1790
2381
|
}
|
|
1791
2382
|
/** Mint the two scoped creds the delivery daemon's membership feed loads (broker-sourced graph
|
|
1792
2383
|
* membership), at the FRESH `cotal up` while the in-memory `$SYS` signing seed still exists:
|
|
@@ -1815,9 +2406,9 @@ async function provisionMembershipCreds(auth, root) {
|
|
|
1815
2406
|
// deny-new-only (durable reauth) — surfaced loudly by the removal path, never silent.
|
|
1816
2407
|
const evictor = await mintConnectionEvictorCreds(auth, newIdentity());
|
|
1817
2408
|
mkSecretDir(cotalPath()); // harden .cotal/ before the creds land (born under a private ACL, no race)
|
|
1818
|
-
writeSecretFile(cotalPath(
|
|
2409
|
+
writeSecretFile(cotalPath(SYSTEM_CREDS_FILES[0]), observer);
|
|
1819
2410
|
await workspaceSecretStore(root).put(MEMBERSHIP_RW_CREDS_KEY, rw); // migrated kind: through the seam (0600 FS put)
|
|
1820
|
-
writeSecretFile(cotalPath(
|
|
2411
|
+
writeSecretFile(cotalPath(SYSTEM_CREDS_FILES[1]), evictor);
|
|
1821
2412
|
writeSecretFile(cotalPath("membership.json"), JSON.stringify({ accountId: auth.account.pub }));
|
|
1822
2413
|
}
|
|
1823
2414
|
catch (e) {
|