@cotal-ai/cli 0.11.6 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/dist/commands/backup.d.ts +13 -0
  2. package/dist/commands/backup.d.ts.map +1 -0
  3. package/dist/commands/backup.js +443 -0
  4. package/dist/commands/backup.js.map +1 -0
  5. package/dist/commands/channels.d.ts +10 -0
  6. package/dist/commands/channels.d.ts.map +1 -1
  7. package/dist/commands/channels.js.map +1 -1
  8. package/dist/commands/clean.d.ts +12 -2
  9. package/dist/commands/clean.d.ts.map +1 -1
  10. package/dist/commands/clean.js +161 -28
  11. package/dist/commands/clean.js.map +1 -1
  12. package/dist/commands/down.d.ts +5 -0
  13. package/dist/commands/down.d.ts.map +1 -1
  14. package/dist/commands/down.js +323 -5
  15. package/dist/commands/down.js.map +1 -1
  16. package/dist/commands/spawn.d.ts +4 -0
  17. package/dist/commands/spawn.d.ts.map +1 -1
  18. package/dist/commands/spawn.js +43 -23
  19. package/dist/commands/spawn.js.map +1 -1
  20. package/dist/commands/status.d.ts.map +1 -1
  21. package/dist/commands/status.js +3 -2
  22. package/dist/commands/status.js.map +1 -1
  23. package/dist/commands/up.d.ts +12 -0
  24. package/dist/commands/up.d.ts.map +1 -1
  25. package/dist/commands/up.js +1149 -193
  26. package/dist/commands/up.js.map +1 -1
  27. package/dist/index.d.ts.map +1 -1
  28. package/dist/index.js +19 -4
  29. package/dist/index.js.map +1 -1
  30. package/dist/lib/backup-artifact.d.ts +50 -0
  31. package/dist/lib/backup-artifact.d.ts.map +1 -0
  32. package/dist/lib/backup-artifact.js +290 -0
  33. package/dist/lib/backup-artifact.js.map +1 -0
  34. package/dist/lib/delivery-proc.d.ts +7 -2
  35. package/dist/lib/delivery-proc.d.ts.map +1 -1
  36. package/dist/lib/delivery-proc.js +29 -19
  37. package/dist/lib/delivery-proc.js.map +1 -1
  38. package/dist/lib/endpoint-cut.d.ts +5 -0
  39. package/dist/lib/endpoint-cut.d.ts.map +1 -0
  40. package/dist/lib/endpoint-cut.js +53 -0
  41. package/dist/lib/endpoint-cut.js.map +1 -0
  42. package/dist/lib/isolated-broker.d.ts +70 -0
  43. package/dist/lib/isolated-broker.d.ts.map +1 -0
  44. package/dist/lib/isolated-broker.js +427 -0
  45. package/dist/lib/isolated-broker.js.map +1 -0
  46. package/dist/lib/maintenance-files.d.ts +10 -0
  47. package/dist/lib/maintenance-files.d.ts.map +1 -0
  48. package/dist/lib/maintenance-files.js +60 -0
  49. package/dist/lib/maintenance-files.js.map +1 -0
  50. package/dist/lib/manager-proc.d.ts +4 -0
  51. package/dist/lib/manager-proc.d.ts.map +1 -1
  52. package/dist/lib/manager-proc.js +2 -0
  53. package/dist/lib/manager-proc.js.map +1 -1
  54. package/dist/lib/restore.d.ts +52 -0
  55. package/dist/lib/restore.d.ts.map +1 -0
  56. package/dist/lib/restore.js +797 -0
  57. package/dist/lib/restore.js.map +1 -0
  58. package/package.json +7 -4
@@ -1,9 +1,12 @@
1
1
  import { spawn } from "node:child_process";
2
- import { createServer } from "node:net";
3
- import { mkdirSync, writeFileSync, readFileSync, existsSync, openSync, statSync, readSync, closeSync, } from "node:fs";
2
+ import { randomUUID } from "node:crypto";
3
+ import { createConnection, createServer } from "node:net";
4
+ import { hostname } from "node:os";
5
+ import { mkdirSync, writeFileSync, readFileSync, existsSync, openSync, statSync, readSync, closeSync, lstatSync, rmSync, } from "node:fs";
4
6
  import { join, resolve } from "node:path";
5
- import { isReachable, DEFAULT_SERVER, createSpaceAuth, serverConfig, mintCreds, mintConnectionEvictorCreds, mintMembershipObserverCreds, newIdentity, setupSpaceStreams, seedChannelRegistry, ensureDefaultDeliveryClass, mkSecretDir, writeSecretFile, } from "@cotal-ai/core";
6
- import { assertUserAuthInfo, authDir, loadSpaceAuth, saveSpaceAuth, clearCurrent, findMesh, getCurrent, loadMeshes, recordMesh, removeMesh, setCurrent, userAuthStateDir, } from "@cotal-ai/workspace";
7
+ import { isReachable, DEFAULT_SERVER, CONTROL_ADMIN, createSpaceAuth, serverConfig, mintCreds, mintConnectionEvictorCreds, mintMembershipObserverCreds, newIdentity, setupSpaceStreams, standaloneConnectOpts, seedChannelRegistry, ensureDefaultDeliveryClass, mkSecretDir, writeSecretFile, } from "@cotal-ai/core";
8
+ import { connect } from "@nats-io/transport-node";
9
+ import { assertUserAuthInfo, authDir, loadSpaceAuth, saveSpaceAuth, clearCurrent, findMesh, getCurrent, loadMeshes, recordMesh, removeMesh, setCurrent, userAuthStateDir, workspaceSecretStore, acquireMaintenanceLock, assertStoreIdentity, assessRestoreClaim, beginOrdinaryResume, bindOrdinaryResumeListener, consumeRetiredMaintenance, markOrdinaryResumeActive, markOrdinaryResumeDegraded, replaceDeadOrdinaryResumeListener, localProcessOwnerStatus, readMaintenanceJournal, readMaintenanceResumeDocument, readStoreIdentity, recordOrdinaryResumeManagerCommit, releaseMaintenanceLock, retireOrdinaryResume, sameStoreIdentity, } from "@cotal-ai/workspace";
7
10
  import { ensureAuthService, resolveAuthProvider, stopAuthService } from "../lib/auth-proc.js";
8
11
  import { resolveSpace } from "../lib/status.js";
9
12
  import { c } from "../ui.js";
@@ -18,6 +21,10 @@ import { renderUpPlan, renderInherited, renderWarnings } from "../lib/manifest/r
18
21
  import { failManifest } from "./topology.js";
19
22
  import { extensionNames, preflightRuntime } from "../ext-loader.js";
20
23
  import { completingFlagValue } from "../lib/completion.js";
24
+ import { askManager } from "../lib/control.js";
25
+ import { bindPreparedRestoreListener, isManagerCommitResult, isManagerFinalizeResult, isManagerCommittedRestore, markPreparedRestoreActive, markPreparedRestoreDegraded, prepareRestore, recordPreparedRestoreManagerCommit, rehydratePreparedRestore, replacePreparedDeadRestoreListener, } from "../lib/restore.js";
26
+ const pendingRestores = new Map();
27
+ const pendingOrdinaryResumes = new Map();
21
28
  /** `cotal up` flags — colocated with the command (like `spawnFlags`) so its completion can read them. */
22
29
  export const upFlags = [
23
30
  { name: "server", type: "string", value: "<url>", description: "listen URL override" },
@@ -25,6 +32,9 @@ export const upFlags = [
25
32
  { name: "space", type: "string", value: "<s>", description: "space name (default: the folder's)" },
26
33
  { name: "store-dir", type: "string", value: "<dir>", description: "JetStream store directory" },
27
34
  { name: "channels", type: "string", value: "<path>", description: "channel-registry seed file (JSON; default .cotal/channels.json)" },
35
+ { name: "restore", type: "string", value: "<dir>", description: "restore an offline backup before exposing the normal listener" },
36
+ { name: "restore-only", type: "string", value: "<registry>", description: "restore only the registry component" },
37
+ { name: "accept-missing-source", type: "boolean", description: "explicit disaster consent when the inode-bound preserved source is absent" },
28
38
  { name: "open", type: "boolean", description: "unauthenticated dev mesh (no JWT/ACLs)" },
29
39
  { name: "user-auth", type: "boolean", description: "per-USER auth: login + bearer through the space's auth service" },
30
40
  { name: "idp", type: "string", value: "<url>", description: "with --user-auth: the IdP auth base URL to pin (first enable)" },
@@ -41,38 +51,291 @@ export function upComplete(argv) {
41
51
  const flag = completingFlagValue(argv, upFlags);
42
52
  if (flag?.name === "runtime")
43
53
  return { items: ["pty", ...extensionNames("runtime")].filter((v, i, a) => a.indexOf(v) === i).map((value) => ({ value })), directive: "nofiles" };
54
+ if (flag?.name === "restore-only")
55
+ return { items: [{ value: "registry" }], directive: "nofiles" };
56
+ if (flag?.name === "restore")
57
+ return { items: [], directive: "default" };
44
58
  const items = upFlags.map((f) => ({ value: `--${f.name}`, description: f.description }));
45
59
  return { items, directive: items.length ? "nofiles" : "default" };
46
60
  }
47
61
  export async function up(args) {
48
62
  const values = args.values;
63
+ if (values.restore) {
64
+ if (values.file || values.channels)
65
+ throw new Error("--restore cannot be combined with --file/-f or --channels");
66
+ const prepared = await prepareRestore(cotalRoot(), values);
67
+ pendingRestores.set(prepared.attemptId, prepared);
68
+ const next = {
69
+ ...values,
70
+ restore: undefined,
71
+ "restore-only": undefined,
72
+ "accept-missing-source": undefined,
73
+ __restoreAttempt: prepared.attemptId,
74
+ space: values.space ?? prepared.space,
75
+ server: values.server ?? prepared.server,
76
+ host: values.host ?? prepared.host,
77
+ runtime: values.runtime ?? prepared.runtime,
78
+ open: prepared.mode === "open",
79
+ "user-auth": prepared.mode === "user",
80
+ };
81
+ try {
82
+ await up({ ...args, values: next });
83
+ }
84
+ catch (error) {
85
+ pendingRestores.delete(prepared.attemptId);
86
+ markPreparedRestoreDegraded(prepared.root, prepared.attemptId, error.message);
87
+ throw error;
88
+ }
89
+ return;
90
+ }
91
+ if (values["restore-only"] || values["accept-missing-source"])
92
+ throw new Error("--restore-only and --accept-missing-source require --restore <dir>");
93
+ if (!values.__restoreAttempt && !values.__ordinaryResumeAttempt && !values.file) {
94
+ const root = cotalRoot();
95
+ const lock = acquireMaintenanceLock(root);
96
+ let pending;
97
+ let recoveredRestore;
98
+ try {
99
+ const journal = readMaintenanceJournal(root);
100
+ if (journal?.state === "restore-ready") {
101
+ // Ordinary up NEVER rolls back an in-progress restore: a live attempt's target may still be
102
+ // written by its isolated broker. Refuse with the exact recovery recourse instead.
103
+ const attempt = journal.restore.attemptId;
104
+ const assessment = assessRestoreClaim(journal);
105
+ if (assessment === "live")
106
+ throw new Error(`cotal up is refused: restore attempt ${attempt} is in progress (claim live until ${journal.claim.deadline}); wait for it to complete or become provably stale`);
107
+ if (assessment === "ambiguous")
108
+ throw new Error(`cotal up is refused: restore attempt ${attempt} owners cannot be proven dead; inspect the recorded coordinator, watchdog, and broker processes before recovery`);
109
+ throw new Error(`cotal up is refused: stale restore attempt ${attempt} holds the store; roll it back with \`cotal clean restore-attempt --attempt ${attempt} --force\`, or recover it with \`cotal up --restore\``);
110
+ }
111
+ if (journal && (journal.state === "commit-intent" || journal.state === "manager-committed" || journal.state === "degraded")) {
112
+ recoveredRestore = rehydratePreparedRestore(root, journal);
113
+ pendingRestores.set(recoveredRestore.attemptId, recoveredRestore);
114
+ }
115
+ else if (journal?.state === "active") {
116
+ if (!isManagerCommittedRestore(journal))
117
+ throw new Error(`restore attempt ${journal.restore.attemptId} is active without durable manager commit evidence`);
118
+ if (!journal.listenerProof)
119
+ throw new Error(`restore attempt ${journal.restore.attemptId} is active without a bound listener proof`);
120
+ const status = localProcessOwnerStatus(journal.listenerProof.processOwner);
121
+ if (status === "unknown")
122
+ throw new Error(`restore attempt ${journal.restore.attemptId} active listener ownership is ambiguous; refusing ordinary startup`);
123
+ if (status === "alive") {
124
+ recoveredRestore = rehydratePreparedRestore(root, journal);
125
+ pendingRestores.set(recoveredRestore.attemptId, recoveredRestore);
126
+ }
127
+ }
128
+ else if (journal?.state === "ready") {
129
+ const resume = readMaintenanceResumeDocument(root, journal.resume);
130
+ const attemptId = `resume-${randomUUID()}`;
131
+ const launch = resume.launch;
132
+ const agents = resume.inventory.agents ?? [];
133
+ const runtimes = [...new Set(agents.map((agent) => agent.launch?.runtime).filter((value) => typeof value === "string"))];
134
+ if (runtimes.length > 1)
135
+ throw new Error(`resume inventory requires multiple runtimes: ${runtimes.join(", ")}`);
136
+ const resumeServer = values.server ?? (typeof launch.server === "string" ? launch.server : DEFAULT_SERVER);
137
+ const requestedStore = values["store-dir"] ?? (typeof launch.storeDir === "string" ? launch.storeDir : journal.source.path);
138
+ // Ordinary up resumes the exact preserved source: a different valid store passed via
139
+ // --store-dir must not consume this maintenance cut under the recorded space's trust.
140
+ if (!sameStoreIdentity(readStoreIdentity(resolve(requestedStore)), journal.source))
141
+ throw new Error(`--store-dir ${requestedStore} is not the preserved source store; ordinary up resumes exactly ${journal.source.path}`);
142
+ // Journal the CANONICAL identity-checked path, never the caller's spelling: a relative or
143
+ // symlinked spelling re-resolved later (other cwd, retargeted link) could open another store.
144
+ const resumeStore = journal.source.path;
145
+ if (values.runtime && runtimes[0] && values.runtime !== runtimes[0])
146
+ throw new Error(`--runtime ${values.runtime} contradicts the preserved agent runtime ${runtimes[0]}; omit it to resume the same principals`);
147
+ const resumeRuntime = values.runtime ?? runtimes[0] ?? "pty";
148
+ const serverNonce = randomUUID().replaceAll("-", "");
149
+ const serverName = `${attemptId}-${serverNonce}`;
150
+ beginOrdinaryResume(lock, {
151
+ attemptId,
152
+ launch: {
153
+ server: resumeServer,
154
+ storeDir: resumeStore,
155
+ runtime: resumeRuntime,
156
+ detached: Boolean(values.detach),
157
+ serverName,
158
+ serverNonce,
159
+ },
160
+ });
161
+ pending = {
162
+ root,
163
+ attemptId,
164
+ space: journal.space,
165
+ mode: journal.mode,
166
+ server: resumeServer,
167
+ storeDir: resumeStore,
168
+ runtime: resumeRuntime,
169
+ detached: Boolean(values.detach),
170
+ inventory: resume.inventory,
171
+ journalState: "resume-intent",
172
+ serverName,
173
+ serverNonce,
174
+ };
175
+ pendingOrdinaryResumes.set(attemptId, pending);
176
+ }
177
+ else if (journal && (journal.state === "resume-intent" || journal.state === "resume-active" || journal.state === "resume-committed" || journal.state === "resume-degraded")) {
178
+ const resume = readMaintenanceResumeDocument(root, journal.resume);
179
+ const launch = journal.ordinaryResume.launch;
180
+ const attemptId = journal.ordinaryResume.attemptId;
181
+ // Recovery re-asserts the source identity rather than trusting the journaled spelling.
182
+ assertStoreIdentity(journal.source);
183
+ // The bound listener decides re-entry: a live exact listener is ADOPTED, a provably dead
184
+ // one is durably retired before a fresh spawn, ambiguity refuses, and a manager-committed
185
+ // resume never replaces the listener its durable token is bound to.
186
+ let adoptProof;
187
+ let recoveredProof = journal.listenerProof;
188
+ let recoveredState = journal.state;
189
+ if (recoveredProof) {
190
+ const status = localProcessOwnerStatus(recoveredProof.processOwner);
191
+ if (status === "unknown")
192
+ throw new Error(`resume attempt ${attemptId} listener ownership is ambiguous; refusing replacement`);
193
+ if (status === "alive") {
194
+ adoptProof = recoveredProof;
195
+ }
196
+ else if (journal.state === "resume-committed") {
197
+ throw new Error(`resume attempt ${attemptId} is manager-committed but its bound listener is dead; preserving the commit token and retained suppression`);
198
+ }
199
+ else {
200
+ if (journal.state === "resume-active") {
201
+ markOrdinaryResumeDegraded(lock, "bound resume listener died after activation", [{
202
+ action: "repair",
203
+ description: "Retire the dead listener and re-run the same-principal activation.",
204
+ paths: [journal.source.path],
205
+ }]);
206
+ recoveredState = "resume-degraded";
207
+ }
208
+ replaceDeadOrdinaryResumeListener(lock, recoveredProof);
209
+ recoveredProof = undefined;
210
+ }
211
+ }
212
+ const freshNonce = randomUUID().replaceAll("-", "");
213
+ const serverNonce = adoptProof?.serverNonce ??
214
+ (recoveredProof || typeof launch.serverNonce !== "string" || journal.listenerReplacements?.length
215
+ ? freshNonce
216
+ : launch.serverNonce);
217
+ const serverName = adoptProof?.serverName ?? `${attemptId}-${serverNonce}`;
218
+ pending = {
219
+ root,
220
+ attemptId,
221
+ space: journal.space,
222
+ mode: journal.mode,
223
+ server: typeof launch.server === "string" ? launch.server : DEFAULT_SERVER,
224
+ storeDir: journal.source.path,
225
+ runtime: typeof launch.runtime === "string" ? launch.runtime : "pty",
226
+ detached: launch.detached === true,
227
+ inventory: resume.inventory,
228
+ journalState: recoveredState,
229
+ managerCommit: journal.state === "resume-committed" ? journal.managerCommit : undefined,
230
+ serverName,
231
+ serverNonce,
232
+ ...(adoptProof ? { adoptProof } : {}),
233
+ };
234
+ pendingOrdinaryResumes.set(attemptId, pending);
235
+ }
236
+ else if (journal?.state === "resume-retired") {
237
+ consumeRetiredMaintenance(lock);
238
+ }
239
+ else if (journal) {
240
+ throw new Error(`cotal up is refused while maintenance state is ${journal.state}; follow the recorded recovery`);
241
+ }
242
+ }
243
+ finally {
244
+ releaseMaintenanceLock(lock);
245
+ }
246
+ if (recoveredRestore) {
247
+ try {
248
+ await up({
249
+ ...args,
250
+ values: {
251
+ ...values,
252
+ __restoreAttempt: recoveredRestore.attemptId,
253
+ space: recoveredRestore.space,
254
+ server: recoveredRestore.server,
255
+ host: recoveredRestore.host,
256
+ "store-dir": recoveredRestore.targetPath,
257
+ runtime: recoveredRestore.runtime,
258
+ detach: recoveredRestore.detached,
259
+ open: recoveredRestore.mode === "open",
260
+ "user-auth": recoveredRestore.mode === "user",
261
+ },
262
+ });
263
+ }
264
+ catch (error) {
265
+ markPendingResumeDegraded(recoveredRestore.attemptId, error instanceof Error ? error.message : String(error));
266
+ throw error;
267
+ }
268
+ return;
269
+ }
270
+ if (pending) {
271
+ try {
272
+ await up({
273
+ ...args,
274
+ values: {
275
+ ...values,
276
+ __ordinaryResumeAttempt: pending.attemptId,
277
+ space: pending.space,
278
+ server: pending.server,
279
+ "store-dir": pending.storeDir,
280
+ runtime: pending.runtime,
281
+ detach: pending.detached,
282
+ open: pending.mode === "open",
283
+ "user-auth": pending.mode === "user",
284
+ },
285
+ });
286
+ }
287
+ catch (error) {
288
+ markPendingResumeDegraded(pending.attemptId, error instanceof Error ? error.message : String(error));
289
+ throw error;
290
+ }
291
+ return;
292
+ }
293
+ }
49
294
  const wantUser = Boolean(values["user-auth"]);
50
295
  // The auth-mode flags contradict each other loudly, never silently (a user-auth space quietly
51
296
  // started open would run agents on the wrong identity plane — the exact failure per-user-auth
52
297
  // exists to prevent).
53
298
  if (wantUser && values.open) {
54
- console.error(c.red("--user-auth and --open contradict - a user-auth space cannot run unauthenticated"));
55
- process.exit(1);
299
+ throw new Error("--user-auth and --open contradict - a user-auth space cannot run unauthenticated");
56
300
  }
57
301
  if (values.idp && !wantUser && !values.file) {
58
- console.error(c.red('--idp is for user-auth spaces; pair it with --user-auth, or set broker.auth: "user" in a manifest'));
59
- process.exit(1);
302
+ throw new Error('--idp is for user-auth spaces; pair it with --user-auth, or set broker.auth: "user" in a manifest');
60
303
  }
61
304
  // `up -f cotal.yaml` is a distinct path: bring up a FRESH mesh described by a manifest (broker +
62
305
  // channels + booted agents). It owns the whole space; deploying onto a RUNNING mesh is `spawn -f`.
63
306
  // CLI flags override the manifest (flag > manifest > default) so the same file runs at a different
64
307
  // port / runtime / space / auth without editing it.
65
308
  if (values.file) {
66
- await upManifest(values.file, {
67
- dryRun: Boolean(values["dry-run"]),
68
- server: values.server,
69
- host: values.host,
70
- space: values.space,
71
- runtime: values.runtime,
72
- open: values.open,
73
- userAuth: wantUser,
74
- idp: values.idp,
75
- });
309
+ if (values["dry-run"]) {
310
+ await upManifest(values.file, {
311
+ dryRun: true,
312
+ server: values.server,
313
+ host: values.host,
314
+ space: values.space,
315
+ runtime: values.runtime,
316
+ open: values.open,
317
+ userAuth: wantUser,
318
+ idp: values.idp,
319
+ });
320
+ return;
321
+ }
322
+ const lock = acquireMaintenanceLock(cotalRoot());
323
+ try {
324
+ assertOrdinaryUpAllowed(cotalRoot());
325
+ await upManifest(values.file, {
326
+ dryRun: Boolean(values["dry-run"]),
327
+ server: values.server,
328
+ host: values.host,
329
+ space: values.space,
330
+ runtime: values.runtime,
331
+ open: values.open,
332
+ userAuth: wantUser,
333
+ idp: values.idp,
334
+ });
335
+ }
336
+ finally {
337
+ releaseMaintenanceLock(lock);
338
+ }
76
339
  return;
77
340
  }
78
341
  // `--runtime <name>` selects the backend the mesh's manager spawns agents through. The `-f` path
@@ -82,182 +345,852 @@ export async function up(args) {
82
345
  // recourse) instead of a silent fallback in a detached child. No fallbacks. (`pty`/unset: no-op.)
83
346
  if (values.runtime)
84
347
  await preflightRuntime(values.runtime);
85
- let server = values.server ?? DEFAULT_SERVER;
86
- const host = values.host ?? "127.0.0.1";
87
- if (await isReachable(server)) {
88
- const space = values.space ?? resolveSpace(process.cwd());
89
- const root = cotalRoot();
90
- // A broker is already on this port. Same root means "this project is already up" unless the
91
- // operator explicitly asked for a second space in the same `.cotal/` root (unsupported today: pid,
92
- // auth, and logs are root-scoped). Different root / unrecorded broker on the implicit default port
93
- // gets a fresh free port instead of making the user hunt for one.
94
- const held = loadMeshes().find((m) => m.server === server);
95
- if (held && held.root === root && (held.space === space || values.space === undefined)) {
96
- // A refresh of the SAME already-running mesh — its mode is fixed by how the live broker was
97
- // started. A flag asking for a DIFFERENT mode must fail loud (silently preserving the old
98
- // mode would hand the operator a mesh on the wrong identity plane); a bare refresh keeps the
99
- // held mode.
100
- const requested = wantUser ? "user" : values.open ? "open" : undefined;
101
- if (requested && requested !== held.mode) {
102
- const label = { auth: "static JWT auth", open: "no auth (--open)", user: "per-user auth" }[held.mode];
103
- console.error(c.red(`✗ mesh "${held.space}" is already running at ${server} with ${label} - a running broker can't change auth mode; \`cotal down\` it first, then \`cotal up ${wantUser ? "--user-auth" : "--open"}\``));
104
- process.exit(1);
348
+ const resumeAttempt = values.__restoreAttempt ?? values.__ordinaryResumeAttempt;
349
+ let startupLock = resumeAttempt ? undefined : acquireMaintenanceLock(cotalRoot());
350
+ const releaseStartupLock = () => {
351
+ if (!startupLock)
352
+ return;
353
+ releaseMaintenanceLock(startupLock);
354
+ startupLock = undefined;
355
+ };
356
+ try {
357
+ if (startupLock)
358
+ assertOrdinaryUpAllowed(cotalRoot(), values["store-dir"] ? resolve(values["store-dir"]) : cotalPath("nats"));
359
+ let server = values.server ?? DEFAULT_SERVER;
360
+ const host = values.host ?? "127.0.0.1";
361
+ const restoredAttempt = resumeAttempt ? pendingRestores.get(resumeAttempt) : undefined;
362
+ if (restoredAttempt?.reentry) {
363
+ if (!restoredAttempt.listenerProof)
364
+ throw new Error(`restore attempt ${resumeAttempt} re-entry has no bound listener proof; preserving recovery state`);
365
+ const ownerStatus = localProcessOwnerStatus(restoredAttempt.listenerProof.processOwner);
366
+ if (ownerStatus === "alive") {
367
+ await resumeProvenRestoreListener(restoredAttempt);
368
+ return;
105
369
  }
106
- console.log(c.green(`✓ mesh "${held.space}" already running at ${server}`));
107
- // USER MODE: re-upping IS the documented recovery for a dead auth service (the provider's
108
- // failure copy says "restart it with `cotal up`"), so a refresh must re-ensure the service —
109
- // never just reprint "already running" over a dead callout. No broker config is (re)written
110
- // here, so healing on a bare `cotal up` is safe: the mode can't drift, only the daemon heals.
111
- let userAuth = held.userAuth;
112
- if (held.mode === "user") {
113
- const auth = loadSpaceAuth(authDir(root));
114
- if (!auth) {
115
- console.error(c.red(`✗ mesh "${held.space}" is user-auth but this root has no trust material under ${authDir(root)} - \`cotal down\` it, restore or re-provision \`.cotal/auth\`, then \`cotal up --user-auth\``));
370
+ if (ownerStatus === "unknown")
371
+ throw new Error(`restore attempt ${resumeAttempt} listener ownership is ambiguous; refusing replacement`);
372
+ if (restoredAttempt.managerCommit)
373
+ throw new Error(`restore attempt ${resumeAttempt} is manager-committed but its bound listener is dead; preserving the commit token and retained suppression`);
374
+ if (await isReachable(restoredAttempt.server))
375
+ throw new Error(`restore attempt ${resumeAttempt} refuses the occupied foreign listener at ${restoredAttempt.server}`);
376
+ replacePreparedDeadRestoreListener(restoredAttempt);
377
+ }
378
+ const ordinaryAttempt = resumeAttempt ? pendingOrdinaryResumes.get(resumeAttempt) : undefined;
379
+ if (ordinaryAttempt?.adoptProof) {
380
+ // The recovered attempt's exact bound listener is alive: prove it over the wire and adopt it
381
+ // instead of spawning a competitor over the same store.
382
+ await resumeProvenOrdinaryListener(ordinaryAttempt);
383
+ return;
384
+ }
385
+ if (ordinaryAttempt?.journalState === "resume-committed")
386
+ throw new Error(`resume attempt ${resumeAttempt} is manager-committed but has no adoptable bound listener; preserving the commit token and retained suppression`);
387
+ const listenerReachable = await isReachable(server);
388
+ if (resumeAttempt && listenerReachable)
389
+ throw new Error(`resume attempt ${resumeAttempt} refuses the unproven occupied listener at ${server}`);
390
+ if (listenerReachable) {
391
+ const space = values.space ?? resolveSpace(process.cwd());
392
+ const root = cotalRoot();
393
+ // A broker is already on this port. Same root means "this project is already up" unless the
394
+ // operator explicitly asked for a second space in the same `.cotal/` root (unsupported today: pid,
395
+ // auth, and logs are root-scoped). Different root / unrecorded broker on the implicit default port
396
+ // gets a fresh free port instead of making the user hunt for one.
397
+ const held = loadMeshes().find((m) => m.server === server);
398
+ if (held && held.root === root && (held.space === space || values.space === undefined)) {
399
+ // A refresh of the SAME already-running mesh — its mode is fixed by how the live broker was
400
+ // started. A flag asking for a DIFFERENT mode must fail loud (silently preserving the old
401
+ // mode would hand the operator a mesh on the wrong identity plane); a bare refresh keeps the
402
+ // held mode.
403
+ const requested = wantUser ? "user" : values.open ? "open" : undefined;
404
+ if (requested && requested !== held.mode) {
405
+ const label = { auth: "static JWT auth", open: "no auth (--open)", user: "per-user auth" }[held.mode];
406
+ console.error(c.red(`✗ mesh "${held.space}" is already running at ${server} with ${label} - a running broker can't change auth mode; \`cotal down\` it first, then \`cotal up ${wantUser ? "--user-auth" : "--open"}\``));
116
407
  process.exit(1);
117
408
  }
118
- const stateDir = userAuthStateDir(root, held.space);
119
- let prepared;
120
- try {
121
- prepared = await resolveAuthProvider().prepareServer({
122
- space: held.space,
123
- operatorSeed: auth.operator.seed,
124
- account: { pub: auth.account.pub, signingSeed: auth.account.signingSeed },
125
- dir: stateDir,
126
- idpUrl: values.idp,
127
- });
409
+ console.log(c.green(`✓ mesh "${held.space}" already running at ${server}`));
410
+ // USER MODE: re-upping IS the documented recovery for a dead auth service (the provider's
411
+ // failure copy says "restart it with `cotal up`"), so a refresh must re-ensure the service —
412
+ // never just reprint "already running" over a dead callout. No broker config is (re)written
413
+ // here, so healing on a bare `cotal up` is safe: the mode can't drift, only the daemon heals.
414
+ let userAuth = held.userAuth;
415
+ if (held.mode === "user") {
416
+ const auth = loadSpaceAuth(authDir(root));
417
+ if (!auth) {
418
+ console.error(c.red(`✗ mesh "${held.space}" is user-auth but this root has no trust material under ${authDir(root)} - \`cotal down\` it, restore or re-provision \`.cotal/auth\`, then \`cotal up --user-auth\``));
419
+ process.exit(1);
420
+ }
421
+ const stateDir = userAuthStateDir(root, held.space);
422
+ let prepared;
423
+ try {
424
+ prepared = await resolveAuthProvider().prepareServer({
425
+ space: held.space,
426
+ operatorSeed: auth.operator.seed,
427
+ account: { pub: auth.account.pub, signingSeed: auth.account.signingSeed },
428
+ store: workspaceSecretStore(root),
429
+ dir: stateDir,
430
+ idpUrl: values.idp,
431
+ });
432
+ }
433
+ catch (e) {
434
+ console.error(c.red(`✗ ${e.message}`));
435
+ process.exit(1);
436
+ }
437
+ const svc = await startUserAuthService(held.space, server, { prepared, stateDir });
438
+ userAuth = svc.userAuth;
439
+ // The refresh IS the recovery command — a heal that didn't heal must not exit 0.
440
+ if (!svc.ok)
441
+ process.exitCode = 1;
128
442
  }
129
- catch (e) {
130
- console.error(c.red(`✗ ${e.message}`));
131
- process.exit(1);
443
+ // Auth/user meshes also need their resident renewal owner. A same-root refresh is the normal
444
+ // repair command after a stale/missing manager, so ensure the delivery daemon + manager before
445
+ // claiming the running mesh is healthy. Open meshes have no auth creds or delivery daemon.
446
+ // Re-ensure the control plane on a refresh. The manager is ensured for every mode that reaches
447
+ // here (a heal after a dead/missing manager adopts `--runtime`); the delivery daemon self-gates
448
+ // to auth mode inside `ensureControlPlane`. Open meshes normally skip this (a bare refresh has
449
+ // nothing to heal that must be touched), but a `--runtime` request must be honored there too,
450
+ // not silently dropped.
451
+ if (held.mode !== "open" || values.runtime) {
452
+ // Warn only when the manager is genuinely REUSED: a live delivery-aware (this-build) manager
453
+ // is kept as-is by `ensureManager`, so its runtime can't change. An old hosting manager (no
454
+ // delivery marker) is stopped and REPLACED by the ensure below carrying the requested runtime,
455
+ // so that's not a reuse - don't claim the runtime is fixed. A dead/absent manager is (re)started
456
+ // with it.
457
+ if (values.runtime && managerUp() && managerHasDeliveryMarker())
458
+ console.error(c.dim(`! manager already running for "${held.space}" - its runtime is fixed at start; \`cotal down\` then \`cotal up --runtime ${values.runtime}\` to change it`));
459
+ const controlPlane = await startDeliveryWithBroker(held.space, server, { runtime: values.runtime });
460
+ if (!controlPlane)
461
+ process.exitCode = 1;
132
462
  }
133
- const svc = await startUserAuthService(held.space, server, { prepared, stateDir });
134
- userAuth = svc.userAuth;
135
- // The refresh IS the recovery command — a heal that didn't heal must not exit 0.
136
- if (!svc.ok)
137
- process.exitCode = 1;
463
+ recordOurMesh({ space: held.space, server, root, mode: held.mode, ...(userAuth ? { userAuth } : {}), ts: new Date().toISOString() });
464
+ return;
465
+ }
466
+ const who = held ? `mesh "${held.space}" (${held.root})` : "a broker not started here";
467
+ if (values.server === undefined && (!held || held.root !== root)) {
468
+ const next = await serverWithFreePort(server, host);
469
+ console.log(c.dim(`${server} is already in use by ${who}; starting "${space}" at ${next} instead`));
470
+ server = next;
138
471
  }
139
- // Auth/user meshes also need their resident renewal owner. A same-root refresh is the normal
140
- // repair command after a stale/missing manager, so ensure the delivery daemon + manager before
141
- // claiming the running mesh is healthy. Open meshes have no auth creds or delivery daemon.
142
- // Re-ensure the control plane on a refresh. The manager is ensured for every mode that reaches
143
- // here (a heal after a dead/missing manager adopts `--runtime`); the delivery daemon self-gates
144
- // to auth mode inside `ensureControlPlane`. Open meshes normally skip this (a bare refresh has
145
- // nothing to heal that must be touched), but a `--runtime` request must be honored there too,
146
- // not silently dropped.
147
- if (held.mode !== "open" || values.runtime) {
148
- // Warn only when the manager is genuinely REUSED: a live delivery-aware (this-build) manager
149
- // is kept as-is by `ensureManager`, so its runtime can't change. An old hosting manager (no
150
- // delivery marker) is stopped and REPLACED by the ensure below carrying the requested runtime,
151
- // so that's not a reuse - don't claim the runtime is fixed. A dead/absent manager is (re)started
152
- // with it.
153
- if (values.runtime && managerUp() && managerHasDeliveryMarker())
154
- console.error(c.dim(`! manager already running for "${held.space}" - its runtime is fixed at start; \`cotal down\` then \`cotal up --runtime ${values.runtime}\` to change it`));
155
- const controlPlane = await startDeliveryWithBroker(held.space, server, { runtime: values.runtime });
156
- if (!controlPlane)
157
- process.exitCode = 1;
472
+ else {
473
+ console.error(c.red(`✗ ${server} is already in use by ${who} - to run "${space}" use \`--server nats://${host}:<port>\` with a free port`));
474
+ process.exit(1);
158
475
  }
159
- recordOurMesh({ space: held.space, server, root, mode: held.mode, ...(userAuth ? { userAuth } : {}), ts: new Date().toISOString() });
476
+ }
477
+ if (values.detach) {
478
+ const restored = resumeAttempt ? pendingRestores.get(resumeAttempt) : undefined;
479
+ const { pid, source, authService, controlPlane, delivery, manager } = await startMeshDetached({
480
+ server,
481
+ storeDir: values["store-dir"],
482
+ space: values.space,
483
+ open: values.open,
484
+ userAuth: wantUser ? { idpUrl: values.idp } : undefined,
485
+ channels: values.channels,
486
+ host,
487
+ runtime: values.runtime,
488
+ resumeAttempt,
489
+ resumeCommitToken: restored?.managerCommit?.durableCommitToken ?? ordinaryAttempt?.managerCommit?.durableCommitToken,
490
+ ...(restored ? {
491
+ boundListener: {
492
+ serverName: restored.serverName,
493
+ serverNonce: restored.serverNonce,
494
+ onSpawn: (pid, startedAt) => bindSpawnedRestoreListener(restored, pid, startedAt),
495
+ verify: async () => { await provePreparedRestoreListener(restored); },
496
+ },
497
+ } : ordinaryAttempt ? {
498
+ boundListener: {
499
+ serverName: ordinaryAttempt.serverName,
500
+ serverNonce: ordinaryAttempt.serverNonce,
501
+ onSpawn: (pid, startedAt) => bindSpawnedOrdinaryResumeListener(ordinaryAttempt, pid, startedAt),
502
+ verify: async () => { await verifySpawnedOrdinaryListener(ordinaryAttempt); },
503
+ },
504
+ } : {}),
505
+ skipPostStart: Boolean(resumeAttempt),
506
+ });
507
+ console.log(c.dim(`Started nats-server (${source}).`));
508
+ console.log(c.green(renderDetachedSummary({ pid, delivery, authService: wantUser && authService, manager })));
509
+ if (restored && process.env.COTAL_SMOKE_FAIL_AFTER_RESTORE_LISTENER_READY === "1")
510
+ throw new Error("smoke-injected failure after restore listener readiness");
511
+ // A user mesh whose auth service never became ready is recorded + running (a re-`cotal up`
512
+ // heals it), but this `up` did NOT deliver what was asked — automation must see that in the
513
+ // exit code, not only in the red line above.
514
+ if (!authService)
515
+ process.exitCode = 1;
516
+ await completeResumeActivation(resumeAttempt, controlPlane && authService, !authService ? "normal listener started but the user-auth service is unavailable" : "normal listener started but the control plane is degraded", server);
160
517
  return;
161
518
  }
162
- const who = held ? `mesh "${held.space}" (${held.root})` : "a broker not started here";
163
- if (values.server === undefined && (!held || held.root !== root)) {
164
- const next = await serverWithFreePort(server, host);
165
- console.log(c.dim(`${server} is already in use by ${who}; starting "${space}" at ${next} instead`));
166
- server = next;
519
+ const useAuth = !values.open;
520
+ const space = values.space ?? resolveSpace(process.cwd());
521
+ ensureRootForSpace(useAuth, space); // may pin the cwd as this space's root — before any cotalPath use
522
+ refuseOpenOverUserState(Boolean(values.open), space);
523
+ const storeDir = values["store-dir"] ? resolve(values["store-dir"]) : cotalPath("nats");
524
+ if (values.__ordinaryResumeAttempt) {
525
+ // Final identity re-assertion immediately before JetStream opens: the journaled canonical path
526
+ // must still be the exact preserved inode (no symlink retarget or replacement since intent).
527
+ const resumeJournal = readMaintenanceJournal(cotalRoot());
528
+ if (!resumeJournal || !("ordinaryResume" in resumeJournal))
529
+ throw new Error(`resume attempt ${values.__ordinaryResumeAttempt} lost its durable journal before the store open`);
530
+ if (!sameStoreIdentity(readStoreIdentity(storeDir), resumeJournal.source))
531
+ throw new Error(`resume store ${storeDir} no longer matches the preserved source identity; refusing to open JetStream over it`);
167
532
  }
168
- else {
169
- console.error(c.red(`✗ ${server} is already in use by ${who} - to run "${space}" use \`--server nats://${host}:<port>\` with a free port`));
170
- process.exit(1);
533
+ mkdirSync(storeDir, { recursive: true });
534
+ await claimSpace(space, server, cotalRoot());
535
+ const seedFile = loadChannelsFile(values.channels);
536
+ const setup = useAuth ? await authSetup(storeDir, server, space, host, wantUser ? { idpUrl: values.idp } : undefined) : undefined;
537
+ const port = Number(new URL(server).port) || 4222;
538
+ const restored = resumeAttempt ? pendingRestores.get(resumeAttempt) : undefined;
539
+ const natsArgs = [
540
+ ...(setup ? ["-c", setup.confPath] : ["-js", "-sd", storeDir, "-p", String(port), "-a", host]),
541
+ ...(restored ? ["--name", restored.serverName]
542
+ : ordinaryAttempt ? ["--name", ordinaryAttempt.serverName]
543
+ : []),
544
+ ];
545
+ const { bin, source } = await resolveNatsServer();
546
+ console.log(c.dim(`Starting nats-server (JetStream, ${useAuth ? "JWT auth" : "OPEN/no-auth"}, ${source}) - store: ${storeDir}, bind: ${host}`));
547
+ console.log(c.dim("Press Ctrl-C to stop.\n"));
548
+ const listenerStartedAt = new Date().toISOString();
549
+ const child = spawn(bin, natsArgs, { stdio: "inherit" });
550
+ let activationFinished = !resumeAttempt;
551
+ if (child.pid)
552
+ writeFileSync(cotalPath("nats.pid"), String(child.pid));
553
+ if (restored && process.env.COTAL_SMOKE_EXIT_AFTER_RESTORE_LISTENER_SPAWN === "1")
554
+ process.exit(87);
555
+ if (restored)
556
+ try {
557
+ bindSpawnedRestoreListener(restored, child.pid ?? 0, listenerStartedAt);
558
+ }
559
+ catch (error) {
560
+ await stopUnboundRestoreListener(child);
561
+ removeMatchingNatsPid(child.pid ?? 0);
562
+ throw error;
563
+ }
564
+ if (ordinaryAttempt)
565
+ try {
566
+ bindSpawnedOrdinaryResumeListener(ordinaryAttempt, child.pid ?? 0, listenerStartedAt);
567
+ }
568
+ catch (error) {
569
+ await stopUnboundRestoreListener(child);
570
+ removeMatchingNatsPid(child.pid ?? 0);
571
+ throw error;
572
+ }
573
+ releaseStartupLock();
574
+ child.on("error", (err) => {
575
+ console.error(c.red(`Failed to start nats-server: ${err.message}`));
576
+ if (!resumeAttempt)
577
+ process.exit(1);
578
+ });
579
+ // The control plane is coupled to the broker: stop the delivery daemon AND the detached manager
580
+ // (AND the space's user-auth service) when this `up` stops (Ctrl-C), so none outlives the broker
581
+ // it serves — a surviving manager would reconnect-loop invisibly against the dead (or the NEXT)
582
+ // broker (the documented orphan-supervisor failure mode). All kill by pidfile, symmetric; the
583
+ // auth service's pid is space-scoped so no other space's daemon can ever be hit.
584
+ // stopDelivery is async (its creds delete goes through the secret store); the rest of the teardown
585
+ // must run even if it fails — the failure is logged, never swallowed silently, and the daemon kill
586
+ // itself happens inside stopDelivery's finally. Order preserved: delivery, manager, auth, broker.
587
+ const stop = () => {
588
+ void stopDelivery()
589
+ .catch((e) => console.error(`! delivery teardown: ${e.message}`))
590
+ .then(() => {
591
+ stopManager();
592
+ stopAuthService(space);
593
+ child.kill("SIGTERM");
594
+ });
595
+ };
596
+ process.on("SIGINT", stop);
597
+ process.on("SIGTERM", stop);
598
+ // The broker is gone — drop it from the registry (and the `current` pointer if it was the default)
599
+ // so a later `cotal spawn` doesn't try to join a dead mesh.
600
+ child.on("exit", async (code) => {
601
+ rmSync(cotalPath("nats.pid"), { force: true });
602
+ // Logged, never silently swallowed; the daemon kill runs in stopDelivery's finally regardless.
603
+ await stopDelivery().catch((e) => console.error(`! delivery teardown: ${e.message}`));
604
+ stopManager();
605
+ stopAuthService(space);
606
+ // Only unrecord if the registry still points at THIS broker. A newer broker for the same space
607
+ // (a concurrent `up`, or a different-port re-up that recorded after us) may have replaced our
608
+ // record — removing by name would clobber the live winner and hide it from the registry.
609
+ const mine = findMesh(space);
610
+ if (mine && mine.server === server && mine.root === cotalRoot()) {
611
+ removeMesh(space);
612
+ if (getCurrent() === space)
613
+ clearCurrent();
614
+ }
615
+ if (activationFinished)
616
+ process.exit(code ?? 0);
617
+ });
618
+ const ready = await waitReady(server, setup?.creds);
619
+ if (!ready) {
620
+ child.kill("SIGTERM");
621
+ const reason = `nats-server did not become ready at ${server}`;
622
+ markPendingResumeDegraded(resumeAttempt ?? "", reason);
623
+ throw new Error(reason);
624
+ }
625
+ if (restored)
626
+ await provePreparedRestoreListener(restored);
627
+ {
628
+ if (!resumeAttempt)
629
+ await postStart(server, space, setup, seedFile);
630
+ // USER MODE: the auth service comes up FIRST among the daemons — until its callout answers,
631
+ // every user-mode connect to this broker is denied, so `up` must not report a usable user mesh
632
+ // (nor let agents race it) on a half-started auth plane. (Foreground `up` doesn't exit here, so
633
+ // `ok` has no exit code to carry — the red consequence line above is the operator signal.)
634
+ const svc = await startUserAuthService(space, server, setup);
635
+ // Record BEFORE the control plane comes up: the manager's fail-closed mode detection requires
636
+ // an authoritative registry entry (marker-without-registry is a refused start, not a guess),
637
+ // so the record must exist by the time it boots. A manager/delivery failure after this leaves
638
+ // a recorded-but-degraded mesh — the documented, healable posture.
639
+ recordOurMesh({
640
+ space, server, root: cotalRoot(),
641
+ mode: setup?.prepared ? "user" : useAuth ? "auth" : "open",
642
+ ...(svc.userAuth ? { userAuth: svc.userAuth } : {}),
643
+ ts: new Date().toISOString(),
644
+ });
645
+ // Bring up the delivery daemon WITH the server (auth mode only — it self-gates on `.cotal/auth`).
646
+ // It is part of the server, so `cotal up` starts it by default; open dev mode has no daemon.
647
+ // Class-2 credential renewal is NOT wired here: the MANAGER is the renewal owner (it is resident
648
+ // in every mesh mode — foreground, --detach, refresh — where this foreground process is not).
649
+ const controlPlane = await startDeliveryWithBroker(space, server, {
650
+ runtime: values.runtime,
651
+ resumeAttempt,
652
+ resumeCommitToken: restored?.managerCommit?.durableCommitToken ?? ordinaryAttempt?.managerCommit?.durableCommitToken,
653
+ });
654
+ if (restored && process.env.COTAL_SMOKE_FAIL_AFTER_RESTORE_LISTENER_READY === "1")
655
+ throw new Error("smoke-injected failure after restore listener readiness");
656
+ await completeResumeActivation(resumeAttempt, controlPlane && svc.ok, !svc.ok ? "normal listener started but the user-auth service is unavailable" : "normal listener started but the control plane is degraded", server);
657
+ activationFinished = true;
171
658
  }
659
+ await new Promise(() => { });
172
660
  }
173
- if (values.detach) {
174
- const { pid, source, authService, delivery, manager } = await startMeshDetached({
175
- server,
176
- storeDir: values["store-dir"],
177
- space: values.space,
178
- open: values.open,
179
- userAuth: wantUser ? { idpUrl: values.idp } : undefined,
180
- channels: values.channels,
181
- host,
182
- runtime: values.runtime,
183
- });
184
- console.log(c.dim(`Started nats-server (${source}).`));
185
- console.log(c.green(renderDetachedSummary({ pid, delivery, authService: wantUser && authService, manager })));
186
- // A user mesh whose auth service never became ready is recorded + running (a re-`cotal up`
187
- // heals it), but this `up` did NOT deliver what was asked — automation must see that in the
188
- // exit code, not only in the red line above.
189
- if (!authService)
190
- process.exitCode = 1;
661
+ finally {
662
+ releaseStartupLock();
663
+ }
664
+ }
665
+ function assertOrdinaryUpAllowed(root, storeDir) {
666
+ const maintenance = readMaintenanceJournal(root);
667
+ if (!maintenance)
668
+ return;
669
+ if (maintenance.state === "active") {
670
+ if (!isManagerCommittedRestore(maintenance))
671
+ throw new Error(`restore attempt ${maintenance.restore.attemptId} is active without durable manager commit evidence`);
672
+ if (!maintenance.listenerProof)
673
+ throw new Error(`restore attempt ${maintenance.restore.attemptId} is active without a bound listener proof`);
674
+ const status = localProcessOwnerStatus(maintenance.listenerProof.processOwner);
675
+ if (status !== "dead")
676
+ throw new Error(`restore attempt ${maintenance.restore.attemptId} active listener ownership is ${status}; refusing ordinary startup`);
677
+ // Relaunching after restore must serve the exact recorded active target, not whatever store the
678
+ // caller happens to name — the journal's provenance would otherwise describe a different mesh.
679
+ if (!storeDir)
680
+ throw new Error(`restore attempt ${maintenance.restore.attemptId} is active; relaunch with bare \`cotal up\` over the recorded target ${maintenance.restore.target.path}`);
681
+ if (!sameStoreIdentity(readStoreIdentity(resolve(storeDir)), maintenance.restore.target))
682
+ throw new Error(`ordinary startup after restore must use the recorded active target store ${maintenance.restore.target.path}, not ${storeDir}`);
191
683
  return;
192
684
  }
193
- const useAuth = !values.open;
194
- const space = values.space ?? resolveSpace(process.cwd());
195
- ensureRootForSpace(useAuth, space); // may pin the cwd as this space's root before any cotalPath use
196
- refuseOpenOverUserState(Boolean(values.open), space);
197
- const storeDir = values["store-dir"] ? resolve(values["store-dir"]) : cotalPath("nats");
198
- mkdirSync(storeDir, { recursive: true });
199
- await claimSpace(space, server, cotalRoot());
200
- const seedFile = loadChannelsFile(values.channels);
201
- const setup = useAuth ? await authSetup(storeDir, server, space, host, wantUser ? { idpUrl: values.idp } : undefined) : undefined;
202
- const port = Number(new URL(server).port) || 4222;
203
- const natsArgs = setup ? ["-c", setup.confPath] : ["-js", "-sd", storeDir, "-p", String(port), "-a", host];
204
- const { bin, source } = await resolveNatsServer();
205
- console.log(c.dim(`Starting nats-server (JetStream, ${useAuth ? "JWT auth" : "OPEN/no-auth"}, ${source}) - store: ${storeDir}, bind: ${host}`));
206
- console.log(c.dim("Press Ctrl-C to stop.\n"));
207
- const child = spawn(bin, natsArgs, { stdio: "inherit" });
208
- child.on("error", (err) => {
209
- console.error(c.red(`Failed to start nats-server: ${err.message}`));
210
- process.exit(1);
685
+ if (maintenance.state === "ready")
686
+ throw new Error("ordinary resume must begin through the attempt-bound startup path");
687
+ throw new Error(`cotal up is refused while maintenance state is ${maintenance.state}; follow the recorded restore recovery`);
688
+ }
689
+ function markPendingResumeDegraded(attemptId, reason) {
690
+ const ordinary = pendingOrdinaryResumes.get(attemptId);
691
+ if (ordinary) {
692
+ const lock = acquireMaintenanceLock(ordinary.root);
693
+ try {
694
+ const journal = readMaintenanceJournal(ordinary.root);
695
+ if (journal && (journal.state === "resume-intent" || journal.state === "resume-active"))
696
+ markOrdinaryResumeDegraded(lock, reason, [{
697
+ action: "repair",
698
+ description: "Preserve the store and retained resume inventory; repair forward, then retry the same-principal activation.",
699
+ paths: [journal.source.path],
700
+ }]);
701
+ }
702
+ finally {
703
+ releaseMaintenanceLock(lock);
704
+ }
705
+ return;
706
+ }
707
+ const restored = pendingRestores.get(attemptId);
708
+ if (restored)
709
+ markPreparedRestoreDegraded(restored.root, restored.attemptId, reason);
710
+ }
711
+ async function resumeControlAuth(root, mode) {
712
+ if (mode === "open")
713
+ return {};
714
+ const auth = loadSpaceAuth(authDir(root));
715
+ if (!auth)
716
+ throw new Error("same-principal resume requires the existing space trust material");
717
+ const identity = newIdentity();
718
+ return {
719
+ creds: await mintCreds(auth, identity, "control-caller-admin", {
720
+ expiresAt: Math.floor((Date.now() + 30 * 60 * 1000) / 1000),
721
+ }),
722
+ };
723
+ }
724
+ function restoreListenerOwner(pid, nonce, startedAt) {
725
+ if (!Number.isInteger(pid) || pid <= 0)
726
+ throw new Error("restore listener spawn returned no pid");
727
+ return { pid, host: hostname(), startedAt, id: `restore-listener-${nonce}` };
728
+ }
729
+ function removeMatchingNatsPid(pid) {
730
+ const path = cotalPath("nats.pid");
731
+ try {
732
+ const stat = lstatSync(path);
733
+ if (stat.isFile() && !stat.isSymbolicLink() && readFileSync(path, "utf8") === String(pid))
734
+ rmSync(path);
735
+ }
736
+ catch { /* absent, changed, or not owned by this spawn */ }
737
+ }
738
+ function waitForChildExit(child, timeoutMs) {
739
+ if (child.exitCode !== null || child.signalCode !== null)
740
+ return Promise.resolve(true);
741
+ return new Promise((resolveExit) => {
742
+ const onExit = () => finish(true);
743
+ const timer = setTimeout(() => finish(false), timeoutMs);
744
+ const finish = (exited) => {
745
+ clearTimeout(timer);
746
+ child.off("exit", onExit);
747
+ resolveExit(exited);
748
+ };
749
+ child.once("exit", onExit);
211
750
  });
212
- // The control plane is coupled to the broker: stop the delivery daemon AND the detached manager
213
- // (AND the space's user-auth service) when this `up` stops (Ctrl-C), so none outlives the broker
214
- // it serves a surviving manager would reconnect-loop invisibly against the dead (or the NEXT)
215
- // broker (the documented orphan-supervisor failure mode). All kill by pidfile, symmetric; the
216
- // auth service's pid is space-scoped so no other space's daemon can ever be hit.
217
- const stop = () => { stopDelivery(); stopManager(); stopAuthService(space); child.kill("SIGTERM"); };
218
- process.on("SIGINT", stop);
219
- process.on("SIGTERM", stop);
220
- // The broker is gone — drop it from the registry (and the `current` pointer if it was the default)
221
- // so a later `cotal spawn` doesn't try to join a dead mesh.
222
- child.on("exit", (code) => {
223
- stopDelivery();
224
- stopManager();
225
- stopAuthService(space);
226
- // Only unrecord if the registry still points at THIS broker. A newer broker for the same space
227
- // (a concurrent `up`, or a different-port re-up that recorded after us) may have replaced our
228
- // record removing by name would clobber the live winner and hide it from the registry.
229
- const mine = findMesh(space);
230
- if (mine && mine.server === server && mine.root === cotalRoot()) {
231
- removeMesh(space);
232
- if (getCurrent() === space)
233
- clearCurrent();
234
- }
235
- process.exit(code ?? 0);
751
+ }
752
+ async function stopUnboundRestoreListener(child) {
753
+ if (child.exitCode !== null || child.signalCode !== null)
754
+ return;
755
+ child.kill("SIGTERM");
756
+ if (await waitForChildExit(child, 5_000))
757
+ return;
758
+ child.kill("SIGKILL");
759
+ if (!await waitForChildExit(child, 5_000))
760
+ throw new Error(`unbound restore listener process ${child.pid ?? "unknown"} did not exit`);
761
+ }
762
+ function bindSpawnedRestoreListener(prepared, pid, startedAt) {
763
+ bindRestoreListenerOwner(prepared, restoreListenerOwner(pid, prepared.serverNonce, startedAt));
764
+ }
765
+ function bindRestoreListenerOwner(prepared, processOwner) {
766
+ bindPreparedRestoreListener(prepared, processOwner);
767
+ if (process.env.COTAL_SMOKE_EXIT_AFTER_RESTORE_LISTENER_BIND === "1")
768
+ process.exit(86);
769
+ }
770
+ function sameProcessOwner(a, b) {
771
+ return a.pid === b.pid && a.host === b.host && a.startedAt === b.startedAt && a.id === b.id;
772
+ }
773
+ function sameRestoreListenerProof(a, b) {
774
+ return a.attemptId === b.attemptId && a.serverName === b.serverName &&
775
+ a.serverNonce === b.serverNonce && sameProcessOwner(a.processOwner, b.processOwner) &&
776
+ a.serverEndpoint === b.serverEndpoint && sameStoreIdentity(a.target, b.target);
777
+ }
778
+ function readNatsInfo(endpoint, timeoutMs = 2_000) {
779
+ return new Promise((resolveInfo, rejectInfo) => {
780
+ let settled = false;
781
+ let socket;
782
+ const finish = (error, info) => {
783
+ if (settled)
784
+ return;
785
+ settled = true;
786
+ socket?.destroy();
787
+ if (error)
788
+ rejectInfo(error);
789
+ else
790
+ resolveInfo(info);
791
+ };
792
+ let url;
793
+ try {
794
+ url = new URL(endpoint);
795
+ }
796
+ catch {
797
+ rejectInfo(new Error(`invalid restore listener endpoint ${endpoint}`));
798
+ return;
799
+ }
800
+ socket = createConnection({ host: url.hostname, port: Number(url.port) || 4222 });
801
+ socket.setTimeout(timeoutMs);
802
+ let input = "";
803
+ socket.on("data", (chunk) => {
804
+ input += chunk.toString("utf8");
805
+ if (input.length > 64 * 1024)
806
+ return finish(new Error("restore listener INFO exceeds 64 KiB"));
807
+ const newline = input.indexOf("\r\n");
808
+ if (newline < 0)
809
+ return;
810
+ const line = input.slice(0, newline);
811
+ const brace = line.indexOf("{");
812
+ if (!/^INFO\b/.test(line) || brace < 0)
813
+ return finish(new Error("restore listener did not present a NATS INFO greeting"));
814
+ try {
815
+ const info = JSON.parse(line.slice(brace));
816
+ if (!info || typeof info !== "object" || Array.isArray(info))
817
+ throw new Error("INFO is not an object");
818
+ finish(undefined, info);
819
+ }
820
+ catch (error) {
821
+ finish(new Error(`restore listener presented invalid NATS INFO: ${error.message}`));
822
+ }
823
+ });
824
+ socket.on("timeout", () => finish(new Error("restore listener INFO timed out")));
825
+ socket.on("error", (error) => finish(new Error(`restore listener INFO failed: ${error.message}`)));
826
+ socket.on("close", () => finish(new Error("restore listener closed before NATS INFO")));
236
827
  });
237
- if (await waitReady(server, setup?.creds)) {
238
- await postStart(server, space, setup, seedFile);
239
- // USER MODE: the auth service comes up FIRST among the daemons — until its callout answers,
240
- // every user-mode connect to this broker is denied, so `up` must not report a usable user mesh
241
- // (nor let agents race it) on a half-started auth plane. (Foreground `up` doesn't exit here, so
242
- // `ok` has no exit code to carry — the red consequence line above is the operator signal.)
243
- const svc = await startUserAuthService(space, server, setup);
244
- // Record BEFORE the control plane comes up: the manager's fail-closed mode detection requires
245
- // an authoritative registry entry (marker-without-registry is a refused start, not a guess),
246
- // so the record must exist by the time it boots. A manager/delivery failure after this leaves
247
- // a recorded-but-degraded mesh — the documented, healable posture.
248
- recordOurMesh({
249
- space, server, root: cotalRoot(),
250
- mode: setup?.prepared ? "user" : useAuth ? "auth" : "open",
251
- ...(svc.userAuth ? { userAuth: svc.userAuth } : {}),
252
- ts: new Date().toISOString(),
828
+ }
829
+ async function provePreparedRestoreListener(prepared) {
830
+ const proof = prepared.listenerProof;
831
+ if (!proof)
832
+ throw new Error(`restore attempt ${prepared.attemptId} has no bound listener proof`);
833
+ if (proof.attemptId !== prepared.attemptId || proof.serverName !== prepared.serverName ||
834
+ proof.serverNonce !== prepared.serverNonce || proof.serverEndpoint !== prepared.server ||
835
+ proof.serverName !== `${proof.attemptId}-${proof.serverNonce}` || !/^[0-9a-f]{32}$/.test(proof.serverNonce) ||
836
+ proof.processOwner.id !== `restore-listener-${proof.serverNonce}`)
837
+ throw new Error(`restore attempt ${prepared.attemptId} listener proof does not match its launch record`);
838
+ const journal = readMaintenanceJournal(prepared.root);
839
+ if (!journal || (journal.state !== "commit-intent" && journal.state !== "manager-committed" && journal.state !== "degraded" && journal.state !== "active") ||
840
+ journal.restore.attemptId !== prepared.attemptId || !journal.listenerProof ||
841
+ !sameRestoreListenerProof(journal.listenerProof, proof) || journal.launch.server !== proof.serverEndpoint)
842
+ throw new Error(`restore attempt ${prepared.attemptId} listener proof does not exactly match durable recovery state`);
843
+ if (proof.processOwner.host !== hostname())
844
+ throw new Error(`restore attempt ${prepared.attemptId} listener process ownership is not live and local`);
845
+ try {
846
+ process.kill(proof.processOwner.pid, 0);
847
+ }
848
+ catch {
849
+ throw new Error(`restore attempt ${prepared.attemptId} listener process ownership is not live and local`);
850
+ }
851
+ const pidPath = join(prepared.root, ".cotal", "nats.pid");
852
+ let exactPidFile = false;
853
+ try {
854
+ const stat = lstatSync(pidPath);
855
+ exactPidFile = stat.isFile() && !stat.isSymbolicLink() &&
856
+ readFileSync(pidPath, "utf8") === String(proof.processOwner.pid);
857
+ }
858
+ catch { /* absent or unprovable */ }
859
+ if (!exactPidFile)
860
+ throw new Error(`restore attempt ${prepared.attemptId} listener pidfile does not match its process proof`);
861
+ const target = readStoreIdentity(prepared.targetPath);
862
+ if (!sameStoreIdentity(target, proof.target) || !sameStoreIdentity(journal.restore.target, proof.target))
863
+ throw new Error(`restore attempt ${prepared.attemptId} target identity changed`);
864
+ const mesh = findMesh(prepared.space);
865
+ if (mesh && (mesh.server !== prepared.server || mesh.root !== prepared.root || mesh.mode !== prepared.mode))
866
+ throw new Error(`restore attempt ${prepared.attemptId} conflicts with the recorded mesh identity`);
867
+ const info = await readNatsInfo(proof.serverEndpoint);
868
+ if (info.server_name !== proof.serverName)
869
+ throw new Error(`restore attempt ${prepared.attemptId} reached a foreign NATS server name/nonce`);
870
+ const auth = await resumeControlAuth(prepared.root, prepared.mode);
871
+ const nc = await connect({
872
+ servers: prepared.server,
873
+ ...standaloneConnectOpts(auth),
874
+ maxReconnectAttempts: 0,
875
+ });
876
+ try {
877
+ if (nc.info?.server_name !== proof.serverName)
878
+ throw new Error(`restore attempt ${prepared.attemptId} authenticated to a foreign NATS server name/nonce`);
879
+ if (!nc.info.jetstream)
880
+ throw new Error(`restore attempt ${prepared.attemptId} listener has no JetStream`);
881
+ }
882
+ finally {
883
+ await nc.drain().catch(() => { });
884
+ }
885
+ return proof;
886
+ }
887
+ function bindSpawnedOrdinaryResumeListener(pending, pid, startedAt) {
888
+ if (!Number.isInteger(pid) || pid <= 0)
889
+ throw new Error("resume listener spawn returned no pid");
890
+ const lock = acquireMaintenanceLock(pending.root);
891
+ try {
892
+ const journal = readMaintenanceJournal(pending.root);
893
+ if (!journal || !("ordinaryResume" in journal) || journal.ordinaryResume.attemptId !== pending.attemptId)
894
+ throw new Error(`resume listener bind does not match attempt ${pending.attemptId}`);
895
+ bindOrdinaryResumeListener(lock, {
896
+ attemptId: pending.attemptId,
897
+ serverName: pending.serverName,
898
+ serverNonce: pending.serverNonce,
899
+ processOwner: { pid, host: hostname(), startedAt, id: `resume-listener-${pending.serverNonce}` },
900
+ serverEndpoint: pending.server,
901
+ target: journal.source,
253
902
  });
254
- // Bring up the delivery daemon WITH the server (auth mode only — it self-gates on `.cotal/auth`).
255
- // It is part of the server, so `cotal up` starts it by default; open dev mode has no daemon.
256
- // Class-2 credential renewal is NOT wired here: the MANAGER is the renewal owner (it is resident
257
- // in every mesh mode — foreground, --detach, refresh — where this foreground process is not).
258
- await startDeliveryWithBroker(space, server, { runtime: values.runtime });
259
903
  }
260
- await new Promise(() => { });
904
+ finally {
905
+ releaseMaintenanceLock(lock);
906
+ }
907
+ }
908
+ /** Prove the recovered attempt's live bound listener IS ours end-to-end before adoption: launch
909
+ * binding, durable journal match, local pid + exact pidfile, source identity, mesh identity, the
910
+ * raw NATS INFO server name, and an authenticated connect confirming name + JetStream. */
911
+ async function proveOrdinaryResumeListener(pending) {
912
+ const proof = pending.adoptProof;
913
+ if (!proof)
914
+ throw new Error(`resume attempt ${pending.attemptId} has no bound listener proof`);
915
+ if (proof.attemptId !== pending.attemptId || proof.serverName !== pending.serverName ||
916
+ proof.serverNonce !== pending.serverNonce || proof.serverEndpoint !== pending.server ||
917
+ proof.serverName !== `${proof.attemptId}-${proof.serverNonce}` || !/^[0-9a-f]{32}$/.test(proof.serverNonce) ||
918
+ proof.processOwner.id !== `resume-listener-${proof.serverNonce}`)
919
+ throw new Error(`resume attempt ${pending.attemptId} listener proof does not match its launch record`);
920
+ const journal = readMaintenanceJournal(pending.root);
921
+ if (!journal || !("ordinaryResume" in journal) || journal.ordinaryResume.attemptId !== pending.attemptId ||
922
+ !journal.listenerProof || !sameRestoreListenerProof(journal.listenerProof, proof) ||
923
+ journal.ordinaryResume.launch.server !== proof.serverEndpoint)
924
+ throw new Error(`resume attempt ${pending.attemptId} listener proof does not exactly match durable recovery state`);
925
+ if (proof.processOwner.host !== hostname())
926
+ throw new Error(`resume attempt ${pending.attemptId} listener process ownership is not live and local`);
927
+ try {
928
+ process.kill(proof.processOwner.pid, 0);
929
+ }
930
+ catch {
931
+ throw new Error(`resume attempt ${pending.attemptId} listener process ownership is not live and local`);
932
+ }
933
+ const pidPath = join(pending.root, ".cotal", "nats.pid");
934
+ let exactPidFile = false;
935
+ try {
936
+ const stat = lstatSync(pidPath);
937
+ exactPidFile = stat.isFile() && !stat.isSymbolicLink() &&
938
+ readFileSync(pidPath, "utf8") === String(proof.processOwner.pid);
939
+ }
940
+ catch { /* absent or unprovable */ }
941
+ if (!exactPidFile)
942
+ throw new Error(`resume attempt ${pending.attemptId} listener pidfile does not match its process proof`);
943
+ const source = readStoreIdentity(pending.storeDir);
944
+ if (!sameStoreIdentity(source, proof.target) || !sameStoreIdentity(journal.source, proof.target))
945
+ throw new Error(`resume attempt ${pending.attemptId} preserved source identity changed`);
946
+ const mesh = findMesh(pending.space);
947
+ if (mesh && (mesh.server !== pending.server || mesh.root !== pending.root || mesh.mode !== pending.mode))
948
+ throw new Error(`resume attempt ${pending.attemptId} conflicts with the recorded mesh identity`);
949
+ const info = await readNatsInfo(proof.serverEndpoint);
950
+ if (info.server_name !== proof.serverName)
951
+ throw new Error(`resume attempt ${pending.attemptId} reached a foreign NATS server name/nonce`);
952
+ const auth = await resumeControlAuth(pending.root, pending.mode);
953
+ const nc = await connect({
954
+ servers: pending.server,
955
+ ...standaloneConnectOpts(auth),
956
+ maxReconnectAttempts: 0,
957
+ });
958
+ try {
959
+ if (nc.info?.server_name !== proof.serverName)
960
+ throw new Error(`resume attempt ${pending.attemptId} authenticated to a foreign NATS server name/nonce`);
961
+ if (!nc.info.jetstream)
962
+ throw new Error(`resume attempt ${pending.attemptId} listener has no JetStream`);
963
+ }
964
+ finally {
965
+ await nc.drain().catch(() => { });
966
+ }
967
+ return proof;
968
+ }
969
+ /** Light wire check for a freshly SPAWNED ordinary-resume listener: the greeted server must carry
970
+ * the attempt's exact name/nonce (the full seven-point prove is for adopting a survivor). */
971
+ async function verifySpawnedOrdinaryListener(pending) {
972
+ const info = await readNatsInfo(pending.server);
973
+ if (info.server_name !== pending.serverName)
974
+ throw new Error(`resume attempt ${pending.attemptId} spawned listener reports a foreign NATS server name`);
975
+ }
976
+ async function resumeProvenOrdinaryListener(pending) {
977
+ await proveOrdinaryResumeListener(pending);
978
+ const svc = await ensureRecoveredUserAuth(pending);
979
+ recordOurMesh({
980
+ space: pending.space,
981
+ server: pending.server,
982
+ root: pending.root,
983
+ mode: pending.mode,
984
+ ...(svc.userAuth ? { userAuth: svc.userAuth } : {}),
985
+ ts: new Date().toISOString(),
986
+ });
987
+ const controlPlane = await startDeliveryWithBroker(pending.space, pending.server, {
988
+ runtime: pending.runtime,
989
+ resumeAttempt: pending.attemptId,
990
+ resumeCommitToken: pending.managerCommit?.durableCommitToken,
991
+ });
992
+ await completeResumeActivation(pending.attemptId, controlPlane && svc.ok, !svc.ok ? "adopted resume listener has no user-auth service" : "adopted resume listener has a degraded control plane", pending.server);
993
+ }
994
+ async function ensureRecoveredUserAuth(prepared) {
995
+ if (prepared.mode !== "user")
996
+ return { ok: true };
997
+ const auth = loadSpaceAuth(authDir(prepared.root));
998
+ if (!auth)
999
+ throw new Error("restored user-auth listener has no retained trust material");
1000
+ const stateDir = userAuthStateDir(prepared.root, prepared.space);
1001
+ const provider = await resolveAuthProvider().prepareServer({
1002
+ space: prepared.space,
1003
+ operatorSeed: auth.operator.seed,
1004
+ account: { pub: auth.account.pub, signingSeed: auth.account.signingSeed },
1005
+ store: workspaceSecretStore(prepared.root),
1006
+ dir: stateDir,
1007
+ });
1008
+ return startUserAuthService(prepared.space, prepared.server, { prepared: provider, stateDir });
1009
+ }
1010
+ async function resumeProvenRestoreListener(prepared) {
1011
+ await provePreparedRestoreListener(prepared);
1012
+ const svc = await ensureRecoveredUserAuth(prepared);
1013
+ recordOurMesh({
1014
+ space: prepared.space,
1015
+ server: prepared.server,
1016
+ root: prepared.root,
1017
+ mode: prepared.mode,
1018
+ ...(svc.userAuth ? { userAuth: svc.userAuth } : {}),
1019
+ ts: new Date().toISOString(),
1020
+ });
1021
+ const controlPlane = await startDeliveryWithBroker(prepared.space, prepared.server, {
1022
+ runtime: prepared.runtime,
1023
+ resumeAttempt: prepared.attemptId,
1024
+ resumeCommitToken: prepared.managerCommit?.durableCommitToken,
1025
+ });
1026
+ await completeResumeActivation(prepared.attemptId, controlPlane && svc.ok, !svc.ok ? "proven restore listener has no user-auth service" : "proven restore listener has a degraded control plane", prepared.server);
1027
+ }
1028
+ async function completeResumeActivation(attemptId, healthy, reason, server) {
1029
+ if (!attemptId)
1030
+ return;
1031
+ const ordinary = pendingOrdinaryResumes.get(attemptId);
1032
+ const restored = pendingRestores.get(attemptId);
1033
+ const pending = ordinary ?? restored;
1034
+ if (!pending)
1035
+ throw new Error(`resume activation lost attempt context ${attemptId}`);
1036
+ if (!healthy) {
1037
+ markPendingResumeDegraded(attemptId, reason);
1038
+ throw new Error(reason);
1039
+ }
1040
+ let journal = readMaintenanceJournal(pending.root);
1041
+ if (!journal)
1042
+ throw new Error(`resume attempt ${attemptId} lost its durable workspace journal`);
1043
+ if (restored) {
1044
+ if (!("restore" in journal) || journal.restore.attemptId !== attemptId)
1045
+ throw new Error(`restore activation journal does not match attempt ${attemptId}`);
1046
+ if (journal.state === "active") {
1047
+ restored.cleanupStage();
1048
+ pendingRestores.delete(attemptId);
1049
+ return;
1050
+ }
1051
+ }
1052
+ else {
1053
+ if (!("ordinaryResume" in journal) || journal.ordinaryResume.attemptId !== attemptId)
1054
+ throw new Error(`ordinary resume journal does not match attempt ${attemptId}`);
1055
+ if (journal.state === "resume-retired") {
1056
+ const lock = acquireMaintenanceLock(ordinary.root);
1057
+ try {
1058
+ consumeRetiredMaintenance(lock);
1059
+ }
1060
+ finally {
1061
+ releaseMaintenanceLock(lock);
1062
+ }
1063
+ pendingOrdinaryResumes.delete(attemptId);
1064
+ return;
1065
+ }
1066
+ }
1067
+ let managerCommit;
1068
+ if (restored && (journal.state === "manager-committed" ||
1069
+ (journal.state === "degraded" && journal.managerCommit))) {
1070
+ managerCommit = journal.managerCommit;
1071
+ restored.managerCommit = managerCommit;
1072
+ restored.journalState = journal.state;
1073
+ }
1074
+ else if (ordinary && journal.state === "resume-committed") {
1075
+ managerCommit = journal.managerCommit;
1076
+ ordinary.journalState = "resume-committed";
1077
+ }
1078
+ const auth = await resumeControlAuth(pending.root, pending.mode);
1079
+ const readinessDeadline = Date.now() + 20_000;
1080
+ for (;;) {
1081
+ const ready = await askManager(pending.space, server, "ps", undefined, auth, CONTROL_ADMIN, 2_000);
1082
+ if (!ready.error?.startsWith("no manager reachable"))
1083
+ break;
1084
+ if (Date.now() >= readinessDeadline) {
1085
+ markPendingResumeDegraded(attemptId, ready.error);
1086
+ throw new Error(ready.error);
1087
+ }
1088
+ await new Promise((resolveWait) => setTimeout(resolveWait, 150));
1089
+ }
1090
+ const resumed = await askManager(pending.space, server, "resumePreserved", {
1091
+ attemptId,
1092
+ inventory: restored?.selection === "registry"
1093
+ ? { ...pending.inventory, agents: [] }
1094
+ : pending.inventory,
1095
+ }, auth, CONTROL_ADMIN, 10 * 60 * 1000);
1096
+ if (!resumed.ok) {
1097
+ const detail = resumed.data ? ` (${JSON.stringify(resumed.data)})` : "";
1098
+ const message = `${resumed.error ?? "retained-agent resume failed"}${detail}`;
1099
+ markPendingResumeDegraded(attemptId, message);
1100
+ throw new Error(message);
1101
+ }
1102
+ if (restored && process.env.COTAL_SMOKE_EXIT_AFTER_RESUME_PRESERVED === "1")
1103
+ process.exit(88);
1104
+ if (ordinary && !managerCommit) {
1105
+ const lock = acquireMaintenanceLock(ordinary.root);
1106
+ try {
1107
+ const current = readMaintenanceJournal(ordinary.root);
1108
+ if (current?.state !== "resume-active") {
1109
+ markOrdinaryResumeActive(lock, {
1110
+ operation: "resumePreserved",
1111
+ attemptId,
1112
+ state: "awaitingCommit",
1113
+ observedAt: new Date().toISOString(),
1114
+ });
1115
+ }
1116
+ ordinary.journalState = "resume-active";
1117
+ }
1118
+ finally {
1119
+ releaseMaintenanceLock(lock);
1120
+ }
1121
+ }
1122
+ if (!managerCommit) {
1123
+ const committed = await askManager(pending.space, server, "commitResume", { attemptId }, auth, CONTROL_ADMIN, 40_000);
1124
+ if (!committed.ok) {
1125
+ const message = committed.error ?? "manager resume commit failed";
1126
+ markPendingResumeDegraded(attemptId, message);
1127
+ throw new Error(message);
1128
+ }
1129
+ if (!isManagerCommitResult(committed.data, attemptId)) {
1130
+ const message = `manager resume commit returned invalid awaiting-finalize evidence for attempt ${attemptId}`;
1131
+ markPendingResumeDegraded(attemptId, message);
1132
+ throw new Error(message);
1133
+ }
1134
+ managerCommit = committed.data;
1135
+ if (ordinary) {
1136
+ const lock = acquireMaintenanceLock(ordinary.root);
1137
+ try {
1138
+ recordOrdinaryResumeManagerCommit(lock, managerCommit);
1139
+ ordinary.journalState = "resume-committed";
1140
+ ordinary.managerCommit = managerCommit;
1141
+ }
1142
+ finally {
1143
+ releaseMaintenanceLock(lock);
1144
+ }
1145
+ }
1146
+ else {
1147
+ recordPreparedRestoreManagerCommit(restored, managerCommit);
1148
+ }
1149
+ if (process.env.COTAL_SMOKE_EXIT_AFTER_RESUME_COMMIT === "1")
1150
+ process.exit(89);
1151
+ }
1152
+ else {
1153
+ const recommitted = await askManager(pending.space, server, "commitResume", { attemptId }, auth, CONTROL_ADMIN, 40_000);
1154
+ // A surviving manager that already finalized legitimately answers {state:"active"} with the
1155
+ // exact durable token: accept both committed shapes idempotently, then reissue the token-bound
1156
+ // finalize (itself idempotent) below.
1157
+ const recovered = recommitted.ok ? recommitted.data : undefined;
1158
+ const exactToken = Boolean(recovered && typeof recovered === "object" &&
1159
+ recovered.attemptId === attemptId &&
1160
+ recovered.durableCommitToken === managerCommit.durableCommitToken &&
1161
+ (recovered.state === "awaitingFinalize" || recovered.state === "active"));
1162
+ if (!exactToken)
1163
+ throw new Error(`replacement manager did not recover the durable commit token for attempt ${attemptId}`);
1164
+ }
1165
+ const finalized = await askManager(pending.space, server, "finalizeResume", { attemptId, durableCommitToken: managerCommit.durableCommitToken }, auth, CONTROL_ADMIN, 40_000);
1166
+ if (!finalized.ok)
1167
+ throw new Error(finalized.error ?? `manager resume finalize failed for attempt ${attemptId}`);
1168
+ if (!isManagerFinalizeResult(finalized.data, attemptId))
1169
+ throw new Error(`manager resume finalize returned invalid active evidence for attempt ${attemptId}`);
1170
+ const finalizeEvidence = {
1171
+ attemptId,
1172
+ state: "active",
1173
+ durableCommitToken: managerCommit.durableCommitToken,
1174
+ };
1175
+ if (process.env.COTAL_SMOKE_EXIT_AFTER_RESUME_FINALIZE === "1")
1176
+ process.exit(91);
1177
+ if (ordinary) {
1178
+ const lock = acquireMaintenanceLock(ordinary.root);
1179
+ try {
1180
+ retireOrdinaryResume(lock, finalizeEvidence);
1181
+ consumeRetiredMaintenance(lock);
1182
+ }
1183
+ finally {
1184
+ releaseMaintenanceLock(lock);
1185
+ }
1186
+ pendingOrdinaryResumes.delete(attemptId);
1187
+ }
1188
+ else {
1189
+ restored.managerCommit = managerCommit;
1190
+ markPreparedRestoreActive(restored, finalizeEvidence);
1191
+ restored.cleanupStage();
1192
+ pendingRestores.delete(attemptId);
1193
+ }
261
1194
  }
262
1195
  /** Bring the space's USER-AUTH service up with the broker (user mode only — `setup.prepared` is the
263
1196
  * provider's output). Loud both ways (U5): a ready service prints the login line; a service that
@@ -347,7 +1280,7 @@ async function upManifest(file, opts) {
347
1280
  try {
348
1281
  mkdirSync(cotalPath("nats"), { recursive: true });
349
1282
  const setup = await authSetup(cotalPath("nats"), server, m.space, host, userAuth);
350
- owner = await resolveAuthProvider().ownerForLogin({ dir: setup.stateDir, space: m.space });
1283
+ owner = await resolveAuthProvider().ownerForLogin({ store: workspaceSecretStore(cotalRoot()), dir: setup.stateDir, space: m.space });
351
1284
  }
352
1285
  catch (e) {
353
1286
  console.error(c.red(`✗ ${e.message}`));
@@ -461,13 +1394,30 @@ export async function startMeshDetached(opts = {}) {
461
1394
  const host = opts.host ?? "127.0.0.1";
462
1395
  const setup = useAuth ? await authSetup(storeDir, server, space, host, opts.userAuth) : undefined;
463
1396
  const port = Number(new URL(server).port) || 4222;
464
- const args = setup ? ["-c", setup.confPath] : ["-js", "-sd", storeDir, "-p", String(port), "-a", host];
1397
+ const args = [
1398
+ ...(setup ? ["-c", setup.confPath] : ["-js", "-sd", storeDir, "-p", String(port), "-a", host]),
1399
+ ...(opts.boundListener ? ["--name", opts.boundListener.serverName] : []),
1400
+ ];
465
1401
  const { bin, source } = await resolveNatsServer();
466
1402
  const logPath = cotalPath("nats.log");
467
1403
  const startOffset = existsSync(logPath) ? statSync(logPath).size : 0;
468
1404
  const fd = openSync(logPath, "a");
1405
+ const listenerStartedAt = new Date().toISOString();
469
1406
  const child = spawn(bin, args, { detached: true, stdio: ["ignore", fd, fd] });
470
1407
  closeSync(fd);
1408
+ if (opts.boundListener) {
1409
+ writeFileSync(cotalPath("nats.pid"), String(child.pid));
1410
+ if (process.env.COTAL_SMOKE_EXIT_AFTER_RESTORE_LISTENER_SPAWN === "1")
1411
+ process.exit(87);
1412
+ try {
1413
+ opts.boundListener.onSpawn(child.pid ?? 0, listenerStartedAt);
1414
+ }
1415
+ catch (error) {
1416
+ await stopUnboundRestoreListener(child);
1417
+ removeMatchingNatsPid(child.pid ?? 0);
1418
+ throw error;
1419
+ }
1420
+ }
471
1421
  child.unref();
472
1422
  let tailing = Boolean(opts.onLine);
473
1423
  if (opts.onLine)
@@ -476,10 +1426,16 @@ export async function startMeshDetached(opts = {}) {
476
1426
  tailing = false;
477
1427
  if (!ready) {
478
1428
  child.kill("SIGTERM");
1429
+ if (opts.boundListener)
1430
+ rmSync(cotalPath("nats.pid"), { force: true });
479
1431
  throw new Error(`nats-server did not become reachable at ${server} - see ${logPath}`);
480
1432
  }
481
- writeFileSync(cotalPath("nats.pid"), String(child.pid));
482
- await postStart(server, space, setup, seedFile);
1433
+ if (!opts.boundListener)
1434
+ writeFileSync(cotalPath("nats.pid"), String(child.pid));
1435
+ if (opts.boundListener)
1436
+ await opts.boundListener.verify();
1437
+ if (!opts.skipPostStart)
1438
+ await postStart(server, space, setup, seedFile);
483
1439
  // USER MODE: the auth service comes up FIRST among the daemons (see the foreground path).
484
1440
  const svc = await startUserAuthService(space, server, setup);
485
1441
  // Record BEFORE the control plane: the manager's fail-closed mode detection needs the
@@ -492,7 +1448,12 @@ export async function startMeshDetached(opts = {}) {
492
1448
  ts: new Date().toISOString(),
493
1449
  });
494
1450
  // Bring up the delivery daemon WITH the detached broker (auth mode only; `cotal down` tears both down).
495
- const controlPlane = await startDeliveryWithBroker(space, server, { runtime: opts.runtime, launch: opts.launch });
1451
+ const controlPlane = await startDeliveryWithBroker(space, server, {
1452
+ runtime: opts.runtime,
1453
+ launch: opts.launch,
1454
+ resumeAttempt: opts.resumeAttempt,
1455
+ resumeCommitToken: opts.resumeCommitToken,
1456
+ });
496
1457
  return {
497
1458
  server,
498
1459
  pid: child.pid ?? 0,
@@ -513,8 +1474,7 @@ function refuseOpenOverUserState(open, space) {
513
1474
  const stateDir = userAuthStateDir(cotalRoot(), space);
514
1475
  if (!existsSync(stateDir))
515
1476
  return;
516
- console.error(c.red(`✗ space "${space}" has user auth enabled (state under ${stateDir}) - \`--open\` would serve its streams without auth. Start it with \`cotal up --user-auth\`, or remove that directory deliberately to disable user auth (existing logins/grants die with it)`));
517
- process.exit(1);
1477
+ throw new Error(`space "${space}" has user auth enabled (state under ${stateDir}) - \`--open\` would serve its streams without auth. Start it with \`cotal up --user-auth\`, or remove that directory deliberately to disable user auth (existing logins/grants die with it)`);
518
1478
  }
519
1479
  /** Today a root's `.cotal/auth` is created for one space (its account is space-bound), so an
520
1480
  * explicit `--space` naming a different space cannot run against this root's trust material — the
@@ -537,8 +1497,7 @@ function ensureRootForSpace(useAuth, space) {
537
1497
  console.log(c.dim(`nearest mesh root ${root} is space "${existing.space}" - making this folder its own root for "${space}"`));
538
1498
  return;
539
1499
  }
540
- console.error(c.red(`✗ this folder is the root of space "${existing.space}" (${authDir(root)}), so it can't also run "${space}" - drop \`--space\` to run "${existing.space}", or start "${space}" from a different folder (it becomes that mesh's own root)`));
541
- process.exit(1);
1500
+ throw new Error(`this folder is the root of space "${existing.space}" (${authDir(root)}), so it can't also run "${space}" - drop \`--space\` to run "${existing.space}", or start "${space}" from a different folder (it becomes that mesh's own root)`);
542
1501
  }
543
1502
  /** A space name maps to one mesh in the registry (the key `--space`/`use`/`down` act on). Before
544
1503
  * starting a broker, refuse to reuse a space already claimed by a DIFFERENT live mesh — a stale/dead
@@ -551,8 +1510,7 @@ async function claimSpace(space, server, root) {
551
1510
  if (!existing || (existing.server === server && existing.root === root))
552
1511
  return;
553
1512
  if (await isReachable(existing.server)) {
554
- console.error(c.red(`✗ space "${space}" is already in use by a mesh at ${existing.server} (${existing.root}) - pick a different \`--space\`, or \`cotal down\` it first`));
555
- process.exit(1);
1513
+ throw new Error(`space "${space}" is already in use by a mesh at ${existing.server} (${existing.root}) - pick a different \`--space\`, or \`cotal down\` it first`);
556
1514
  }
557
1515
  removeMesh(space); // the prior holder's broker is gone — reclaim the name
558
1516
  }
@@ -658,8 +1616,7 @@ function loadChannelsFile(explicit) {
658
1616
  const path = explicit ? resolve(explicit) : cotalPath("channels.json");
659
1617
  if (!existsSync(path)) {
660
1618
  if (explicit) {
661
- console.error(c.red(`channels file not found: ${path}`));
662
- process.exit(1);
1619
+ throw new Error(`channels file not found: ${path}`);
663
1620
  }
664
1621
  return undefined;
665
1622
  }
@@ -684,8 +1641,7 @@ async function authSetup(storeDir, server, space, host = "127.0.0.1", user) {
684
1641
  }
685
1642
  const stateDir = userAuthStateDir(cotalRoot(), space); // the provider's space-scoped state dir
686
1643
  if (!user && existsSync(stateDir)) {
687
- console.error(c.red(`✗ space "${space}" has user auth enabled (state under ${stateDir}) - start it with \`--user-auth\`, or remove that directory deliberately to disable user auth (existing logins/grants die with it)`));
688
- process.exit(1);
1644
+ throw new Error(`space "${space}" has user auth enabled (state under ${stateDir}) - start it with \`cotal up --user-auth\`, or remove that directory deliberately to disable user auth (existing logins/grants die with it)`);
689
1645
  }
690
1646
  let prepared;
691
1647
  if (user) {
@@ -698,13 +1654,13 @@ async function authSetup(storeDir, server, space, host = "127.0.0.1", user) {
698
1654
  space,
699
1655
  operatorSeed: auth.operator.seed,
700
1656
  account: { pub: auth.account.pub, signingSeed: auth.account.signingSeed },
1657
+ store: workspaceSecretStore(cotalRoot()),
701
1658
  dir: stateDir,
702
1659
  idpUrl: user.idpUrl,
703
1660
  });
704
1661
  }
705
1662
  catch (e) {
706
- console.error(c.red(`✗ ${e.message}`));
707
- process.exit(1);
1663
+ throw e instanceof Error ? e : new Error(String(e));
708
1664
  }
709
1665
  }
710
1666
  const port = Number(new URL(server).port) || 4222;