@cotal-ai/cli 0.11.6 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/backup.d.ts +13 -0
- package/dist/commands/backup.d.ts.map +1 -0
- package/dist/commands/backup.js +443 -0
- package/dist/commands/backup.js.map +1 -0
- package/dist/commands/channels.d.ts +10 -0
- package/dist/commands/channels.d.ts.map +1 -1
- package/dist/commands/channels.js.map +1 -1
- package/dist/commands/clean.d.ts +12 -2
- package/dist/commands/clean.d.ts.map +1 -1
- package/dist/commands/clean.js +161 -28
- package/dist/commands/clean.js.map +1 -1
- package/dist/commands/down.d.ts +5 -0
- package/dist/commands/down.d.ts.map +1 -1
- package/dist/commands/down.js +323 -5
- package/dist/commands/down.js.map +1 -1
- package/dist/commands/spawn.d.ts +4 -0
- package/dist/commands/spawn.d.ts.map +1 -1
- package/dist/commands/spawn.js +43 -23
- package/dist/commands/spawn.js.map +1 -1
- package/dist/commands/status.d.ts.map +1 -1
- package/dist/commands/status.js +3 -2
- package/dist/commands/status.js.map +1 -1
- package/dist/commands/up.d.ts +12 -0
- package/dist/commands/up.d.ts.map +1 -1
- package/dist/commands/up.js +1149 -193
- package/dist/commands/up.js.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +19 -4
- package/dist/index.js.map +1 -1
- package/dist/lib/backup-artifact.d.ts +50 -0
- package/dist/lib/backup-artifact.d.ts.map +1 -0
- package/dist/lib/backup-artifact.js +290 -0
- package/dist/lib/backup-artifact.js.map +1 -0
- package/dist/lib/delivery-proc.d.ts +7 -2
- package/dist/lib/delivery-proc.d.ts.map +1 -1
- package/dist/lib/delivery-proc.js +29 -19
- package/dist/lib/delivery-proc.js.map +1 -1
- package/dist/lib/endpoint-cut.d.ts +5 -0
- package/dist/lib/endpoint-cut.d.ts.map +1 -0
- package/dist/lib/endpoint-cut.js +53 -0
- package/dist/lib/endpoint-cut.js.map +1 -0
- package/dist/lib/isolated-broker.d.ts +70 -0
- package/dist/lib/isolated-broker.d.ts.map +1 -0
- package/dist/lib/isolated-broker.js +427 -0
- package/dist/lib/isolated-broker.js.map +1 -0
- package/dist/lib/maintenance-files.d.ts +10 -0
- package/dist/lib/maintenance-files.d.ts.map +1 -0
- package/dist/lib/maintenance-files.js +60 -0
- package/dist/lib/maintenance-files.js.map +1 -0
- package/dist/lib/manager-proc.d.ts +4 -0
- package/dist/lib/manager-proc.d.ts.map +1 -1
- package/dist/lib/manager-proc.js +2 -0
- package/dist/lib/manager-proc.js.map +1 -1
- package/dist/lib/restore.d.ts +52 -0
- package/dist/lib/restore.d.ts.map +1 -0
- package/dist/lib/restore.js +797 -0
- package/dist/lib/restore.js.map +1 -0
- package/package.json +7 -4
package/dist/commands/up.js
CHANGED
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
3
|
+
import { createConnection, createServer } from "node:net";
|
|
4
|
+
import { hostname } from "node:os";
|
|
5
|
+
import { mkdirSync, writeFileSync, readFileSync, existsSync, openSync, statSync, readSync, closeSync, lstatSync, rmSync, } from "node:fs";
|
|
4
6
|
import { join, resolve } from "node:path";
|
|
5
|
-
import { isReachable, DEFAULT_SERVER, createSpaceAuth, serverConfig, mintCreds, mintConnectionEvictorCreds, mintMembershipObserverCreds, newIdentity, setupSpaceStreams, seedChannelRegistry, ensureDefaultDeliveryClass, mkSecretDir, writeSecretFile, } from "@cotal-ai/core";
|
|
6
|
-
import {
|
|
7
|
+
import { isReachable, DEFAULT_SERVER, CONTROL_ADMIN, createSpaceAuth, serverConfig, mintCreds, mintConnectionEvictorCreds, mintMembershipObserverCreds, newIdentity, setupSpaceStreams, standaloneConnectOpts, seedChannelRegistry, ensureDefaultDeliveryClass, mkSecretDir, writeSecretFile, } from "@cotal-ai/core";
|
|
8
|
+
import { connect } from "@nats-io/transport-node";
|
|
9
|
+
import { assertUserAuthInfo, authDir, loadSpaceAuth, saveSpaceAuth, clearCurrent, findMesh, getCurrent, loadMeshes, recordMesh, removeMesh, setCurrent, userAuthStateDir, workspaceSecretStore, acquireMaintenanceLock, assertStoreIdentity, assessRestoreClaim, beginOrdinaryResume, bindOrdinaryResumeListener, consumeRetiredMaintenance, markOrdinaryResumeActive, markOrdinaryResumeDegraded, replaceDeadOrdinaryResumeListener, localProcessOwnerStatus, readMaintenanceJournal, readMaintenanceResumeDocument, readStoreIdentity, recordOrdinaryResumeManagerCommit, releaseMaintenanceLock, retireOrdinaryResume, sameStoreIdentity, } from "@cotal-ai/workspace";
|
|
7
10
|
import { ensureAuthService, resolveAuthProvider, stopAuthService } from "../lib/auth-proc.js";
|
|
8
11
|
import { resolveSpace } from "../lib/status.js";
|
|
9
12
|
import { c } from "../ui.js";
|
|
@@ -18,6 +21,10 @@ import { renderUpPlan, renderInherited, renderWarnings } from "../lib/manifest/r
|
|
|
18
21
|
import { failManifest } from "./topology.js";
|
|
19
22
|
import { extensionNames, preflightRuntime } from "../ext-loader.js";
|
|
20
23
|
import { completingFlagValue } from "../lib/completion.js";
|
|
24
|
+
import { askManager } from "../lib/control.js";
|
|
25
|
+
import { bindPreparedRestoreListener, isManagerCommitResult, isManagerFinalizeResult, isManagerCommittedRestore, markPreparedRestoreActive, markPreparedRestoreDegraded, prepareRestore, recordPreparedRestoreManagerCommit, rehydratePreparedRestore, replacePreparedDeadRestoreListener, } from "../lib/restore.js";
|
|
26
|
+
const pendingRestores = new Map();
|
|
27
|
+
const pendingOrdinaryResumes = new Map();
|
|
21
28
|
/** `cotal up` flags — colocated with the command (like `spawnFlags`) so its completion can read them. */
|
|
22
29
|
export const upFlags = [
|
|
23
30
|
{ name: "server", type: "string", value: "<url>", description: "listen URL override" },
|
|
@@ -25,6 +32,9 @@ export const upFlags = [
|
|
|
25
32
|
{ name: "space", type: "string", value: "<s>", description: "space name (default: the folder's)" },
|
|
26
33
|
{ name: "store-dir", type: "string", value: "<dir>", description: "JetStream store directory" },
|
|
27
34
|
{ name: "channels", type: "string", value: "<path>", description: "channel-registry seed file (JSON; default .cotal/channels.json)" },
|
|
35
|
+
{ name: "restore", type: "string", value: "<dir>", description: "restore an offline backup before exposing the normal listener" },
|
|
36
|
+
{ name: "restore-only", type: "string", value: "<registry>", description: "restore only the registry component" },
|
|
37
|
+
{ name: "accept-missing-source", type: "boolean", description: "explicit disaster consent when the inode-bound preserved source is absent" },
|
|
28
38
|
{ name: "open", type: "boolean", description: "unauthenticated dev mesh (no JWT/ACLs)" },
|
|
29
39
|
{ name: "user-auth", type: "boolean", description: "per-USER auth: login + bearer through the space's auth service" },
|
|
30
40
|
{ name: "idp", type: "string", value: "<url>", description: "with --user-auth: the IdP auth base URL to pin (first enable)" },
|
|
@@ -41,38 +51,291 @@ export function upComplete(argv) {
|
|
|
41
51
|
const flag = completingFlagValue(argv, upFlags);
|
|
42
52
|
if (flag?.name === "runtime")
|
|
43
53
|
return { items: ["pty", ...extensionNames("runtime")].filter((v, i, a) => a.indexOf(v) === i).map((value) => ({ value })), directive: "nofiles" };
|
|
54
|
+
if (flag?.name === "restore-only")
|
|
55
|
+
return { items: [{ value: "registry" }], directive: "nofiles" };
|
|
56
|
+
if (flag?.name === "restore")
|
|
57
|
+
return { items: [], directive: "default" };
|
|
44
58
|
const items = upFlags.map((f) => ({ value: `--${f.name}`, description: f.description }));
|
|
45
59
|
return { items, directive: items.length ? "nofiles" : "default" };
|
|
46
60
|
}
|
|
47
61
|
export async function up(args) {
|
|
48
62
|
const values = args.values;
|
|
63
|
+
if (values.restore) {
|
|
64
|
+
if (values.file || values.channels)
|
|
65
|
+
throw new Error("--restore cannot be combined with --file/-f or --channels");
|
|
66
|
+
const prepared = await prepareRestore(cotalRoot(), values);
|
|
67
|
+
pendingRestores.set(prepared.attemptId, prepared);
|
|
68
|
+
const next = {
|
|
69
|
+
...values,
|
|
70
|
+
restore: undefined,
|
|
71
|
+
"restore-only": undefined,
|
|
72
|
+
"accept-missing-source": undefined,
|
|
73
|
+
__restoreAttempt: prepared.attemptId,
|
|
74
|
+
space: values.space ?? prepared.space,
|
|
75
|
+
server: values.server ?? prepared.server,
|
|
76
|
+
host: values.host ?? prepared.host,
|
|
77
|
+
runtime: values.runtime ?? prepared.runtime,
|
|
78
|
+
open: prepared.mode === "open",
|
|
79
|
+
"user-auth": prepared.mode === "user",
|
|
80
|
+
};
|
|
81
|
+
try {
|
|
82
|
+
await up({ ...args, values: next });
|
|
83
|
+
}
|
|
84
|
+
catch (error) {
|
|
85
|
+
pendingRestores.delete(prepared.attemptId);
|
|
86
|
+
markPreparedRestoreDegraded(prepared.root, prepared.attemptId, error.message);
|
|
87
|
+
throw error;
|
|
88
|
+
}
|
|
89
|
+
return;
|
|
90
|
+
}
|
|
91
|
+
if (values["restore-only"] || values["accept-missing-source"])
|
|
92
|
+
throw new Error("--restore-only and --accept-missing-source require --restore <dir>");
|
|
93
|
+
if (!values.__restoreAttempt && !values.__ordinaryResumeAttempt && !values.file) {
|
|
94
|
+
const root = cotalRoot();
|
|
95
|
+
const lock = acquireMaintenanceLock(root);
|
|
96
|
+
let pending;
|
|
97
|
+
let recoveredRestore;
|
|
98
|
+
try {
|
|
99
|
+
const journal = readMaintenanceJournal(root);
|
|
100
|
+
if (journal?.state === "restore-ready") {
|
|
101
|
+
// Ordinary up NEVER rolls back an in-progress restore: a live attempt's target may still be
|
|
102
|
+
// written by its isolated broker. Refuse with the exact recovery recourse instead.
|
|
103
|
+
const attempt = journal.restore.attemptId;
|
|
104
|
+
const assessment = assessRestoreClaim(journal);
|
|
105
|
+
if (assessment === "live")
|
|
106
|
+
throw new Error(`cotal up is refused: restore attempt ${attempt} is in progress (claim live until ${journal.claim.deadline}); wait for it to complete or become provably stale`);
|
|
107
|
+
if (assessment === "ambiguous")
|
|
108
|
+
throw new Error(`cotal up is refused: restore attempt ${attempt} owners cannot be proven dead; inspect the recorded coordinator, watchdog, and broker processes before recovery`);
|
|
109
|
+
throw new Error(`cotal up is refused: stale restore attempt ${attempt} holds the store; roll it back with \`cotal clean restore-attempt --attempt ${attempt} --force\`, or recover it with \`cotal up --restore\``);
|
|
110
|
+
}
|
|
111
|
+
if (journal && (journal.state === "commit-intent" || journal.state === "manager-committed" || journal.state === "degraded")) {
|
|
112
|
+
recoveredRestore = rehydratePreparedRestore(root, journal);
|
|
113
|
+
pendingRestores.set(recoveredRestore.attemptId, recoveredRestore);
|
|
114
|
+
}
|
|
115
|
+
else if (journal?.state === "active") {
|
|
116
|
+
if (!isManagerCommittedRestore(journal))
|
|
117
|
+
throw new Error(`restore attempt ${journal.restore.attemptId} is active without durable manager commit evidence`);
|
|
118
|
+
if (!journal.listenerProof)
|
|
119
|
+
throw new Error(`restore attempt ${journal.restore.attemptId} is active without a bound listener proof`);
|
|
120
|
+
const status = localProcessOwnerStatus(journal.listenerProof.processOwner);
|
|
121
|
+
if (status === "unknown")
|
|
122
|
+
throw new Error(`restore attempt ${journal.restore.attemptId} active listener ownership is ambiguous; refusing ordinary startup`);
|
|
123
|
+
if (status === "alive") {
|
|
124
|
+
recoveredRestore = rehydratePreparedRestore(root, journal);
|
|
125
|
+
pendingRestores.set(recoveredRestore.attemptId, recoveredRestore);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
else if (journal?.state === "ready") {
|
|
129
|
+
const resume = readMaintenanceResumeDocument(root, journal.resume);
|
|
130
|
+
const attemptId = `resume-${randomUUID()}`;
|
|
131
|
+
const launch = resume.launch;
|
|
132
|
+
const agents = resume.inventory.agents ?? [];
|
|
133
|
+
const runtimes = [...new Set(agents.map((agent) => agent.launch?.runtime).filter((value) => typeof value === "string"))];
|
|
134
|
+
if (runtimes.length > 1)
|
|
135
|
+
throw new Error(`resume inventory requires multiple runtimes: ${runtimes.join(", ")}`);
|
|
136
|
+
const resumeServer = values.server ?? (typeof launch.server === "string" ? launch.server : DEFAULT_SERVER);
|
|
137
|
+
const requestedStore = values["store-dir"] ?? (typeof launch.storeDir === "string" ? launch.storeDir : journal.source.path);
|
|
138
|
+
// Ordinary up resumes the exact preserved source: a different valid store passed via
|
|
139
|
+
// --store-dir must not consume this maintenance cut under the recorded space's trust.
|
|
140
|
+
if (!sameStoreIdentity(readStoreIdentity(resolve(requestedStore)), journal.source))
|
|
141
|
+
throw new Error(`--store-dir ${requestedStore} is not the preserved source store; ordinary up resumes exactly ${journal.source.path}`);
|
|
142
|
+
// Journal the CANONICAL identity-checked path, never the caller's spelling: a relative or
|
|
143
|
+
// symlinked spelling re-resolved later (other cwd, retargeted link) could open another store.
|
|
144
|
+
const resumeStore = journal.source.path;
|
|
145
|
+
if (values.runtime && runtimes[0] && values.runtime !== runtimes[0])
|
|
146
|
+
throw new Error(`--runtime ${values.runtime} contradicts the preserved agent runtime ${runtimes[0]}; omit it to resume the same principals`);
|
|
147
|
+
const resumeRuntime = values.runtime ?? runtimes[0] ?? "pty";
|
|
148
|
+
const serverNonce = randomUUID().replaceAll("-", "");
|
|
149
|
+
const serverName = `${attemptId}-${serverNonce}`;
|
|
150
|
+
beginOrdinaryResume(lock, {
|
|
151
|
+
attemptId,
|
|
152
|
+
launch: {
|
|
153
|
+
server: resumeServer,
|
|
154
|
+
storeDir: resumeStore,
|
|
155
|
+
runtime: resumeRuntime,
|
|
156
|
+
detached: Boolean(values.detach),
|
|
157
|
+
serverName,
|
|
158
|
+
serverNonce,
|
|
159
|
+
},
|
|
160
|
+
});
|
|
161
|
+
pending = {
|
|
162
|
+
root,
|
|
163
|
+
attemptId,
|
|
164
|
+
space: journal.space,
|
|
165
|
+
mode: journal.mode,
|
|
166
|
+
server: resumeServer,
|
|
167
|
+
storeDir: resumeStore,
|
|
168
|
+
runtime: resumeRuntime,
|
|
169
|
+
detached: Boolean(values.detach),
|
|
170
|
+
inventory: resume.inventory,
|
|
171
|
+
journalState: "resume-intent",
|
|
172
|
+
serverName,
|
|
173
|
+
serverNonce,
|
|
174
|
+
};
|
|
175
|
+
pendingOrdinaryResumes.set(attemptId, pending);
|
|
176
|
+
}
|
|
177
|
+
else if (journal && (journal.state === "resume-intent" || journal.state === "resume-active" || journal.state === "resume-committed" || journal.state === "resume-degraded")) {
|
|
178
|
+
const resume = readMaintenanceResumeDocument(root, journal.resume);
|
|
179
|
+
const launch = journal.ordinaryResume.launch;
|
|
180
|
+
const attemptId = journal.ordinaryResume.attemptId;
|
|
181
|
+
// Recovery re-asserts the source identity rather than trusting the journaled spelling.
|
|
182
|
+
assertStoreIdentity(journal.source);
|
|
183
|
+
// The bound listener decides re-entry: a live exact listener is ADOPTED, a provably dead
|
|
184
|
+
// one is durably retired before a fresh spawn, ambiguity refuses, and a manager-committed
|
|
185
|
+
// resume never replaces the listener its durable token is bound to.
|
|
186
|
+
let adoptProof;
|
|
187
|
+
let recoveredProof = journal.listenerProof;
|
|
188
|
+
let recoveredState = journal.state;
|
|
189
|
+
if (recoveredProof) {
|
|
190
|
+
const status = localProcessOwnerStatus(recoveredProof.processOwner);
|
|
191
|
+
if (status === "unknown")
|
|
192
|
+
throw new Error(`resume attempt ${attemptId} listener ownership is ambiguous; refusing replacement`);
|
|
193
|
+
if (status === "alive") {
|
|
194
|
+
adoptProof = recoveredProof;
|
|
195
|
+
}
|
|
196
|
+
else if (journal.state === "resume-committed") {
|
|
197
|
+
throw new Error(`resume attempt ${attemptId} is manager-committed but its bound listener is dead; preserving the commit token and retained suppression`);
|
|
198
|
+
}
|
|
199
|
+
else {
|
|
200
|
+
if (journal.state === "resume-active") {
|
|
201
|
+
markOrdinaryResumeDegraded(lock, "bound resume listener died after activation", [{
|
|
202
|
+
action: "repair",
|
|
203
|
+
description: "Retire the dead listener and re-run the same-principal activation.",
|
|
204
|
+
paths: [journal.source.path],
|
|
205
|
+
}]);
|
|
206
|
+
recoveredState = "resume-degraded";
|
|
207
|
+
}
|
|
208
|
+
replaceDeadOrdinaryResumeListener(lock, recoveredProof);
|
|
209
|
+
recoveredProof = undefined;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
const freshNonce = randomUUID().replaceAll("-", "");
|
|
213
|
+
const serverNonce = adoptProof?.serverNonce ??
|
|
214
|
+
(recoveredProof || typeof launch.serverNonce !== "string" || journal.listenerReplacements?.length
|
|
215
|
+
? freshNonce
|
|
216
|
+
: launch.serverNonce);
|
|
217
|
+
const serverName = adoptProof?.serverName ?? `${attemptId}-${serverNonce}`;
|
|
218
|
+
pending = {
|
|
219
|
+
root,
|
|
220
|
+
attemptId,
|
|
221
|
+
space: journal.space,
|
|
222
|
+
mode: journal.mode,
|
|
223
|
+
server: typeof launch.server === "string" ? launch.server : DEFAULT_SERVER,
|
|
224
|
+
storeDir: journal.source.path,
|
|
225
|
+
runtime: typeof launch.runtime === "string" ? launch.runtime : "pty",
|
|
226
|
+
detached: launch.detached === true,
|
|
227
|
+
inventory: resume.inventory,
|
|
228
|
+
journalState: recoveredState,
|
|
229
|
+
managerCommit: journal.state === "resume-committed" ? journal.managerCommit : undefined,
|
|
230
|
+
serverName,
|
|
231
|
+
serverNonce,
|
|
232
|
+
...(adoptProof ? { adoptProof } : {}),
|
|
233
|
+
};
|
|
234
|
+
pendingOrdinaryResumes.set(attemptId, pending);
|
|
235
|
+
}
|
|
236
|
+
else if (journal?.state === "resume-retired") {
|
|
237
|
+
consumeRetiredMaintenance(lock);
|
|
238
|
+
}
|
|
239
|
+
else if (journal) {
|
|
240
|
+
throw new Error(`cotal up is refused while maintenance state is ${journal.state}; follow the recorded recovery`);
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
finally {
|
|
244
|
+
releaseMaintenanceLock(lock);
|
|
245
|
+
}
|
|
246
|
+
if (recoveredRestore) {
|
|
247
|
+
try {
|
|
248
|
+
await up({
|
|
249
|
+
...args,
|
|
250
|
+
values: {
|
|
251
|
+
...values,
|
|
252
|
+
__restoreAttempt: recoveredRestore.attemptId,
|
|
253
|
+
space: recoveredRestore.space,
|
|
254
|
+
server: recoveredRestore.server,
|
|
255
|
+
host: recoveredRestore.host,
|
|
256
|
+
"store-dir": recoveredRestore.targetPath,
|
|
257
|
+
runtime: recoveredRestore.runtime,
|
|
258
|
+
detach: recoveredRestore.detached,
|
|
259
|
+
open: recoveredRestore.mode === "open",
|
|
260
|
+
"user-auth": recoveredRestore.mode === "user",
|
|
261
|
+
},
|
|
262
|
+
});
|
|
263
|
+
}
|
|
264
|
+
catch (error) {
|
|
265
|
+
markPendingResumeDegraded(recoveredRestore.attemptId, error instanceof Error ? error.message : String(error));
|
|
266
|
+
throw error;
|
|
267
|
+
}
|
|
268
|
+
return;
|
|
269
|
+
}
|
|
270
|
+
if (pending) {
|
|
271
|
+
try {
|
|
272
|
+
await up({
|
|
273
|
+
...args,
|
|
274
|
+
values: {
|
|
275
|
+
...values,
|
|
276
|
+
__ordinaryResumeAttempt: pending.attemptId,
|
|
277
|
+
space: pending.space,
|
|
278
|
+
server: pending.server,
|
|
279
|
+
"store-dir": pending.storeDir,
|
|
280
|
+
runtime: pending.runtime,
|
|
281
|
+
detach: pending.detached,
|
|
282
|
+
open: pending.mode === "open",
|
|
283
|
+
"user-auth": pending.mode === "user",
|
|
284
|
+
},
|
|
285
|
+
});
|
|
286
|
+
}
|
|
287
|
+
catch (error) {
|
|
288
|
+
markPendingResumeDegraded(pending.attemptId, error instanceof Error ? error.message : String(error));
|
|
289
|
+
throw error;
|
|
290
|
+
}
|
|
291
|
+
return;
|
|
292
|
+
}
|
|
293
|
+
}
|
|
49
294
|
const wantUser = Boolean(values["user-auth"]);
|
|
50
295
|
// The auth-mode flags contradict each other loudly, never silently (a user-auth space quietly
|
|
51
296
|
// started open would run agents on the wrong identity plane — the exact failure per-user-auth
|
|
52
297
|
// exists to prevent).
|
|
53
298
|
if (wantUser && values.open) {
|
|
54
|
-
|
|
55
|
-
process.exit(1);
|
|
299
|
+
throw new Error("--user-auth and --open contradict - a user-auth space cannot run unauthenticated");
|
|
56
300
|
}
|
|
57
301
|
if (values.idp && !wantUser && !values.file) {
|
|
58
|
-
|
|
59
|
-
process.exit(1);
|
|
302
|
+
throw new Error('--idp is for user-auth spaces; pair it with --user-auth, or set broker.auth: "user" in a manifest');
|
|
60
303
|
}
|
|
61
304
|
// `up -f cotal.yaml` is a distinct path: bring up a FRESH mesh described by a manifest (broker +
|
|
62
305
|
// channels + booted agents). It owns the whole space; deploying onto a RUNNING mesh is `spawn -f`.
|
|
63
306
|
// CLI flags override the manifest (flag > manifest > default) so the same file runs at a different
|
|
64
307
|
// port / runtime / space / auth without editing it.
|
|
65
308
|
if (values.file) {
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
309
|
+
if (values["dry-run"]) {
|
|
310
|
+
await upManifest(values.file, {
|
|
311
|
+
dryRun: true,
|
|
312
|
+
server: values.server,
|
|
313
|
+
host: values.host,
|
|
314
|
+
space: values.space,
|
|
315
|
+
runtime: values.runtime,
|
|
316
|
+
open: values.open,
|
|
317
|
+
userAuth: wantUser,
|
|
318
|
+
idp: values.idp,
|
|
319
|
+
});
|
|
320
|
+
return;
|
|
321
|
+
}
|
|
322
|
+
const lock = acquireMaintenanceLock(cotalRoot());
|
|
323
|
+
try {
|
|
324
|
+
assertOrdinaryUpAllowed(cotalRoot());
|
|
325
|
+
await upManifest(values.file, {
|
|
326
|
+
dryRun: Boolean(values["dry-run"]),
|
|
327
|
+
server: values.server,
|
|
328
|
+
host: values.host,
|
|
329
|
+
space: values.space,
|
|
330
|
+
runtime: values.runtime,
|
|
331
|
+
open: values.open,
|
|
332
|
+
userAuth: wantUser,
|
|
333
|
+
idp: values.idp,
|
|
334
|
+
});
|
|
335
|
+
}
|
|
336
|
+
finally {
|
|
337
|
+
releaseMaintenanceLock(lock);
|
|
338
|
+
}
|
|
76
339
|
return;
|
|
77
340
|
}
|
|
78
341
|
// `--runtime <name>` selects the backend the mesh's manager spawns agents through. The `-f` path
|
|
@@ -82,182 +345,852 @@ export async function up(args) {
|
|
|
82
345
|
// recourse) instead of a silent fallback in a detached child. No fallbacks. (`pty`/unset: no-op.)
|
|
83
346
|
if (values.runtime)
|
|
84
347
|
await preflightRuntime(values.runtime);
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
348
|
+
const resumeAttempt = values.__restoreAttempt ?? values.__ordinaryResumeAttempt;
|
|
349
|
+
let startupLock = resumeAttempt ? undefined : acquireMaintenanceLock(cotalRoot());
|
|
350
|
+
const releaseStartupLock = () => {
|
|
351
|
+
if (!startupLock)
|
|
352
|
+
return;
|
|
353
|
+
releaseMaintenanceLock(startupLock);
|
|
354
|
+
startupLock = undefined;
|
|
355
|
+
};
|
|
356
|
+
try {
|
|
357
|
+
if (startupLock)
|
|
358
|
+
assertOrdinaryUpAllowed(cotalRoot(), values["store-dir"] ? resolve(values["store-dir"]) : cotalPath("nats"));
|
|
359
|
+
let server = values.server ?? DEFAULT_SERVER;
|
|
360
|
+
const host = values.host ?? "127.0.0.1";
|
|
361
|
+
const restoredAttempt = resumeAttempt ? pendingRestores.get(resumeAttempt) : undefined;
|
|
362
|
+
if (restoredAttempt?.reentry) {
|
|
363
|
+
if (!restoredAttempt.listenerProof)
|
|
364
|
+
throw new Error(`restore attempt ${resumeAttempt} re-entry has no bound listener proof; preserving recovery state`);
|
|
365
|
+
const ownerStatus = localProcessOwnerStatus(restoredAttempt.listenerProof.processOwner);
|
|
366
|
+
if (ownerStatus === "alive") {
|
|
367
|
+
await resumeProvenRestoreListener(restoredAttempt);
|
|
368
|
+
return;
|
|
105
369
|
}
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
370
|
+
if (ownerStatus === "unknown")
|
|
371
|
+
throw new Error(`restore attempt ${resumeAttempt} listener ownership is ambiguous; refusing replacement`);
|
|
372
|
+
if (restoredAttempt.managerCommit)
|
|
373
|
+
throw new Error(`restore attempt ${resumeAttempt} is manager-committed but its bound listener is dead; preserving the commit token and retained suppression`);
|
|
374
|
+
if (await isReachable(restoredAttempt.server))
|
|
375
|
+
throw new Error(`restore attempt ${resumeAttempt} refuses the occupied foreign listener at ${restoredAttempt.server}`);
|
|
376
|
+
replacePreparedDeadRestoreListener(restoredAttempt);
|
|
377
|
+
}
|
|
378
|
+
const ordinaryAttempt = resumeAttempt ? pendingOrdinaryResumes.get(resumeAttempt) : undefined;
|
|
379
|
+
if (ordinaryAttempt?.adoptProof) {
|
|
380
|
+
// The recovered attempt's exact bound listener is alive: prove it over the wire and adopt it
|
|
381
|
+
// instead of spawning a competitor over the same store.
|
|
382
|
+
await resumeProvenOrdinaryListener(ordinaryAttempt);
|
|
383
|
+
return;
|
|
384
|
+
}
|
|
385
|
+
if (ordinaryAttempt?.journalState === "resume-committed")
|
|
386
|
+
throw new Error(`resume attempt ${resumeAttempt} is manager-committed but has no adoptable bound listener; preserving the commit token and retained suppression`);
|
|
387
|
+
const listenerReachable = await isReachable(server);
|
|
388
|
+
if (resumeAttempt && listenerReachable)
|
|
389
|
+
throw new Error(`resume attempt ${resumeAttempt} refuses the unproven occupied listener at ${server}`);
|
|
390
|
+
if (listenerReachable) {
|
|
391
|
+
const space = values.space ?? resolveSpace(process.cwd());
|
|
392
|
+
const root = cotalRoot();
|
|
393
|
+
// A broker is already on this port. Same root means "this project is already up" unless the
|
|
394
|
+
// operator explicitly asked for a second space in the same `.cotal/` root (unsupported today: pid,
|
|
395
|
+
// auth, and logs are root-scoped). Different root / unrecorded broker on the implicit default port
|
|
396
|
+
// gets a fresh free port instead of making the user hunt for one.
|
|
397
|
+
const held = loadMeshes().find((m) => m.server === server);
|
|
398
|
+
if (held && held.root === root && (held.space === space || values.space === undefined)) {
|
|
399
|
+
// A refresh of the SAME already-running mesh — its mode is fixed by how the live broker was
|
|
400
|
+
// started. A flag asking for a DIFFERENT mode must fail loud (silently preserving the old
|
|
401
|
+
// mode would hand the operator a mesh on the wrong identity plane); a bare refresh keeps the
|
|
402
|
+
// held mode.
|
|
403
|
+
const requested = wantUser ? "user" : values.open ? "open" : undefined;
|
|
404
|
+
if (requested && requested !== held.mode) {
|
|
405
|
+
const label = { auth: "static JWT auth", open: "no auth (--open)", user: "per-user auth" }[held.mode];
|
|
406
|
+
console.error(c.red(`✗ mesh "${held.space}" is already running at ${server} with ${label} - a running broker can't change auth mode; \`cotal down\` it first, then \`cotal up ${wantUser ? "--user-auth" : "--open"}\``));
|
|
116
407
|
process.exit(1);
|
|
117
408
|
}
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
409
|
+
console.log(c.green(`✓ mesh "${held.space}" already running at ${server}`));
|
|
410
|
+
// USER MODE: re-upping IS the documented recovery for a dead auth service (the provider's
|
|
411
|
+
// failure copy says "restart it with `cotal up`"), so a refresh must re-ensure the service —
|
|
412
|
+
// never just reprint "already running" over a dead callout. No broker config is (re)written
|
|
413
|
+
// here, so healing on a bare `cotal up` is safe: the mode can't drift, only the daemon heals.
|
|
414
|
+
let userAuth = held.userAuth;
|
|
415
|
+
if (held.mode === "user") {
|
|
416
|
+
const auth = loadSpaceAuth(authDir(root));
|
|
417
|
+
if (!auth) {
|
|
418
|
+
console.error(c.red(`✗ mesh "${held.space}" is user-auth but this root has no trust material under ${authDir(root)} - \`cotal down\` it, restore or re-provision \`.cotal/auth\`, then \`cotal up --user-auth\``));
|
|
419
|
+
process.exit(1);
|
|
420
|
+
}
|
|
421
|
+
const stateDir = userAuthStateDir(root, held.space);
|
|
422
|
+
let prepared;
|
|
423
|
+
try {
|
|
424
|
+
prepared = await resolveAuthProvider().prepareServer({
|
|
425
|
+
space: held.space,
|
|
426
|
+
operatorSeed: auth.operator.seed,
|
|
427
|
+
account: { pub: auth.account.pub, signingSeed: auth.account.signingSeed },
|
|
428
|
+
store: workspaceSecretStore(root),
|
|
429
|
+
dir: stateDir,
|
|
430
|
+
idpUrl: values.idp,
|
|
431
|
+
});
|
|
432
|
+
}
|
|
433
|
+
catch (e) {
|
|
434
|
+
console.error(c.red(`✗ ${e.message}`));
|
|
435
|
+
process.exit(1);
|
|
436
|
+
}
|
|
437
|
+
const svc = await startUserAuthService(held.space, server, { prepared, stateDir });
|
|
438
|
+
userAuth = svc.userAuth;
|
|
439
|
+
// The refresh IS the recovery command — a heal that didn't heal must not exit 0.
|
|
440
|
+
if (!svc.ok)
|
|
441
|
+
process.exitCode = 1;
|
|
128
442
|
}
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
443
|
+
// Auth/user meshes also need their resident renewal owner. A same-root refresh is the normal
|
|
444
|
+
// repair command after a stale/missing manager, so ensure the delivery daemon + manager before
|
|
445
|
+
// claiming the running mesh is healthy. Open meshes have no auth creds or delivery daemon.
|
|
446
|
+
// Re-ensure the control plane on a refresh. The manager is ensured for every mode that reaches
|
|
447
|
+
// here (a heal after a dead/missing manager adopts `--runtime`); the delivery daemon self-gates
|
|
448
|
+
// to auth mode inside `ensureControlPlane`. Open meshes normally skip this (a bare refresh has
|
|
449
|
+
// nothing to heal that must be touched), but a `--runtime` request must be honored there too,
|
|
450
|
+
// not silently dropped.
|
|
451
|
+
if (held.mode !== "open" || values.runtime) {
|
|
452
|
+
// Warn only when the manager is genuinely REUSED: a live delivery-aware (this-build) manager
|
|
453
|
+
// is kept as-is by `ensureManager`, so its runtime can't change. An old hosting manager (no
|
|
454
|
+
// delivery marker) is stopped and REPLACED by the ensure below carrying the requested runtime,
|
|
455
|
+
// so that's not a reuse - don't claim the runtime is fixed. A dead/absent manager is (re)started
|
|
456
|
+
// with it.
|
|
457
|
+
if (values.runtime && managerUp() && managerHasDeliveryMarker())
|
|
458
|
+
console.error(c.dim(`! manager already running for "${held.space}" - its runtime is fixed at start; \`cotal down\` then \`cotal up --runtime ${values.runtime}\` to change it`));
|
|
459
|
+
const controlPlane = await startDeliveryWithBroker(held.space, server, { runtime: values.runtime });
|
|
460
|
+
if (!controlPlane)
|
|
461
|
+
process.exitCode = 1;
|
|
132
462
|
}
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
463
|
+
recordOurMesh({ space: held.space, server, root, mode: held.mode, ...(userAuth ? { userAuth } : {}), ts: new Date().toISOString() });
|
|
464
|
+
return;
|
|
465
|
+
}
|
|
466
|
+
const who = held ? `mesh "${held.space}" (${held.root})` : "a broker not started here";
|
|
467
|
+
if (values.server === undefined && (!held || held.root !== root)) {
|
|
468
|
+
const next = await serverWithFreePort(server, host);
|
|
469
|
+
console.log(c.dim(`${server} is already in use by ${who}; starting "${space}" at ${next} instead`));
|
|
470
|
+
server = next;
|
|
138
471
|
}
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
// Re-ensure the control plane on a refresh. The manager is ensured for every mode that reaches
|
|
143
|
-
// here (a heal after a dead/missing manager adopts `--runtime`); the delivery daemon self-gates
|
|
144
|
-
// to auth mode inside `ensureControlPlane`. Open meshes normally skip this (a bare refresh has
|
|
145
|
-
// nothing to heal that must be touched), but a `--runtime` request must be honored there too,
|
|
146
|
-
// not silently dropped.
|
|
147
|
-
if (held.mode !== "open" || values.runtime) {
|
|
148
|
-
// Warn only when the manager is genuinely REUSED: a live delivery-aware (this-build) manager
|
|
149
|
-
// is kept as-is by `ensureManager`, so its runtime can't change. An old hosting manager (no
|
|
150
|
-
// delivery marker) is stopped and REPLACED by the ensure below carrying the requested runtime,
|
|
151
|
-
// so that's not a reuse - don't claim the runtime is fixed. A dead/absent manager is (re)started
|
|
152
|
-
// with it.
|
|
153
|
-
if (values.runtime && managerUp() && managerHasDeliveryMarker())
|
|
154
|
-
console.error(c.dim(`! manager already running for "${held.space}" - its runtime is fixed at start; \`cotal down\` then \`cotal up --runtime ${values.runtime}\` to change it`));
|
|
155
|
-
const controlPlane = await startDeliveryWithBroker(held.space, server, { runtime: values.runtime });
|
|
156
|
-
if (!controlPlane)
|
|
157
|
-
process.exitCode = 1;
|
|
472
|
+
else {
|
|
473
|
+
console.error(c.red(`✗ ${server} is already in use by ${who} - to run "${space}" use \`--server nats://${host}:<port>\` with a free port`));
|
|
474
|
+
process.exit(1);
|
|
158
475
|
}
|
|
159
|
-
|
|
476
|
+
}
|
|
477
|
+
if (values.detach) {
|
|
478
|
+
const restored = resumeAttempt ? pendingRestores.get(resumeAttempt) : undefined;
|
|
479
|
+
const { pid, source, authService, controlPlane, delivery, manager } = await startMeshDetached({
|
|
480
|
+
server,
|
|
481
|
+
storeDir: values["store-dir"],
|
|
482
|
+
space: values.space,
|
|
483
|
+
open: values.open,
|
|
484
|
+
userAuth: wantUser ? { idpUrl: values.idp } : undefined,
|
|
485
|
+
channels: values.channels,
|
|
486
|
+
host,
|
|
487
|
+
runtime: values.runtime,
|
|
488
|
+
resumeAttempt,
|
|
489
|
+
resumeCommitToken: restored?.managerCommit?.durableCommitToken ?? ordinaryAttempt?.managerCommit?.durableCommitToken,
|
|
490
|
+
...(restored ? {
|
|
491
|
+
boundListener: {
|
|
492
|
+
serverName: restored.serverName,
|
|
493
|
+
serverNonce: restored.serverNonce,
|
|
494
|
+
onSpawn: (pid, startedAt) => bindSpawnedRestoreListener(restored, pid, startedAt),
|
|
495
|
+
verify: async () => { await provePreparedRestoreListener(restored); },
|
|
496
|
+
},
|
|
497
|
+
} : ordinaryAttempt ? {
|
|
498
|
+
boundListener: {
|
|
499
|
+
serverName: ordinaryAttempt.serverName,
|
|
500
|
+
serverNonce: ordinaryAttempt.serverNonce,
|
|
501
|
+
onSpawn: (pid, startedAt) => bindSpawnedOrdinaryResumeListener(ordinaryAttempt, pid, startedAt),
|
|
502
|
+
verify: async () => { await verifySpawnedOrdinaryListener(ordinaryAttempt); },
|
|
503
|
+
},
|
|
504
|
+
} : {}),
|
|
505
|
+
skipPostStart: Boolean(resumeAttempt),
|
|
506
|
+
});
|
|
507
|
+
console.log(c.dim(`Started nats-server (${source}).`));
|
|
508
|
+
console.log(c.green(renderDetachedSummary({ pid, delivery, authService: wantUser && authService, manager })));
|
|
509
|
+
if (restored && process.env.COTAL_SMOKE_FAIL_AFTER_RESTORE_LISTENER_READY === "1")
|
|
510
|
+
throw new Error("smoke-injected failure after restore listener readiness");
|
|
511
|
+
// A user mesh whose auth service never became ready is recorded + running (a re-`cotal up`
|
|
512
|
+
// heals it), but this `up` did NOT deliver what was asked — automation must see that in the
|
|
513
|
+
// exit code, not only in the red line above.
|
|
514
|
+
if (!authService)
|
|
515
|
+
process.exitCode = 1;
|
|
516
|
+
await completeResumeActivation(resumeAttempt, controlPlane && authService, !authService ? "normal listener started but the user-auth service is unavailable" : "normal listener started but the control plane is degraded", server);
|
|
160
517
|
return;
|
|
161
518
|
}
|
|
162
|
-
const
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
519
|
+
const useAuth = !values.open;
|
|
520
|
+
const space = values.space ?? resolveSpace(process.cwd());
|
|
521
|
+
ensureRootForSpace(useAuth, space); // may pin the cwd as this space's root — before any cotalPath use
|
|
522
|
+
refuseOpenOverUserState(Boolean(values.open), space);
|
|
523
|
+
const storeDir = values["store-dir"] ? resolve(values["store-dir"]) : cotalPath("nats");
|
|
524
|
+
if (values.__ordinaryResumeAttempt) {
|
|
525
|
+
// Final identity re-assertion immediately before JetStream opens: the journaled canonical path
|
|
526
|
+
// must still be the exact preserved inode (no symlink retarget or replacement since intent).
|
|
527
|
+
const resumeJournal = readMaintenanceJournal(cotalRoot());
|
|
528
|
+
if (!resumeJournal || !("ordinaryResume" in resumeJournal))
|
|
529
|
+
throw new Error(`resume attempt ${values.__ordinaryResumeAttempt} lost its durable journal before the store open`);
|
|
530
|
+
if (!sameStoreIdentity(readStoreIdentity(storeDir), resumeJournal.source))
|
|
531
|
+
throw new Error(`resume store ${storeDir} no longer matches the preserved source identity; refusing to open JetStream over it`);
|
|
167
532
|
}
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
533
|
+
mkdirSync(storeDir, { recursive: true });
|
|
534
|
+
await claimSpace(space, server, cotalRoot());
|
|
535
|
+
const seedFile = loadChannelsFile(values.channels);
|
|
536
|
+
const setup = useAuth ? await authSetup(storeDir, server, space, host, wantUser ? { idpUrl: values.idp } : undefined) : undefined;
|
|
537
|
+
const port = Number(new URL(server).port) || 4222;
|
|
538
|
+
const restored = resumeAttempt ? pendingRestores.get(resumeAttempt) : undefined;
|
|
539
|
+
const natsArgs = [
|
|
540
|
+
...(setup ? ["-c", setup.confPath] : ["-js", "-sd", storeDir, "-p", String(port), "-a", host]),
|
|
541
|
+
...(restored ? ["--name", restored.serverName]
|
|
542
|
+
: ordinaryAttempt ? ["--name", ordinaryAttempt.serverName]
|
|
543
|
+
: []),
|
|
544
|
+
];
|
|
545
|
+
const { bin, source } = await resolveNatsServer();
|
|
546
|
+
console.log(c.dim(`Starting nats-server (JetStream, ${useAuth ? "JWT auth" : "OPEN/no-auth"}, ${source}) - store: ${storeDir}, bind: ${host}`));
|
|
547
|
+
console.log(c.dim("Press Ctrl-C to stop.\n"));
|
|
548
|
+
const listenerStartedAt = new Date().toISOString();
|
|
549
|
+
const child = spawn(bin, natsArgs, { stdio: "inherit" });
|
|
550
|
+
let activationFinished = !resumeAttempt;
|
|
551
|
+
if (child.pid)
|
|
552
|
+
writeFileSync(cotalPath("nats.pid"), String(child.pid));
|
|
553
|
+
if (restored && process.env.COTAL_SMOKE_EXIT_AFTER_RESTORE_LISTENER_SPAWN === "1")
|
|
554
|
+
process.exit(87);
|
|
555
|
+
if (restored)
|
|
556
|
+
try {
|
|
557
|
+
bindSpawnedRestoreListener(restored, child.pid ?? 0, listenerStartedAt);
|
|
558
|
+
}
|
|
559
|
+
catch (error) {
|
|
560
|
+
await stopUnboundRestoreListener(child);
|
|
561
|
+
removeMatchingNatsPid(child.pid ?? 0);
|
|
562
|
+
throw error;
|
|
563
|
+
}
|
|
564
|
+
if (ordinaryAttempt)
|
|
565
|
+
try {
|
|
566
|
+
bindSpawnedOrdinaryResumeListener(ordinaryAttempt, child.pid ?? 0, listenerStartedAt);
|
|
567
|
+
}
|
|
568
|
+
catch (error) {
|
|
569
|
+
await stopUnboundRestoreListener(child);
|
|
570
|
+
removeMatchingNatsPid(child.pid ?? 0);
|
|
571
|
+
throw error;
|
|
572
|
+
}
|
|
573
|
+
releaseStartupLock();
|
|
574
|
+
child.on("error", (err) => {
|
|
575
|
+
console.error(c.red(`Failed to start nats-server: ${err.message}`));
|
|
576
|
+
if (!resumeAttempt)
|
|
577
|
+
process.exit(1);
|
|
578
|
+
});
|
|
579
|
+
// The control plane is coupled to the broker: stop the delivery daemon AND the detached manager
|
|
580
|
+
// (AND the space's user-auth service) when this `up` stops (Ctrl-C), so none outlives the broker
|
|
581
|
+
// it serves — a surviving manager would reconnect-loop invisibly against the dead (or the NEXT)
|
|
582
|
+
// broker (the documented orphan-supervisor failure mode). All kill by pidfile, symmetric; the
|
|
583
|
+
// auth service's pid is space-scoped so no other space's daemon can ever be hit.
|
|
584
|
+
// stopDelivery is async (its creds delete goes through the secret store); the rest of the teardown
|
|
585
|
+
// must run even if it fails — the failure is logged, never swallowed silently, and the daemon kill
|
|
586
|
+
// itself happens inside stopDelivery's finally. Order preserved: delivery, manager, auth, broker.
|
|
587
|
+
const stop = () => {
|
|
588
|
+
void stopDelivery()
|
|
589
|
+
.catch((e) => console.error(`! delivery teardown: ${e.message}`))
|
|
590
|
+
.then(() => {
|
|
591
|
+
stopManager();
|
|
592
|
+
stopAuthService(space);
|
|
593
|
+
child.kill("SIGTERM");
|
|
594
|
+
});
|
|
595
|
+
};
|
|
596
|
+
process.on("SIGINT", stop);
|
|
597
|
+
process.on("SIGTERM", stop);
|
|
598
|
+
// The broker is gone — drop it from the registry (and the `current` pointer if it was the default)
|
|
599
|
+
// so a later `cotal spawn` doesn't try to join a dead mesh.
|
|
600
|
+
child.on("exit", async (code) => {
|
|
601
|
+
rmSync(cotalPath("nats.pid"), { force: true });
|
|
602
|
+
// Logged, never silently swallowed; the daemon kill runs in stopDelivery's finally regardless.
|
|
603
|
+
await stopDelivery().catch((e) => console.error(`! delivery teardown: ${e.message}`));
|
|
604
|
+
stopManager();
|
|
605
|
+
stopAuthService(space);
|
|
606
|
+
// Only unrecord if the registry still points at THIS broker. A newer broker for the same space
|
|
607
|
+
// (a concurrent `up`, or a different-port re-up that recorded after us) may have replaced our
|
|
608
|
+
// record — removing by name would clobber the live winner and hide it from the registry.
|
|
609
|
+
const mine = findMesh(space);
|
|
610
|
+
if (mine && mine.server === server && mine.root === cotalRoot()) {
|
|
611
|
+
removeMesh(space);
|
|
612
|
+
if (getCurrent() === space)
|
|
613
|
+
clearCurrent();
|
|
614
|
+
}
|
|
615
|
+
if (activationFinished)
|
|
616
|
+
process.exit(code ?? 0);
|
|
617
|
+
});
|
|
618
|
+
const ready = await waitReady(server, setup?.creds);
|
|
619
|
+
if (!ready) {
|
|
620
|
+
child.kill("SIGTERM");
|
|
621
|
+
const reason = `nats-server did not become ready at ${server}`;
|
|
622
|
+
markPendingResumeDegraded(resumeAttempt ?? "", reason);
|
|
623
|
+
throw new Error(reason);
|
|
624
|
+
}
|
|
625
|
+
if (restored)
|
|
626
|
+
await provePreparedRestoreListener(restored);
|
|
627
|
+
{
|
|
628
|
+
if (!resumeAttempt)
|
|
629
|
+
await postStart(server, space, setup, seedFile);
|
|
630
|
+
// USER MODE: the auth service comes up FIRST among the daemons — until its callout answers,
|
|
631
|
+
// every user-mode connect to this broker is denied, so `up` must not report a usable user mesh
|
|
632
|
+
// (nor let agents race it) on a half-started auth plane. (Foreground `up` doesn't exit here, so
|
|
633
|
+
// `ok` has no exit code to carry — the red consequence line above is the operator signal.)
|
|
634
|
+
const svc = await startUserAuthService(space, server, setup);
|
|
635
|
+
// Record BEFORE the control plane comes up: the manager's fail-closed mode detection requires
|
|
636
|
+
// an authoritative registry entry (marker-without-registry is a refused start, not a guess),
|
|
637
|
+
// so the record must exist by the time it boots. A manager/delivery failure after this leaves
|
|
638
|
+
// a recorded-but-degraded mesh — the documented, healable posture.
|
|
639
|
+
recordOurMesh({
|
|
640
|
+
space, server, root: cotalRoot(),
|
|
641
|
+
mode: setup?.prepared ? "user" : useAuth ? "auth" : "open",
|
|
642
|
+
...(svc.userAuth ? { userAuth: svc.userAuth } : {}),
|
|
643
|
+
ts: new Date().toISOString(),
|
|
644
|
+
});
|
|
645
|
+
// Bring up the delivery daemon WITH the server (auth mode only — it self-gates on `.cotal/auth`).
|
|
646
|
+
// It is part of the server, so `cotal up` starts it by default; open dev mode has no daemon.
|
|
647
|
+
// Class-2 credential renewal is NOT wired here: the MANAGER is the renewal owner (it is resident
|
|
648
|
+
// in every mesh mode — foreground, --detach, refresh — where this foreground process is not).
|
|
649
|
+
const controlPlane = await startDeliveryWithBroker(space, server, {
|
|
650
|
+
runtime: values.runtime,
|
|
651
|
+
resumeAttempt,
|
|
652
|
+
resumeCommitToken: restored?.managerCommit?.durableCommitToken ?? ordinaryAttempt?.managerCommit?.durableCommitToken,
|
|
653
|
+
});
|
|
654
|
+
if (restored && process.env.COTAL_SMOKE_FAIL_AFTER_RESTORE_LISTENER_READY === "1")
|
|
655
|
+
throw new Error("smoke-injected failure after restore listener readiness");
|
|
656
|
+
await completeResumeActivation(resumeAttempt, controlPlane && svc.ok, !svc.ok ? "normal listener started but the user-auth service is unavailable" : "normal listener started but the control plane is degraded", server);
|
|
657
|
+
activationFinished = true;
|
|
171
658
|
}
|
|
659
|
+
await new Promise(() => { });
|
|
172
660
|
}
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
661
|
+
finally {
|
|
662
|
+
releaseStartupLock();
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
function assertOrdinaryUpAllowed(root, storeDir) {
|
|
666
|
+
const maintenance = readMaintenanceJournal(root);
|
|
667
|
+
if (!maintenance)
|
|
668
|
+
return;
|
|
669
|
+
if (maintenance.state === "active") {
|
|
670
|
+
if (!isManagerCommittedRestore(maintenance))
|
|
671
|
+
throw new Error(`restore attempt ${maintenance.restore.attemptId} is active without durable manager commit evidence`);
|
|
672
|
+
if (!maintenance.listenerProof)
|
|
673
|
+
throw new Error(`restore attempt ${maintenance.restore.attemptId} is active without a bound listener proof`);
|
|
674
|
+
const status = localProcessOwnerStatus(maintenance.listenerProof.processOwner);
|
|
675
|
+
if (status !== "dead")
|
|
676
|
+
throw new Error(`restore attempt ${maintenance.restore.attemptId} active listener ownership is ${status}; refusing ordinary startup`);
|
|
677
|
+
// Relaunching after restore must serve the exact recorded active target, not whatever store the
|
|
678
|
+
// caller happens to name — the journal's provenance would otherwise describe a different mesh.
|
|
679
|
+
if (!storeDir)
|
|
680
|
+
throw new Error(`restore attempt ${maintenance.restore.attemptId} is active; relaunch with bare \`cotal up\` over the recorded target ${maintenance.restore.target.path}`);
|
|
681
|
+
if (!sameStoreIdentity(readStoreIdentity(resolve(storeDir)), maintenance.restore.target))
|
|
682
|
+
throw new Error(`ordinary startup after restore must use the recorded active target store ${maintenance.restore.target.path}, not ${storeDir}`);
|
|
191
683
|
return;
|
|
192
684
|
}
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
685
|
+
if (maintenance.state === "ready")
|
|
686
|
+
throw new Error("ordinary resume must begin through the attempt-bound startup path");
|
|
687
|
+
throw new Error(`cotal up is refused while maintenance state is ${maintenance.state}; follow the recorded restore recovery`);
|
|
688
|
+
}
|
|
689
|
+
function markPendingResumeDegraded(attemptId, reason) {
|
|
690
|
+
const ordinary = pendingOrdinaryResumes.get(attemptId);
|
|
691
|
+
if (ordinary) {
|
|
692
|
+
const lock = acquireMaintenanceLock(ordinary.root);
|
|
693
|
+
try {
|
|
694
|
+
const journal = readMaintenanceJournal(ordinary.root);
|
|
695
|
+
if (journal && (journal.state === "resume-intent" || journal.state === "resume-active"))
|
|
696
|
+
markOrdinaryResumeDegraded(lock, reason, [{
|
|
697
|
+
action: "repair",
|
|
698
|
+
description: "Preserve the store and retained resume inventory; repair forward, then retry the same-principal activation.",
|
|
699
|
+
paths: [journal.source.path],
|
|
700
|
+
}]);
|
|
701
|
+
}
|
|
702
|
+
finally {
|
|
703
|
+
releaseMaintenanceLock(lock);
|
|
704
|
+
}
|
|
705
|
+
return;
|
|
706
|
+
}
|
|
707
|
+
const restored = pendingRestores.get(attemptId);
|
|
708
|
+
if (restored)
|
|
709
|
+
markPreparedRestoreDegraded(restored.root, restored.attemptId, reason);
|
|
710
|
+
}
|
|
711
|
+
async function resumeControlAuth(root, mode) {
|
|
712
|
+
if (mode === "open")
|
|
713
|
+
return {};
|
|
714
|
+
const auth = loadSpaceAuth(authDir(root));
|
|
715
|
+
if (!auth)
|
|
716
|
+
throw new Error("same-principal resume requires the existing space trust material");
|
|
717
|
+
const identity = newIdentity();
|
|
718
|
+
return {
|
|
719
|
+
creds: await mintCreds(auth, identity, "control-caller-admin", {
|
|
720
|
+
expiresAt: Math.floor((Date.now() + 30 * 60 * 1000) / 1000),
|
|
721
|
+
}),
|
|
722
|
+
};
|
|
723
|
+
}
|
|
724
|
+
function restoreListenerOwner(pid, nonce, startedAt) {
|
|
725
|
+
if (!Number.isInteger(pid) || pid <= 0)
|
|
726
|
+
throw new Error("restore listener spawn returned no pid");
|
|
727
|
+
return { pid, host: hostname(), startedAt, id: `restore-listener-${nonce}` };
|
|
728
|
+
}
|
|
729
|
+
function removeMatchingNatsPid(pid) {
|
|
730
|
+
const path = cotalPath("nats.pid");
|
|
731
|
+
try {
|
|
732
|
+
const stat = lstatSync(path);
|
|
733
|
+
if (stat.isFile() && !stat.isSymbolicLink() && readFileSync(path, "utf8") === String(pid))
|
|
734
|
+
rmSync(path);
|
|
735
|
+
}
|
|
736
|
+
catch { /* absent, changed, or not owned by this spawn */ }
|
|
737
|
+
}
|
|
738
|
+
function waitForChildExit(child, timeoutMs) {
|
|
739
|
+
if (child.exitCode !== null || child.signalCode !== null)
|
|
740
|
+
return Promise.resolve(true);
|
|
741
|
+
return new Promise((resolveExit) => {
|
|
742
|
+
const onExit = () => finish(true);
|
|
743
|
+
const timer = setTimeout(() => finish(false), timeoutMs);
|
|
744
|
+
const finish = (exited) => {
|
|
745
|
+
clearTimeout(timer);
|
|
746
|
+
child.off("exit", onExit);
|
|
747
|
+
resolveExit(exited);
|
|
748
|
+
};
|
|
749
|
+
child.once("exit", onExit);
|
|
211
750
|
});
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
751
|
+
}
|
|
752
|
+
async function stopUnboundRestoreListener(child) {
|
|
753
|
+
if (child.exitCode !== null || child.signalCode !== null)
|
|
754
|
+
return;
|
|
755
|
+
child.kill("SIGTERM");
|
|
756
|
+
if (await waitForChildExit(child, 5_000))
|
|
757
|
+
return;
|
|
758
|
+
child.kill("SIGKILL");
|
|
759
|
+
if (!await waitForChildExit(child, 5_000))
|
|
760
|
+
throw new Error(`unbound restore listener process ${child.pid ?? "unknown"} did not exit`);
|
|
761
|
+
}
|
|
762
|
+
function bindSpawnedRestoreListener(prepared, pid, startedAt) {
|
|
763
|
+
bindRestoreListenerOwner(prepared, restoreListenerOwner(pid, prepared.serverNonce, startedAt));
|
|
764
|
+
}
|
|
765
|
+
function bindRestoreListenerOwner(prepared, processOwner) {
|
|
766
|
+
bindPreparedRestoreListener(prepared, processOwner);
|
|
767
|
+
if (process.env.COTAL_SMOKE_EXIT_AFTER_RESTORE_LISTENER_BIND === "1")
|
|
768
|
+
process.exit(86);
|
|
769
|
+
}
|
|
770
|
+
function sameProcessOwner(a, b) {
|
|
771
|
+
return a.pid === b.pid && a.host === b.host && a.startedAt === b.startedAt && a.id === b.id;
|
|
772
|
+
}
|
|
773
|
+
function sameRestoreListenerProof(a, b) {
|
|
774
|
+
return a.attemptId === b.attemptId && a.serverName === b.serverName &&
|
|
775
|
+
a.serverNonce === b.serverNonce && sameProcessOwner(a.processOwner, b.processOwner) &&
|
|
776
|
+
a.serverEndpoint === b.serverEndpoint && sameStoreIdentity(a.target, b.target);
|
|
777
|
+
}
|
|
778
|
+
function readNatsInfo(endpoint, timeoutMs = 2_000) {
|
|
779
|
+
return new Promise((resolveInfo, rejectInfo) => {
|
|
780
|
+
let settled = false;
|
|
781
|
+
let socket;
|
|
782
|
+
const finish = (error, info) => {
|
|
783
|
+
if (settled)
|
|
784
|
+
return;
|
|
785
|
+
settled = true;
|
|
786
|
+
socket?.destroy();
|
|
787
|
+
if (error)
|
|
788
|
+
rejectInfo(error);
|
|
789
|
+
else
|
|
790
|
+
resolveInfo(info);
|
|
791
|
+
};
|
|
792
|
+
let url;
|
|
793
|
+
try {
|
|
794
|
+
url = new URL(endpoint);
|
|
795
|
+
}
|
|
796
|
+
catch {
|
|
797
|
+
rejectInfo(new Error(`invalid restore listener endpoint ${endpoint}`));
|
|
798
|
+
return;
|
|
799
|
+
}
|
|
800
|
+
socket = createConnection({ host: url.hostname, port: Number(url.port) || 4222 });
|
|
801
|
+
socket.setTimeout(timeoutMs);
|
|
802
|
+
let input = "";
|
|
803
|
+
socket.on("data", (chunk) => {
|
|
804
|
+
input += chunk.toString("utf8");
|
|
805
|
+
if (input.length > 64 * 1024)
|
|
806
|
+
return finish(new Error("restore listener INFO exceeds 64 KiB"));
|
|
807
|
+
const newline = input.indexOf("\r\n");
|
|
808
|
+
if (newline < 0)
|
|
809
|
+
return;
|
|
810
|
+
const line = input.slice(0, newline);
|
|
811
|
+
const brace = line.indexOf("{");
|
|
812
|
+
if (!/^INFO\b/.test(line) || brace < 0)
|
|
813
|
+
return finish(new Error("restore listener did not present a NATS INFO greeting"));
|
|
814
|
+
try {
|
|
815
|
+
const info = JSON.parse(line.slice(brace));
|
|
816
|
+
if (!info || typeof info !== "object" || Array.isArray(info))
|
|
817
|
+
throw new Error("INFO is not an object");
|
|
818
|
+
finish(undefined, info);
|
|
819
|
+
}
|
|
820
|
+
catch (error) {
|
|
821
|
+
finish(new Error(`restore listener presented invalid NATS INFO: ${error.message}`));
|
|
822
|
+
}
|
|
823
|
+
});
|
|
824
|
+
socket.on("timeout", () => finish(new Error("restore listener INFO timed out")));
|
|
825
|
+
socket.on("error", (error) => finish(new Error(`restore listener INFO failed: ${error.message}`)));
|
|
826
|
+
socket.on("close", () => finish(new Error("restore listener closed before NATS INFO")));
|
|
236
827
|
});
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
828
|
+
}
|
|
829
|
+
async function provePreparedRestoreListener(prepared) {
|
|
830
|
+
const proof = prepared.listenerProof;
|
|
831
|
+
if (!proof)
|
|
832
|
+
throw new Error(`restore attempt ${prepared.attemptId} has no bound listener proof`);
|
|
833
|
+
if (proof.attemptId !== prepared.attemptId || proof.serverName !== prepared.serverName ||
|
|
834
|
+
proof.serverNonce !== prepared.serverNonce || proof.serverEndpoint !== prepared.server ||
|
|
835
|
+
proof.serverName !== `${proof.attemptId}-${proof.serverNonce}` || !/^[0-9a-f]{32}$/.test(proof.serverNonce) ||
|
|
836
|
+
proof.processOwner.id !== `restore-listener-${proof.serverNonce}`)
|
|
837
|
+
throw new Error(`restore attempt ${prepared.attemptId} listener proof does not match its launch record`);
|
|
838
|
+
const journal = readMaintenanceJournal(prepared.root);
|
|
839
|
+
if (!journal || (journal.state !== "commit-intent" && journal.state !== "manager-committed" && journal.state !== "degraded" && journal.state !== "active") ||
|
|
840
|
+
journal.restore.attemptId !== prepared.attemptId || !journal.listenerProof ||
|
|
841
|
+
!sameRestoreListenerProof(journal.listenerProof, proof) || journal.launch.server !== proof.serverEndpoint)
|
|
842
|
+
throw new Error(`restore attempt ${prepared.attemptId} listener proof does not exactly match durable recovery state`);
|
|
843
|
+
if (proof.processOwner.host !== hostname())
|
|
844
|
+
throw new Error(`restore attempt ${prepared.attemptId} listener process ownership is not live and local`);
|
|
845
|
+
try {
|
|
846
|
+
process.kill(proof.processOwner.pid, 0);
|
|
847
|
+
}
|
|
848
|
+
catch {
|
|
849
|
+
throw new Error(`restore attempt ${prepared.attemptId} listener process ownership is not live and local`);
|
|
850
|
+
}
|
|
851
|
+
const pidPath = join(prepared.root, ".cotal", "nats.pid");
|
|
852
|
+
let exactPidFile = false;
|
|
853
|
+
try {
|
|
854
|
+
const stat = lstatSync(pidPath);
|
|
855
|
+
exactPidFile = stat.isFile() && !stat.isSymbolicLink() &&
|
|
856
|
+
readFileSync(pidPath, "utf8") === String(proof.processOwner.pid);
|
|
857
|
+
}
|
|
858
|
+
catch { /* absent or unprovable */ }
|
|
859
|
+
if (!exactPidFile)
|
|
860
|
+
throw new Error(`restore attempt ${prepared.attemptId} listener pidfile does not match its process proof`);
|
|
861
|
+
const target = readStoreIdentity(prepared.targetPath);
|
|
862
|
+
if (!sameStoreIdentity(target, proof.target) || !sameStoreIdentity(journal.restore.target, proof.target))
|
|
863
|
+
throw new Error(`restore attempt ${prepared.attemptId} target identity changed`);
|
|
864
|
+
const mesh = findMesh(prepared.space);
|
|
865
|
+
if (mesh && (mesh.server !== prepared.server || mesh.root !== prepared.root || mesh.mode !== prepared.mode))
|
|
866
|
+
throw new Error(`restore attempt ${prepared.attemptId} conflicts with the recorded mesh identity`);
|
|
867
|
+
const info = await readNatsInfo(proof.serverEndpoint);
|
|
868
|
+
if (info.server_name !== proof.serverName)
|
|
869
|
+
throw new Error(`restore attempt ${prepared.attemptId} reached a foreign NATS server name/nonce`);
|
|
870
|
+
const auth = await resumeControlAuth(prepared.root, prepared.mode);
|
|
871
|
+
const nc = await connect({
|
|
872
|
+
servers: prepared.server,
|
|
873
|
+
...standaloneConnectOpts(auth),
|
|
874
|
+
maxReconnectAttempts: 0,
|
|
875
|
+
});
|
|
876
|
+
try {
|
|
877
|
+
if (nc.info?.server_name !== proof.serverName)
|
|
878
|
+
throw new Error(`restore attempt ${prepared.attemptId} authenticated to a foreign NATS server name/nonce`);
|
|
879
|
+
if (!nc.info.jetstream)
|
|
880
|
+
throw new Error(`restore attempt ${prepared.attemptId} listener has no JetStream`);
|
|
881
|
+
}
|
|
882
|
+
finally {
|
|
883
|
+
await nc.drain().catch(() => { });
|
|
884
|
+
}
|
|
885
|
+
return proof;
|
|
886
|
+
}
|
|
887
|
+
function bindSpawnedOrdinaryResumeListener(pending, pid, startedAt) {
|
|
888
|
+
if (!Number.isInteger(pid) || pid <= 0)
|
|
889
|
+
throw new Error("resume listener spawn returned no pid");
|
|
890
|
+
const lock = acquireMaintenanceLock(pending.root);
|
|
891
|
+
try {
|
|
892
|
+
const journal = readMaintenanceJournal(pending.root);
|
|
893
|
+
if (!journal || !("ordinaryResume" in journal) || journal.ordinaryResume.attemptId !== pending.attemptId)
|
|
894
|
+
throw new Error(`resume listener bind does not match attempt ${pending.attemptId}`);
|
|
895
|
+
bindOrdinaryResumeListener(lock, {
|
|
896
|
+
attemptId: pending.attemptId,
|
|
897
|
+
serverName: pending.serverName,
|
|
898
|
+
serverNonce: pending.serverNonce,
|
|
899
|
+
processOwner: { pid, host: hostname(), startedAt, id: `resume-listener-${pending.serverNonce}` },
|
|
900
|
+
serverEndpoint: pending.server,
|
|
901
|
+
target: journal.source,
|
|
253
902
|
});
|
|
254
|
-
// Bring up the delivery daemon WITH the server (auth mode only — it self-gates on `.cotal/auth`).
|
|
255
|
-
// It is part of the server, so `cotal up` starts it by default; open dev mode has no daemon.
|
|
256
|
-
// Class-2 credential renewal is NOT wired here: the MANAGER is the renewal owner (it is resident
|
|
257
|
-
// in every mesh mode — foreground, --detach, refresh — where this foreground process is not).
|
|
258
|
-
await startDeliveryWithBroker(space, server, { runtime: values.runtime });
|
|
259
903
|
}
|
|
260
|
-
|
|
904
|
+
finally {
|
|
905
|
+
releaseMaintenanceLock(lock);
|
|
906
|
+
}
|
|
907
|
+
}
|
|
908
|
+
/** Prove the recovered attempt's live bound listener IS ours end-to-end before adoption: launch
|
|
909
|
+
* binding, durable journal match, local pid + exact pidfile, source identity, mesh identity, the
|
|
910
|
+
* raw NATS INFO server name, and an authenticated connect confirming name + JetStream. */
|
|
911
|
+
async function proveOrdinaryResumeListener(pending) {
|
|
912
|
+
const proof = pending.adoptProof;
|
|
913
|
+
if (!proof)
|
|
914
|
+
throw new Error(`resume attempt ${pending.attemptId} has no bound listener proof`);
|
|
915
|
+
if (proof.attemptId !== pending.attemptId || proof.serverName !== pending.serverName ||
|
|
916
|
+
proof.serverNonce !== pending.serverNonce || proof.serverEndpoint !== pending.server ||
|
|
917
|
+
proof.serverName !== `${proof.attemptId}-${proof.serverNonce}` || !/^[0-9a-f]{32}$/.test(proof.serverNonce) ||
|
|
918
|
+
proof.processOwner.id !== `resume-listener-${proof.serverNonce}`)
|
|
919
|
+
throw new Error(`resume attempt ${pending.attemptId} listener proof does not match its launch record`);
|
|
920
|
+
const journal = readMaintenanceJournal(pending.root);
|
|
921
|
+
if (!journal || !("ordinaryResume" in journal) || journal.ordinaryResume.attemptId !== pending.attemptId ||
|
|
922
|
+
!journal.listenerProof || !sameRestoreListenerProof(journal.listenerProof, proof) ||
|
|
923
|
+
journal.ordinaryResume.launch.server !== proof.serverEndpoint)
|
|
924
|
+
throw new Error(`resume attempt ${pending.attemptId} listener proof does not exactly match durable recovery state`);
|
|
925
|
+
if (proof.processOwner.host !== hostname())
|
|
926
|
+
throw new Error(`resume attempt ${pending.attemptId} listener process ownership is not live and local`);
|
|
927
|
+
try {
|
|
928
|
+
process.kill(proof.processOwner.pid, 0);
|
|
929
|
+
}
|
|
930
|
+
catch {
|
|
931
|
+
throw new Error(`resume attempt ${pending.attemptId} listener process ownership is not live and local`);
|
|
932
|
+
}
|
|
933
|
+
const pidPath = join(pending.root, ".cotal", "nats.pid");
|
|
934
|
+
let exactPidFile = false;
|
|
935
|
+
try {
|
|
936
|
+
const stat = lstatSync(pidPath);
|
|
937
|
+
exactPidFile = stat.isFile() && !stat.isSymbolicLink() &&
|
|
938
|
+
readFileSync(pidPath, "utf8") === String(proof.processOwner.pid);
|
|
939
|
+
}
|
|
940
|
+
catch { /* absent or unprovable */ }
|
|
941
|
+
if (!exactPidFile)
|
|
942
|
+
throw new Error(`resume attempt ${pending.attemptId} listener pidfile does not match its process proof`);
|
|
943
|
+
const source = readStoreIdentity(pending.storeDir);
|
|
944
|
+
if (!sameStoreIdentity(source, proof.target) || !sameStoreIdentity(journal.source, proof.target))
|
|
945
|
+
throw new Error(`resume attempt ${pending.attemptId} preserved source identity changed`);
|
|
946
|
+
const mesh = findMesh(pending.space);
|
|
947
|
+
if (mesh && (mesh.server !== pending.server || mesh.root !== pending.root || mesh.mode !== pending.mode))
|
|
948
|
+
throw new Error(`resume attempt ${pending.attemptId} conflicts with the recorded mesh identity`);
|
|
949
|
+
const info = await readNatsInfo(proof.serverEndpoint);
|
|
950
|
+
if (info.server_name !== proof.serverName)
|
|
951
|
+
throw new Error(`resume attempt ${pending.attemptId} reached a foreign NATS server name/nonce`);
|
|
952
|
+
const auth = await resumeControlAuth(pending.root, pending.mode);
|
|
953
|
+
const nc = await connect({
|
|
954
|
+
servers: pending.server,
|
|
955
|
+
...standaloneConnectOpts(auth),
|
|
956
|
+
maxReconnectAttempts: 0,
|
|
957
|
+
});
|
|
958
|
+
try {
|
|
959
|
+
if (nc.info?.server_name !== proof.serverName)
|
|
960
|
+
throw new Error(`resume attempt ${pending.attemptId} authenticated to a foreign NATS server name/nonce`);
|
|
961
|
+
if (!nc.info.jetstream)
|
|
962
|
+
throw new Error(`resume attempt ${pending.attemptId} listener has no JetStream`);
|
|
963
|
+
}
|
|
964
|
+
finally {
|
|
965
|
+
await nc.drain().catch(() => { });
|
|
966
|
+
}
|
|
967
|
+
return proof;
|
|
968
|
+
}
|
|
969
|
+
/** Light wire check for a freshly SPAWNED ordinary-resume listener: the greeted server must carry
|
|
970
|
+
* the attempt's exact name/nonce (the full seven-point prove is for adopting a survivor). */
|
|
971
|
+
async function verifySpawnedOrdinaryListener(pending) {
|
|
972
|
+
const info = await readNatsInfo(pending.server);
|
|
973
|
+
if (info.server_name !== pending.serverName)
|
|
974
|
+
throw new Error(`resume attempt ${pending.attemptId} spawned listener reports a foreign NATS server name`);
|
|
975
|
+
}
|
|
976
|
+
async function resumeProvenOrdinaryListener(pending) {
|
|
977
|
+
await proveOrdinaryResumeListener(pending);
|
|
978
|
+
const svc = await ensureRecoveredUserAuth(pending);
|
|
979
|
+
recordOurMesh({
|
|
980
|
+
space: pending.space,
|
|
981
|
+
server: pending.server,
|
|
982
|
+
root: pending.root,
|
|
983
|
+
mode: pending.mode,
|
|
984
|
+
...(svc.userAuth ? { userAuth: svc.userAuth } : {}),
|
|
985
|
+
ts: new Date().toISOString(),
|
|
986
|
+
});
|
|
987
|
+
const controlPlane = await startDeliveryWithBroker(pending.space, pending.server, {
|
|
988
|
+
runtime: pending.runtime,
|
|
989
|
+
resumeAttempt: pending.attemptId,
|
|
990
|
+
resumeCommitToken: pending.managerCommit?.durableCommitToken,
|
|
991
|
+
});
|
|
992
|
+
await completeResumeActivation(pending.attemptId, controlPlane && svc.ok, !svc.ok ? "adopted resume listener has no user-auth service" : "adopted resume listener has a degraded control plane", pending.server);
|
|
993
|
+
}
|
|
994
|
+
async function ensureRecoveredUserAuth(prepared) {
|
|
995
|
+
if (prepared.mode !== "user")
|
|
996
|
+
return { ok: true };
|
|
997
|
+
const auth = loadSpaceAuth(authDir(prepared.root));
|
|
998
|
+
if (!auth)
|
|
999
|
+
throw new Error("restored user-auth listener has no retained trust material");
|
|
1000
|
+
const stateDir = userAuthStateDir(prepared.root, prepared.space);
|
|
1001
|
+
const provider = await resolveAuthProvider().prepareServer({
|
|
1002
|
+
space: prepared.space,
|
|
1003
|
+
operatorSeed: auth.operator.seed,
|
|
1004
|
+
account: { pub: auth.account.pub, signingSeed: auth.account.signingSeed },
|
|
1005
|
+
store: workspaceSecretStore(prepared.root),
|
|
1006
|
+
dir: stateDir,
|
|
1007
|
+
});
|
|
1008
|
+
return startUserAuthService(prepared.space, prepared.server, { prepared: provider, stateDir });
|
|
1009
|
+
}
|
|
1010
|
+
async function resumeProvenRestoreListener(prepared) {
|
|
1011
|
+
await provePreparedRestoreListener(prepared);
|
|
1012
|
+
const svc = await ensureRecoveredUserAuth(prepared);
|
|
1013
|
+
recordOurMesh({
|
|
1014
|
+
space: prepared.space,
|
|
1015
|
+
server: prepared.server,
|
|
1016
|
+
root: prepared.root,
|
|
1017
|
+
mode: prepared.mode,
|
|
1018
|
+
...(svc.userAuth ? { userAuth: svc.userAuth } : {}),
|
|
1019
|
+
ts: new Date().toISOString(),
|
|
1020
|
+
});
|
|
1021
|
+
const controlPlane = await startDeliveryWithBroker(prepared.space, prepared.server, {
|
|
1022
|
+
runtime: prepared.runtime,
|
|
1023
|
+
resumeAttempt: prepared.attemptId,
|
|
1024
|
+
resumeCommitToken: prepared.managerCommit?.durableCommitToken,
|
|
1025
|
+
});
|
|
1026
|
+
await completeResumeActivation(prepared.attemptId, controlPlane && svc.ok, !svc.ok ? "proven restore listener has no user-auth service" : "proven restore listener has a degraded control plane", prepared.server);
|
|
1027
|
+
}
|
|
1028
|
+
async function completeResumeActivation(attemptId, healthy, reason, server) {
|
|
1029
|
+
if (!attemptId)
|
|
1030
|
+
return;
|
|
1031
|
+
const ordinary = pendingOrdinaryResumes.get(attemptId);
|
|
1032
|
+
const restored = pendingRestores.get(attemptId);
|
|
1033
|
+
const pending = ordinary ?? restored;
|
|
1034
|
+
if (!pending)
|
|
1035
|
+
throw new Error(`resume activation lost attempt context ${attemptId}`);
|
|
1036
|
+
if (!healthy) {
|
|
1037
|
+
markPendingResumeDegraded(attemptId, reason);
|
|
1038
|
+
throw new Error(reason);
|
|
1039
|
+
}
|
|
1040
|
+
let journal = readMaintenanceJournal(pending.root);
|
|
1041
|
+
if (!journal)
|
|
1042
|
+
throw new Error(`resume attempt ${attemptId} lost its durable workspace journal`);
|
|
1043
|
+
if (restored) {
|
|
1044
|
+
if (!("restore" in journal) || journal.restore.attemptId !== attemptId)
|
|
1045
|
+
throw new Error(`restore activation journal does not match attempt ${attemptId}`);
|
|
1046
|
+
if (journal.state === "active") {
|
|
1047
|
+
restored.cleanupStage();
|
|
1048
|
+
pendingRestores.delete(attemptId);
|
|
1049
|
+
return;
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
else {
|
|
1053
|
+
if (!("ordinaryResume" in journal) || journal.ordinaryResume.attemptId !== attemptId)
|
|
1054
|
+
throw new Error(`ordinary resume journal does not match attempt ${attemptId}`);
|
|
1055
|
+
if (journal.state === "resume-retired") {
|
|
1056
|
+
const lock = acquireMaintenanceLock(ordinary.root);
|
|
1057
|
+
try {
|
|
1058
|
+
consumeRetiredMaintenance(lock);
|
|
1059
|
+
}
|
|
1060
|
+
finally {
|
|
1061
|
+
releaseMaintenanceLock(lock);
|
|
1062
|
+
}
|
|
1063
|
+
pendingOrdinaryResumes.delete(attemptId);
|
|
1064
|
+
return;
|
|
1065
|
+
}
|
|
1066
|
+
}
|
|
1067
|
+
let managerCommit;
|
|
1068
|
+
if (restored && (journal.state === "manager-committed" ||
|
|
1069
|
+
(journal.state === "degraded" && journal.managerCommit))) {
|
|
1070
|
+
managerCommit = journal.managerCommit;
|
|
1071
|
+
restored.managerCommit = managerCommit;
|
|
1072
|
+
restored.journalState = journal.state;
|
|
1073
|
+
}
|
|
1074
|
+
else if (ordinary && journal.state === "resume-committed") {
|
|
1075
|
+
managerCommit = journal.managerCommit;
|
|
1076
|
+
ordinary.journalState = "resume-committed";
|
|
1077
|
+
}
|
|
1078
|
+
const auth = await resumeControlAuth(pending.root, pending.mode);
|
|
1079
|
+
const readinessDeadline = Date.now() + 20_000;
|
|
1080
|
+
for (;;) {
|
|
1081
|
+
const ready = await askManager(pending.space, server, "ps", undefined, auth, CONTROL_ADMIN, 2_000);
|
|
1082
|
+
if (!ready.error?.startsWith("no manager reachable"))
|
|
1083
|
+
break;
|
|
1084
|
+
if (Date.now() >= readinessDeadline) {
|
|
1085
|
+
markPendingResumeDegraded(attemptId, ready.error);
|
|
1086
|
+
throw new Error(ready.error);
|
|
1087
|
+
}
|
|
1088
|
+
await new Promise((resolveWait) => setTimeout(resolveWait, 150));
|
|
1089
|
+
}
|
|
1090
|
+
const resumed = await askManager(pending.space, server, "resumePreserved", {
|
|
1091
|
+
attemptId,
|
|
1092
|
+
inventory: restored?.selection === "registry"
|
|
1093
|
+
? { ...pending.inventory, agents: [] }
|
|
1094
|
+
: pending.inventory,
|
|
1095
|
+
}, auth, CONTROL_ADMIN, 10 * 60 * 1000);
|
|
1096
|
+
if (!resumed.ok) {
|
|
1097
|
+
const detail = resumed.data ? ` (${JSON.stringify(resumed.data)})` : "";
|
|
1098
|
+
const message = `${resumed.error ?? "retained-agent resume failed"}${detail}`;
|
|
1099
|
+
markPendingResumeDegraded(attemptId, message);
|
|
1100
|
+
throw new Error(message);
|
|
1101
|
+
}
|
|
1102
|
+
if (restored && process.env.COTAL_SMOKE_EXIT_AFTER_RESUME_PRESERVED === "1")
|
|
1103
|
+
process.exit(88);
|
|
1104
|
+
if (ordinary && !managerCommit) {
|
|
1105
|
+
const lock = acquireMaintenanceLock(ordinary.root);
|
|
1106
|
+
try {
|
|
1107
|
+
const current = readMaintenanceJournal(ordinary.root);
|
|
1108
|
+
if (current?.state !== "resume-active") {
|
|
1109
|
+
markOrdinaryResumeActive(lock, {
|
|
1110
|
+
operation: "resumePreserved",
|
|
1111
|
+
attemptId,
|
|
1112
|
+
state: "awaitingCommit",
|
|
1113
|
+
observedAt: new Date().toISOString(),
|
|
1114
|
+
});
|
|
1115
|
+
}
|
|
1116
|
+
ordinary.journalState = "resume-active";
|
|
1117
|
+
}
|
|
1118
|
+
finally {
|
|
1119
|
+
releaseMaintenanceLock(lock);
|
|
1120
|
+
}
|
|
1121
|
+
}
|
|
1122
|
+
if (!managerCommit) {
|
|
1123
|
+
const committed = await askManager(pending.space, server, "commitResume", { attemptId }, auth, CONTROL_ADMIN, 40_000);
|
|
1124
|
+
if (!committed.ok) {
|
|
1125
|
+
const message = committed.error ?? "manager resume commit failed";
|
|
1126
|
+
markPendingResumeDegraded(attemptId, message);
|
|
1127
|
+
throw new Error(message);
|
|
1128
|
+
}
|
|
1129
|
+
if (!isManagerCommitResult(committed.data, attemptId)) {
|
|
1130
|
+
const message = `manager resume commit returned invalid awaiting-finalize evidence for attempt ${attemptId}`;
|
|
1131
|
+
markPendingResumeDegraded(attemptId, message);
|
|
1132
|
+
throw new Error(message);
|
|
1133
|
+
}
|
|
1134
|
+
managerCommit = committed.data;
|
|
1135
|
+
if (ordinary) {
|
|
1136
|
+
const lock = acquireMaintenanceLock(ordinary.root);
|
|
1137
|
+
try {
|
|
1138
|
+
recordOrdinaryResumeManagerCommit(lock, managerCommit);
|
|
1139
|
+
ordinary.journalState = "resume-committed";
|
|
1140
|
+
ordinary.managerCommit = managerCommit;
|
|
1141
|
+
}
|
|
1142
|
+
finally {
|
|
1143
|
+
releaseMaintenanceLock(lock);
|
|
1144
|
+
}
|
|
1145
|
+
}
|
|
1146
|
+
else {
|
|
1147
|
+
recordPreparedRestoreManagerCommit(restored, managerCommit);
|
|
1148
|
+
}
|
|
1149
|
+
if (process.env.COTAL_SMOKE_EXIT_AFTER_RESUME_COMMIT === "1")
|
|
1150
|
+
process.exit(89);
|
|
1151
|
+
}
|
|
1152
|
+
else {
|
|
1153
|
+
const recommitted = await askManager(pending.space, server, "commitResume", { attemptId }, auth, CONTROL_ADMIN, 40_000);
|
|
1154
|
+
// A surviving manager that already finalized legitimately answers {state:"active"} with the
|
|
1155
|
+
// exact durable token: accept both committed shapes idempotently, then reissue the token-bound
|
|
1156
|
+
// finalize (itself idempotent) below.
|
|
1157
|
+
const recovered = recommitted.ok ? recommitted.data : undefined;
|
|
1158
|
+
const exactToken = Boolean(recovered && typeof recovered === "object" &&
|
|
1159
|
+
recovered.attemptId === attemptId &&
|
|
1160
|
+
recovered.durableCommitToken === managerCommit.durableCommitToken &&
|
|
1161
|
+
(recovered.state === "awaitingFinalize" || recovered.state === "active"));
|
|
1162
|
+
if (!exactToken)
|
|
1163
|
+
throw new Error(`replacement manager did not recover the durable commit token for attempt ${attemptId}`);
|
|
1164
|
+
}
|
|
1165
|
+
const finalized = await askManager(pending.space, server, "finalizeResume", { attemptId, durableCommitToken: managerCommit.durableCommitToken }, auth, CONTROL_ADMIN, 40_000);
|
|
1166
|
+
if (!finalized.ok)
|
|
1167
|
+
throw new Error(finalized.error ?? `manager resume finalize failed for attempt ${attemptId}`);
|
|
1168
|
+
if (!isManagerFinalizeResult(finalized.data, attemptId))
|
|
1169
|
+
throw new Error(`manager resume finalize returned invalid active evidence for attempt ${attemptId}`);
|
|
1170
|
+
const finalizeEvidence = {
|
|
1171
|
+
attemptId,
|
|
1172
|
+
state: "active",
|
|
1173
|
+
durableCommitToken: managerCommit.durableCommitToken,
|
|
1174
|
+
};
|
|
1175
|
+
if (process.env.COTAL_SMOKE_EXIT_AFTER_RESUME_FINALIZE === "1")
|
|
1176
|
+
process.exit(91);
|
|
1177
|
+
if (ordinary) {
|
|
1178
|
+
const lock = acquireMaintenanceLock(ordinary.root);
|
|
1179
|
+
try {
|
|
1180
|
+
retireOrdinaryResume(lock, finalizeEvidence);
|
|
1181
|
+
consumeRetiredMaintenance(lock);
|
|
1182
|
+
}
|
|
1183
|
+
finally {
|
|
1184
|
+
releaseMaintenanceLock(lock);
|
|
1185
|
+
}
|
|
1186
|
+
pendingOrdinaryResumes.delete(attemptId);
|
|
1187
|
+
}
|
|
1188
|
+
else {
|
|
1189
|
+
restored.managerCommit = managerCommit;
|
|
1190
|
+
markPreparedRestoreActive(restored, finalizeEvidence);
|
|
1191
|
+
restored.cleanupStage();
|
|
1192
|
+
pendingRestores.delete(attemptId);
|
|
1193
|
+
}
|
|
261
1194
|
}
|
|
262
1195
|
/** Bring the space's USER-AUTH service up with the broker (user mode only — `setup.prepared` is the
|
|
263
1196
|
* provider's output). Loud both ways (U5): a ready service prints the login line; a service that
|
|
@@ -347,7 +1280,7 @@ async function upManifest(file, opts) {
|
|
|
347
1280
|
try {
|
|
348
1281
|
mkdirSync(cotalPath("nats"), { recursive: true });
|
|
349
1282
|
const setup = await authSetup(cotalPath("nats"), server, m.space, host, userAuth);
|
|
350
|
-
owner = await resolveAuthProvider().ownerForLogin({ dir: setup.stateDir, space: m.space });
|
|
1283
|
+
owner = await resolveAuthProvider().ownerForLogin({ store: workspaceSecretStore(cotalRoot()), dir: setup.stateDir, space: m.space });
|
|
351
1284
|
}
|
|
352
1285
|
catch (e) {
|
|
353
1286
|
console.error(c.red(`✗ ${e.message}`));
|
|
@@ -461,13 +1394,30 @@ export async function startMeshDetached(opts = {}) {
|
|
|
461
1394
|
const host = opts.host ?? "127.0.0.1";
|
|
462
1395
|
const setup = useAuth ? await authSetup(storeDir, server, space, host, opts.userAuth) : undefined;
|
|
463
1396
|
const port = Number(new URL(server).port) || 4222;
|
|
464
|
-
const args =
|
|
1397
|
+
const args = [
|
|
1398
|
+
...(setup ? ["-c", setup.confPath] : ["-js", "-sd", storeDir, "-p", String(port), "-a", host]),
|
|
1399
|
+
...(opts.boundListener ? ["--name", opts.boundListener.serverName] : []),
|
|
1400
|
+
];
|
|
465
1401
|
const { bin, source } = await resolveNatsServer();
|
|
466
1402
|
const logPath = cotalPath("nats.log");
|
|
467
1403
|
const startOffset = existsSync(logPath) ? statSync(logPath).size : 0;
|
|
468
1404
|
const fd = openSync(logPath, "a");
|
|
1405
|
+
const listenerStartedAt = new Date().toISOString();
|
|
469
1406
|
const child = spawn(bin, args, { detached: true, stdio: ["ignore", fd, fd] });
|
|
470
1407
|
closeSync(fd);
|
|
1408
|
+
if (opts.boundListener) {
|
|
1409
|
+
writeFileSync(cotalPath("nats.pid"), String(child.pid));
|
|
1410
|
+
if (process.env.COTAL_SMOKE_EXIT_AFTER_RESTORE_LISTENER_SPAWN === "1")
|
|
1411
|
+
process.exit(87);
|
|
1412
|
+
try {
|
|
1413
|
+
opts.boundListener.onSpawn(child.pid ?? 0, listenerStartedAt);
|
|
1414
|
+
}
|
|
1415
|
+
catch (error) {
|
|
1416
|
+
await stopUnboundRestoreListener(child);
|
|
1417
|
+
removeMatchingNatsPid(child.pid ?? 0);
|
|
1418
|
+
throw error;
|
|
1419
|
+
}
|
|
1420
|
+
}
|
|
471
1421
|
child.unref();
|
|
472
1422
|
let tailing = Boolean(opts.onLine);
|
|
473
1423
|
if (opts.onLine)
|
|
@@ -476,10 +1426,16 @@ export async function startMeshDetached(opts = {}) {
|
|
|
476
1426
|
tailing = false;
|
|
477
1427
|
if (!ready) {
|
|
478
1428
|
child.kill("SIGTERM");
|
|
1429
|
+
if (opts.boundListener)
|
|
1430
|
+
rmSync(cotalPath("nats.pid"), { force: true });
|
|
479
1431
|
throw new Error(`nats-server did not become reachable at ${server} - see ${logPath}`);
|
|
480
1432
|
}
|
|
481
|
-
|
|
482
|
-
|
|
1433
|
+
if (!opts.boundListener)
|
|
1434
|
+
writeFileSync(cotalPath("nats.pid"), String(child.pid));
|
|
1435
|
+
if (opts.boundListener)
|
|
1436
|
+
await opts.boundListener.verify();
|
|
1437
|
+
if (!opts.skipPostStart)
|
|
1438
|
+
await postStart(server, space, setup, seedFile);
|
|
483
1439
|
// USER MODE: the auth service comes up FIRST among the daemons (see the foreground path).
|
|
484
1440
|
const svc = await startUserAuthService(space, server, setup);
|
|
485
1441
|
// Record BEFORE the control plane: the manager's fail-closed mode detection needs the
|
|
@@ -492,7 +1448,12 @@ export async function startMeshDetached(opts = {}) {
|
|
|
492
1448
|
ts: new Date().toISOString(),
|
|
493
1449
|
});
|
|
494
1450
|
// Bring up the delivery daemon WITH the detached broker (auth mode only; `cotal down` tears both down).
|
|
495
|
-
const controlPlane = await startDeliveryWithBroker(space, server, {
|
|
1451
|
+
const controlPlane = await startDeliveryWithBroker(space, server, {
|
|
1452
|
+
runtime: opts.runtime,
|
|
1453
|
+
launch: opts.launch,
|
|
1454
|
+
resumeAttempt: opts.resumeAttempt,
|
|
1455
|
+
resumeCommitToken: opts.resumeCommitToken,
|
|
1456
|
+
});
|
|
496
1457
|
return {
|
|
497
1458
|
server,
|
|
498
1459
|
pid: child.pid ?? 0,
|
|
@@ -513,8 +1474,7 @@ function refuseOpenOverUserState(open, space) {
|
|
|
513
1474
|
const stateDir = userAuthStateDir(cotalRoot(), space);
|
|
514
1475
|
if (!existsSync(stateDir))
|
|
515
1476
|
return;
|
|
516
|
-
|
|
517
|
-
process.exit(1);
|
|
1477
|
+
throw new Error(`space "${space}" has user auth enabled (state under ${stateDir}) - \`--open\` would serve its streams without auth. Start it with \`cotal up --user-auth\`, or remove that directory deliberately to disable user auth (existing logins/grants die with it)`);
|
|
518
1478
|
}
|
|
519
1479
|
/** Today a root's `.cotal/auth` is created for one space (its account is space-bound), so an
|
|
520
1480
|
* explicit `--space` naming a different space cannot run against this root's trust material — the
|
|
@@ -537,8 +1497,7 @@ function ensureRootForSpace(useAuth, space) {
|
|
|
537
1497
|
console.log(c.dim(`nearest mesh root ${root} is space "${existing.space}" - making this folder its own root for "${space}"`));
|
|
538
1498
|
return;
|
|
539
1499
|
}
|
|
540
|
-
|
|
541
|
-
process.exit(1);
|
|
1500
|
+
throw new Error(`this folder is the root of space "${existing.space}" (${authDir(root)}), so it can't also run "${space}" - drop \`--space\` to run "${existing.space}", or start "${space}" from a different folder (it becomes that mesh's own root)`);
|
|
542
1501
|
}
|
|
543
1502
|
/** A space name maps to one mesh in the registry (the key `--space`/`use`/`down` act on). Before
|
|
544
1503
|
* starting a broker, refuse to reuse a space already claimed by a DIFFERENT live mesh — a stale/dead
|
|
@@ -551,8 +1510,7 @@ async function claimSpace(space, server, root) {
|
|
|
551
1510
|
if (!existing || (existing.server === server && existing.root === root))
|
|
552
1511
|
return;
|
|
553
1512
|
if (await isReachable(existing.server)) {
|
|
554
|
-
|
|
555
|
-
process.exit(1);
|
|
1513
|
+
throw new Error(`space "${space}" is already in use by a mesh at ${existing.server} (${existing.root}) - pick a different \`--space\`, or \`cotal down\` it first`);
|
|
556
1514
|
}
|
|
557
1515
|
removeMesh(space); // the prior holder's broker is gone — reclaim the name
|
|
558
1516
|
}
|
|
@@ -658,8 +1616,7 @@ function loadChannelsFile(explicit) {
|
|
|
658
1616
|
const path = explicit ? resolve(explicit) : cotalPath("channels.json");
|
|
659
1617
|
if (!existsSync(path)) {
|
|
660
1618
|
if (explicit) {
|
|
661
|
-
|
|
662
|
-
process.exit(1);
|
|
1619
|
+
throw new Error(`channels file not found: ${path}`);
|
|
663
1620
|
}
|
|
664
1621
|
return undefined;
|
|
665
1622
|
}
|
|
@@ -684,8 +1641,7 @@ async function authSetup(storeDir, server, space, host = "127.0.0.1", user) {
|
|
|
684
1641
|
}
|
|
685
1642
|
const stateDir = userAuthStateDir(cotalRoot(), space); // the provider's space-scoped state dir
|
|
686
1643
|
if (!user && existsSync(stateDir)) {
|
|
687
|
-
|
|
688
|
-
process.exit(1);
|
|
1644
|
+
throw new Error(`space "${space}" has user auth enabled (state under ${stateDir}) - start it with \`cotal up --user-auth\`, or remove that directory deliberately to disable user auth (existing logins/grants die with it)`);
|
|
689
1645
|
}
|
|
690
1646
|
let prepared;
|
|
691
1647
|
if (user) {
|
|
@@ -698,13 +1654,13 @@ async function authSetup(storeDir, server, space, host = "127.0.0.1", user) {
|
|
|
698
1654
|
space,
|
|
699
1655
|
operatorSeed: auth.operator.seed,
|
|
700
1656
|
account: { pub: auth.account.pub, signingSeed: auth.account.signingSeed },
|
|
1657
|
+
store: workspaceSecretStore(cotalRoot()),
|
|
701
1658
|
dir: stateDir,
|
|
702
1659
|
idpUrl: user.idpUrl,
|
|
703
1660
|
});
|
|
704
1661
|
}
|
|
705
1662
|
catch (e) {
|
|
706
|
-
|
|
707
|
-
process.exit(1);
|
|
1663
|
+
throw e instanceof Error ? e : new Error(String(e));
|
|
708
1664
|
}
|
|
709
1665
|
}
|
|
710
1666
|
const port = Number(new URL(server).port) || 4222;
|