agent-relay 12.2.7 → 12.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1349 @@
1
+ import crypto from 'node:crypto';
2
+ import fs from 'node:fs';
3
+ import os from 'node:os';
4
+ import path from 'node:path';
5
+ /**
6
+ * The machine-global Relay home that holds `fleet-enrollments.json`.
7
+ *
8
+ * Deliberately not imported from `@agent-relay/cloud`: that barrel pulls the
9
+ * Cloud SSH runtime, and a claim check must stay a couple of syscalls that any
10
+ * command can make. `node-claim.test.ts` pins this to the enrollment store's
11
+ * own directory so the two cannot drift apart.
12
+ */
13
+ function relayHome(env) {
14
+ return env.AGENT_RELAY_HOME ?? path.join(os.homedir(), '.agentworkforce/relay');
15
+ }
16
+ /** Filename the broker writes its API endpoint and pid into, inside its state dir. */
17
+ const BROKER_CONNECTION_FILENAME = 'connection.json';
18
+ /** Thrown when a live local broker already serves the requested node id. */
19
+ export class NodeClaimConflictError extends Error {
20
+ nodeId;
21
+ claim;
22
+ constructor(nodeId, claim) {
23
+ super(`node ${nodeId} is already served by a live local broker (pid ${claim.pid}, state dir ${claim.state_dir}). ` +
24
+ 'Stop it with `agent-relay node down --state-dir <dir>`, serve a different enrolled node, or re-run with --force.');
25
+ this.nodeId = nodeId;
26
+ this.claim = claim;
27
+ this.name = 'NodeClaimConflictError';
28
+ }
29
+ }
30
+ /**
31
+ * Thrown when the kernel-level ownership fence could not be established.
32
+ *
33
+ * The hold descriptor is what makes a claim survive its own supervisor, so a
34
+ * start that cannot open one cannot honour the guarantee it is claiming under.
35
+ * It refuses instead of spawning an unfenced broker.
36
+ */
37
+ export class NodeClaimHoldError extends Error {
38
+ nodeId;
39
+ holdPath;
40
+ constructor(nodeId, holdPath, cause) {
41
+ super(`could not establish the ownership fence for node ${nodeId} at ${holdPath}: ` +
42
+ `${cause instanceof Error ? cause.message : String(cause)}. ` +
43
+ 'Startup was stopped rather than run a broker that another start could silently take the node from. ' +
44
+ 'Ensure the claims directory is writable and has free space, then retry.', { cause });
45
+ this.nodeId = nodeId;
46
+ this.holdPath = holdPath;
47
+ this.name = 'NodeClaimHoldError';
48
+ }
49
+ }
50
+ /**
51
+ * Thrown when ownership could not be established because other starts kept
52
+ * winning the exclusive create.
53
+ *
54
+ * Each attempt is a handful of syscalls, so exhausting them means a pathological
55
+ * amount of contention on one node id. Ownership cannot be established in that
56
+ * state, and starting anyway is exactly the double registration this module
57
+ * exists to prevent.
58
+ */
59
+ export class NodeClaimContentionError extends Error {
60
+ nodeId;
61
+ claimsDir;
62
+ constructor(nodeId, claimsDir, cause) {
63
+ super(`could not take ownership of node ${nodeId}: other agent-relay starts kept winning the claim in ${claimsDir}. ` +
64
+ 'Retry, or run `agent-relay node down` on the state dir that should keep the node.', { cause });
65
+ this.nodeId = nodeId;
66
+ this.claimsDir = claimsDir;
67
+ this.name = 'NodeClaimContentionError';
68
+ }
69
+ }
70
+ /**
71
+ * The enrolled node id a start will register as, or `undefined` when it will
72
+ * mint its own identity.
73
+ *
74
+ * `RELAY_NODE_ID` is sent verbatim in `node.register` (`resolve_broker_node_id`,
75
+ * crates/broker/src/runtime/init.rs), so an explicit node id IS the identity
76
+ * that can evict another broker, and it is claimed on that basis alone.
77
+ *
78
+ * This deliberately does not try to predict whether the broker will manage to
79
+ * authenticate as it. An earlier version claimed the id only when a token was
80
+ * in the environment or already in the broker's on-disk cache, and that was a
81
+ * hole: `init.rs` also wires a workspace-key token minter, and
82
+ * `node_control.rs` mints and connects with no cached token at all, so a start
83
+ * carrying nothing but `RELAY_NODE_ID` and workspace credentials walked past a
84
+ * live claim and took the node's delivery socket — the incident this module
85
+ * exists to close. Enumerating the credential routes instead of the identity
86
+ * just moves the hole to the next route that gets added.
87
+ *
88
+ * The cost is a start that could not have registered at all (no token, no
89
+ * workspace key, nothing to mint with) being refused when some other broker
90
+ * genuinely holds the id. That is recoverable in one flag (`--force`) and the
91
+ * start it refuses was headed for local-only operation anyway, while the
92
+ * opposite error is a silent delivery outage. `--local-only` starts register
93
+ * nothing and are excluded by the callers, not here.
94
+ */
95
+ export function enrolledNodeIdForClaim(env) {
96
+ return env.RELAY_NODE_ID?.trim() || undefined;
97
+ }
98
+ /* ------------------------------------------------------------------ *
99
+ * Claim files and generations
100
+ * ------------------------------------------------------------------ */
101
+ /** Zero padding, so claim generations sort lexically as well as numerically. */
102
+ const GENERATION_DIGITS = 6;
103
+ /** Matches `<stem>.<generation>.json` for any node. */
104
+ const CLAIM_FILENAME_PATTERN = new RegExp(`^(.+)\\.(\\d{${GENERATION_DIGITS},})\\.json$`);
105
+ /** Directory holding the claim files for every node id served on this machine. */
106
+ export function nodeClaimsDir(env = process.env) {
107
+ return path.join(relayHome(env), 'node-claims');
108
+ }
109
+ /**
110
+ * Filename stem for a node id's claims. Node ids are engine-issued
111
+ * (`node_<digits>`), but the stem is sanitized anyway so a hand-edited
112
+ * enrollment store cannot write outside the claims directory. Readers verify
113
+ * `node_id` from the file contents, so a sanitized collision is reported as a
114
+ * conflict rather than silently overwriting another node's claim.
115
+ */
116
+ function nodeClaimStem(nodeId) {
117
+ return (nodeId
118
+ .trim()
119
+ .replace(/[^\w.-]/g, '-')
120
+ .slice(0, 96) || 'unnamed');
121
+ }
122
+ /**
123
+ * Path of one generation of a node id's claim.
124
+ *
125
+ * Every write this module makes is the exclusive creation of a NEW generation,
126
+ * never an overwrite of an existing one — that is what makes takeover safe
127
+ * without a lock file (see {@link acquireNodeClaim}).
128
+ */
129
+ export function nodeClaimPath(nodeId, env = process.env, generation = 1) {
130
+ const suffix = String(generation).padStart(GENERATION_DIGITS, '0');
131
+ return path.join(nodeClaimsDir(env), `${nodeClaimStem(nodeId)}.${suffix}.json`);
132
+ }
133
+ /**
134
+ * Sibling of a claim generation whose OPEN DESCRIPTORS are the ownership
135
+ * evidence, held by the kernel rather than written by a process.
136
+ *
137
+ * `<stem>.<generation>.hold` is created and opened by the supervising CLI
138
+ * BEFORE it spawns a broker, and the descriptor is inherited by the child
139
+ * across `fork`. From the instant a broker child exists — long before it binds
140
+ * its API, writes `connection.json` or queues `node.register` — some live
141
+ * process holds this file open, and the kernel drops the last reference the
142
+ * moment both of them die. That is what closes the window a supervisor
143
+ * SIGKILLed between the spawn and the broker's first write used to leave: there
144
+ * is no interval in which a competing start can see "dead supervisor, nothing
145
+ * published" and conclude the node id is free while the orphan is on its way to
146
+ * registering.
147
+ *
148
+ * Not matched by {@link CLAIM_FILENAME_PATTERN} (which requires `.json`), so it
149
+ * never counts as a generation of its own.
150
+ */
151
+ export function nodeClaimHoldPath(nodeId, env = process.env, generation = 1) {
152
+ return `${nodeClaimPath(nodeId, env, generation).slice(0, -'.json'.length)}.hold`;
153
+ }
154
+ /** Single-quote a path for a shell command (`AGENT_RELAY_HOME` is operator input). */
155
+ function shellQuote(value) {
156
+ // Close the quoted run, emit an escaped quote, reopen: ' -> '\''
157
+ return `'${value.split("'").join("'\\''")}'`;
158
+ }
159
+ /** Resolve a state dir to its canonical path so two spellings compare equal. */
160
+ export function normalizeClaimStateDir(stateDir) {
161
+ try {
162
+ return fs.realpathSync(stateDir);
163
+ }
164
+ catch {
165
+ return path.resolve(stateDir);
166
+ }
167
+ }
168
+ /**
169
+ * Whether a recorded claim names this state dir. Both sides are canonicalised:
170
+ * a claim written by an older CLI, by hand, or before its directory existed
171
+ * (when `realpathSync` could not resolve it) may carry a non-canonical
172
+ * spelling, and on macOS every `/var/...` temp dir is really `/private/var/...`.
173
+ * Comparing a canonical input against the raw record missed all of those.
174
+ */
175
+ export function claimNamesStateDir(claim, stateDir) {
176
+ return normalizeClaimStateDir(claim.state_dir) === normalizeClaimStateDir(stateDir);
177
+ }
178
+ function isOptionalString(value) {
179
+ return value === undefined || typeof value === 'string';
180
+ }
181
+ function isOptionalPositiveInteger(value) {
182
+ return value === undefined || (Number.isSafeInteger(value) && value > 0);
183
+ }
184
+ function hasValidOptionalClaimFields(record) {
185
+ return ((record.api_port === undefined || Number.isSafeInteger(record.api_port)) &&
186
+ isOptionalString(record.broker_name) &&
187
+ isOptionalString(record.broker_binary) &&
188
+ isOptionalString(record.broker_executable) &&
189
+ isOptionalString(record.process_started_at) &&
190
+ isOptionalString(record.supervisor_started_at) &&
191
+ isOptionalString(record.owner_token) &&
192
+ isOptionalPositiveInteger(record.supervisor_pid) &&
193
+ isOptionalPositiveInteger(record.broker_child_pid) &&
194
+ isOptionalPositiveInteger(record.generation) &&
195
+ (record.status === undefined || record.status === 'reserved' || record.status === 'active'));
196
+ }
197
+ function isNodeClaim(value) {
198
+ if (typeof value !== 'object' || value === null)
199
+ return false;
200
+ const record = value;
201
+ // A tombstone occupies a generation number so it can never be reissued, but
202
+ // it is evidence of nothing. Rejecting it here is what makes it read as an
203
+ // unclaimed node id everywhere a claim is read.
204
+ if (record.released === true)
205
+ return false;
206
+ return (record.version === 1 &&
207
+ typeof record.node_id === 'string' &&
208
+ record.node_id.trim().length > 0 &&
209
+ Number.isSafeInteger(record.pid) &&
210
+ record.pid > 0 &&
211
+ typeof record.state_dir === 'string' &&
212
+ record.state_dir.length > 0 &&
213
+ typeof record.claimed_at === 'string' &&
214
+ hasValidOptionalClaimFields(record));
215
+ }
216
+ /** Exact bytes of a claim file, or `null` when it cannot be read at all. */
217
+ function readClaimBytes(file) {
218
+ try {
219
+ return fs.readFileSync(file, 'utf-8');
220
+ }
221
+ catch {
222
+ return null;
223
+ }
224
+ }
225
+ function parseClaim(raw) {
226
+ if (raw === null)
227
+ return null;
228
+ try {
229
+ const parsed = JSON.parse(raw);
230
+ return isNodeClaim(parsed) ? parsed : null;
231
+ }
232
+ catch {
233
+ return null;
234
+ }
235
+ }
236
+ function readClaimFile(file) {
237
+ return parseClaim(readClaimBytes(file));
238
+ }
239
+ /**
240
+ * Every generation file present for `nodeId`, ascending.
241
+ *
242
+ * Unreadable files are kept in the list with a `null` claim: they must still
243
+ * raise the next generation number (or an exclusive create would collide with
244
+ * them forever) even though they are no evidence of a live broker.
245
+ */
246
+ function readClaimGenerations(nodeId, env) {
247
+ const dir = nodeClaimsDir(env);
248
+ const stem = nodeClaimStem(nodeId);
249
+ let filenames;
250
+ try {
251
+ filenames = fs.readdirSync(dir);
252
+ }
253
+ catch {
254
+ return [];
255
+ }
256
+ const generations = [];
257
+ for (const filename of filenames) {
258
+ const match = CLAIM_FILENAME_PATTERN.exec(filename);
259
+ if (!match || match[1] !== stem)
260
+ continue;
261
+ const file = path.join(dir, filename);
262
+ const raw = readClaimBytes(file);
263
+ generations.push({ generation: Number.parseInt(match[2], 10), file, raw, claim: parseClaim(raw) });
264
+ }
265
+ return generations.sort((left, right) => left.generation - right.generation);
266
+ }
267
+ /** The generation that currently owns the node id: the highest readable one. */
268
+ function currentGeneration(generations) {
269
+ for (let index = generations.length - 1; index >= 0; index -= 1) {
270
+ if (generations[index].claim)
271
+ return generations[index];
272
+ }
273
+ return undefined;
274
+ }
275
+ function highestGenerationNumber(generations) {
276
+ return generations.length === 0 ? 0 : generations[generations.length - 1].generation;
277
+ }
278
+ /**
279
+ * The claim that currently owns `nodeId`, or `null`.
280
+ *
281
+ * A missing, unreadable, or malformed file reads as `null`: an unparseable
282
+ * claim is no evidence of a live broker, and treating it as one would brick
283
+ * every later `node up` for that node id.
284
+ */
285
+ export function readNodeClaim(nodeId, env = process.env) {
286
+ return currentGeneration(readClaimGenerations(nodeId, env))?.claim ?? null;
287
+ }
288
+ /** The current claim for every node id on this machine, newest first. */
289
+ export function listNodeClaims(env = process.env) {
290
+ const dir = nodeClaimsDir(env);
291
+ let filenames;
292
+ try {
293
+ filenames = fs.readdirSync(dir);
294
+ }
295
+ catch {
296
+ return [];
297
+ }
298
+ const newestByStem = new Map();
299
+ for (const filename of filenames) {
300
+ const match = CLAIM_FILENAME_PATTERN.exec(filename);
301
+ if (!match)
302
+ continue;
303
+ const claim = readClaimFile(path.join(dir, filename));
304
+ if (!claim)
305
+ continue;
306
+ const generation = Number.parseInt(match[2], 10);
307
+ const seen = newestByStem.get(match[1]);
308
+ if (!seen || seen.generation < generation) {
309
+ newestByStem.set(match[1], { generation, claim });
310
+ }
311
+ }
312
+ return [...newestByStem.values()]
313
+ .map((entry) => entry.claim)
314
+ .sort((left, right) => right.claimed_at.localeCompare(left.claimed_at));
315
+ }
316
+ /**
317
+ * Every readable claim on this machine, across ALL generations, newest first.
318
+ *
319
+ * `listNodeClaims` reports only the current generation per node id, but a
320
+ * `--force` takeover can leave a still-held incumbent beneath the
321
+ * replacement's file. Callers that diagnose or release by broker identity must
322
+ * see every generation, or that hidden claim never surfaces.
323
+ */
324
+ function listAllNodeClaims(env) {
325
+ const dir = nodeClaimsDir(env);
326
+ let filenames;
327
+ try {
328
+ filenames = fs.readdirSync(dir);
329
+ }
330
+ catch {
331
+ return [];
332
+ }
333
+ const claims = [];
334
+ for (const filename of filenames) {
335
+ const match = CLAIM_FILENAME_PATTERN.exec(filename);
336
+ if (!match)
337
+ continue;
338
+ const claim = readClaimFile(path.join(dir, filename));
339
+ if (claim)
340
+ claims.push(claim);
341
+ }
342
+ return claims.sort((left, right) => right.claimed_at.localeCompare(left.claimed_at) || (right.generation ?? 0) - (left.generation ?? 0));
343
+ }
344
+ /* ------------------------------------------------------------------ *
345
+ * Liveness
346
+ * ------------------------------------------------------------------ */
347
+ function isProcessAlive(pid, deps) {
348
+ const kill = deps.killProcess ??
349
+ ((target, signal) => {
350
+ process.kill(target, signal);
351
+ });
352
+ try {
353
+ kill(pid, 0);
354
+ return true;
355
+ }
356
+ catch (error) {
357
+ // Permission denial proves the process exists and is not ours to inspect;
358
+ // that must read as live, never as a free node id.
359
+ return error?.code === 'EPERM';
360
+ }
361
+ }
362
+ /**
363
+ * `ps` birth time for a pid, or `null` when it cannot be read. Mirrors the
364
+ * command `broker-process-identity` uses so both agree on the format.
365
+ */
366
+ async function readProcessStartedAt(pid, deps) {
367
+ const execCommand = deps.execCommand;
368
+ if (!execCommand) {
369
+ return null;
370
+ }
371
+ try {
372
+ const { stdout } = await execCommand(`LC_ALL=C TZ=UTC ps -p ${pid} -o lstart=`);
373
+ const normalized = stdout.trim().replace(/\s+/g, ' ');
374
+ return normalized.length > 0 ? normalized : null;
375
+ }
376
+ catch {
377
+ return null;
378
+ }
379
+ }
380
+ /**
381
+ * Whether `pid` is still the process that was recorded with `startedAt`.
382
+ * A pid that is gone, or one whose birth time moved (the number was recycled),
383
+ * no longer holds anything.
384
+ */
385
+ async function isRecordedProcessAlive(pid, startedAt, deps) {
386
+ if (!isProcessAlive(pid, deps)) {
387
+ return { alive: false, reason: `pid ${pid} is no longer running` };
388
+ }
389
+ if (startedAt) {
390
+ const current = await readProcessStartedAt(pid, deps);
391
+ if (current && current !== startedAt) {
392
+ return { alive: false, reason: `pid ${pid} was recycled by a process started ${current}` };
393
+ }
394
+ }
395
+ return { alive: true, reason: `pid ${pid} is running` };
396
+ }
397
+ /** The owning pid plus, when the claim records one, its supervising CLI. */
398
+ function claimProcesses(claim) {
399
+ const processes = [
400
+ {
401
+ pid: claim.pid,
402
+ startedAt: claim.process_started_at,
403
+ role: claim.status === 'reserved' ? 'supervisor' : 'broker',
404
+ },
405
+ ];
406
+ if (claim.supervisor_pid !== undefined && claim.supervisor_pid !== claim.pid) {
407
+ processes.push({
408
+ pid: claim.supervisor_pid,
409
+ startedAt: claim.supervisor_started_at,
410
+ role: 'supervisor',
411
+ });
412
+ }
413
+ return processes;
414
+ }
415
+ /** Pid recorded in `<state dir>/connection.json`, which the broker writes itself. */
416
+ function readStateDirBrokerPid(stateDir) {
417
+ try {
418
+ const parsed = JSON.parse(fs.readFileSync(path.join(stateDir, BROKER_CONNECTION_FILENAME), 'utf-8'));
419
+ if (typeof parsed !== 'object' || parsed === null)
420
+ return null;
421
+ const pid = parsed.pid;
422
+ return Number.isSafeInteger(pid) && pid > 0 ? pid : null;
423
+ }
424
+ catch {
425
+ return null;
426
+ }
427
+ }
428
+ function brokerExecutableHint(claim) {
429
+ return {
430
+ stateDir: claim.state_dir,
431
+ ...(claim.broker_binary ? { binary: claim.broker_binary } : {}),
432
+ ...(claim.broker_executable ? { object: claim.broker_executable } : {}),
433
+ ...(claim.broker_child_pid ? { childPid: claim.broker_child_pid } : {}),
434
+ };
435
+ }
436
+ /**
437
+ * Identity of the executable file a claim's start was going to run, for the
438
+ * record. `realpathSync` first so a claim written through a symlinked install
439
+ * path still names the file a running broker maps.
440
+ */
441
+ /**
442
+ * `<device>:<inode>` of a file on this machine, in the encoding `lsof -d txt`
443
+ * reports and the claim records, or `null` when it cannot be stat'ed.
444
+ */
445
+ function executableObjectOf(file) {
446
+ try {
447
+ const stats = fs.statSync(file, { bigint: true });
448
+ return `0x${stats.dev.toString(16)}:${stats.ino.toString()}`;
449
+ }
450
+ catch {
451
+ return null;
452
+ }
453
+ }
454
+ function describeBrokerExecutable(binary) {
455
+ if (!binary)
456
+ return {};
457
+ try {
458
+ const resolved = fs.realpathSync(binary);
459
+ const object = executableObjectOf(resolved);
460
+ return {
461
+ broker_binary: resolved,
462
+ ...(object ? { broker_executable: object } : {}),
463
+ };
464
+ }
465
+ catch {
466
+ // The path is still worth recording: a process running it is recognisable
467
+ // by argv even when the file cannot be stat'ed from here.
468
+ return path.isAbsolute(binary) ? { broker_binary: binary } : {};
469
+ }
470
+ }
471
+ /**
472
+ * The executable objects `pid` is running, as `<device>:<inode>` pairs, or
473
+ * `null` when they cannot be read.
474
+ *
475
+ * `txt` mappings identify an executable by device and inode on both macOS and
476
+ * Linux, which is the only handle on "what is this process running" that does
477
+ * not go through a filename. Same command and same field encoding as
478
+ * `readBrokerProcessIdentity` in `broker-process-identity.ts`, so the two agree
479
+ * on what an executable's identity is.
480
+ */
481
+ async function readExecutableObjects(pid, deps) {
482
+ const execCommand = deps.execCommand;
483
+ if (!execCommand)
484
+ return null;
485
+ let stdout;
486
+ try {
487
+ ({ stdout } = await execCommand(`LC_ALL=C lsof -nP -a -p ${pid} -d txt -FfDi`));
488
+ }
489
+ catch {
490
+ return null;
491
+ }
492
+ const fields = stdout.trim().split('\n');
493
+ if (fields.shift() !== `p${pid}`)
494
+ return null;
495
+ const objects = new Set();
496
+ for (let index = 0; index < fields.length; index += 3) {
497
+ const [descriptor, device, inode] = fields.slice(index, index + 3);
498
+ if (descriptor !== 'ftxt' || !/^D0x[0-9a-f]+$/i.test(device ?? '') || !/^i[1-9]\d*$/.test(inode ?? '')) {
499
+ return null;
500
+ }
501
+ objects.add(`${device.slice(1).toLowerCase()}:${inode.slice(1)}`);
502
+ }
503
+ return objects.size > 0 ? objects : null;
504
+ }
505
+ /**
506
+ * Whether a command line names the executable the claim recorded, in a
507
+ * position an interpreter would put it.
508
+ *
509
+ * A shebang launcher does not RUN the file the claim recorded — the kernel runs
510
+ * the interpreter from its `#!` line and passes the script as an argument, so
511
+ * `lsof -d txt` reports `/bin/sh` and the recorded device/inode matches
512
+ * nothing. The script path is still right there in argv, and the file it names
513
+ * has exactly the identity the claim recorded. Comparing that file rather than
514
+ * the string is what keeps this working through a symlinked install path, the
515
+ * same reason the claim records an inode instead of a name.
516
+ *
517
+ * Only the first two arguments are considered: `argv[0]` is the program and
518
+ * `argv[1]` is where an interpreter puts its script. Anything further along is
519
+ * a broker's own argument and proves nothing about what it is running.
520
+ */
521
+ function runsRecordedExecutable(args, hint) {
522
+ if (!hint.binary && !hint.object)
523
+ return false;
524
+ for (const token of args.split(/\s+/, 2)) {
525
+ if (!token.startsWith('/'))
526
+ continue;
527
+ if (hint.binary && token === hint.binary)
528
+ return true;
529
+ if (!hint.object)
530
+ continue;
531
+ const object = executableObjectOf(token);
532
+ if (object !== null && object === hint.object)
533
+ return true;
534
+ }
535
+ return false;
536
+ }
537
+ /**
538
+ * Everything a command line can say IN FAVOUR of a process being this claim's
539
+ * broker. Nothing here is ever read the other way round: a command line that
540
+ * matches none of it is not evidence of anything, because the broker binary is
541
+ * operator-selectable and a start on the default state dir passes no
542
+ * `--state-dir` for argv to carry.
543
+ */
544
+ function argvIdentifiesBroker(args, hint) {
545
+ if (hint.binary && (args === hint.binary || args.startsWith(`${hint.binary} `)))
546
+ return true;
547
+ if (runsRecordedExecutable(args, hint))
548
+ return true;
549
+ return args.includes('agent-relay') || args.includes('relay-broker') || args.includes(hint.stateDir);
550
+ }
551
+ /**
552
+ * Whether `pid` maps the executable object the claim recorded.
553
+ *
554
+ * Mappings that cannot be read are NOT a mismatch — the process may simply not
555
+ * be ours to inspect — so they answer yes, like every other unanswerable probe
556
+ * in this module.
557
+ */
558
+ async function mapsRecordedExecutable(pid, object, deps) {
559
+ const objects = await readExecutableObjects(pid, deps);
560
+ return objects === null || objects.has(object);
561
+ }
562
+ /**
563
+ * Whether `pid` is the broker a claim names, an unrelated process that merely
564
+ * inherited its descriptor or its pid number, or something this claim has no
565
+ * way to tell apart.
566
+ *
567
+ * Executable identity decides it: the device and inode of the file the process
568
+ * is running, against the broker executable the claim recorded. Command lines
569
+ * are consulted only as further POSITIVE evidence and never to rule a process
570
+ * out. `AGENT_RELAY_BIN` / `BROKER_BINARY_PATH` let a supported deployment run
571
+ * the broker under any filename, and with the default state dir such a broker
572
+ * carries neither `agent-relay` nor its state dir in argv — so the name test
573
+ * that used to decide this classified a real, live broker as unrelated and
574
+ * handed its node id to the next start.
575
+ *
576
+ * Those same overrides accept a LAUNCHER — a shell script that sets something
577
+ * up and `exec`s the broker is the ordinary shape of a custom install — and a
578
+ * launcher is not the file the running process maps. Before its `exec` the
579
+ * process runs the interpreter from the script's `#!` line; after it, the
580
+ * binary the script chose. Neither is the file recorded before the spawn, so
581
+ * executable identity alone ruled a live, fenced startup launcher out and
582
+ * reopened exactly the eviction this claim exists to prevent. Two further
583
+ * pieces of positive evidence close that: the recorded file appearing in argv
584
+ * where an interpreter puts its script ({@link runsRecordedExecutable}), which
585
+ * needs nothing recorded after the spawn, and the pid of the child this start
586
+ * spawned ({@link NodeClaim.broker_child_pid}), which `execve` preserves
587
+ * whatever the launcher turns into.
588
+ *
589
+ * Anything that cannot be consulted — no runner, an `lsof` or `ps` that fails —
590
+ * reads as `broker`: this only ever decides whether to KEEP guarding a node id,
591
+ * and a spurious refusal is recoverable (`--force`, `node down`) while a wrong
592
+ * "free" verdict is the silent delivery outage.
593
+ */
594
+ async function classifyHolderProcess(pid, hint, deps) {
595
+ // Recorded at the spawn itself, so it is the one piece of evidence that is
596
+ // immune to whatever the child turns into afterwards.
597
+ if (hint.childPid !== undefined && pid === hint.childPid)
598
+ return 'broker';
599
+ const execCommand = deps.execCommand;
600
+ if (!execCommand)
601
+ return 'broker';
602
+ if (hint.object && (await mapsRecordedExecutable(pid, hint.object, deps)))
603
+ return 'broker';
604
+ let args;
605
+ try {
606
+ const { stdout } = await execCommand(`LC_ALL=C ps -p ${pid} -o args=`);
607
+ args = stdout.trim();
608
+ }
609
+ catch {
610
+ return 'broker';
611
+ }
612
+ // `ps` answered with nothing: the pid is gone, so it holds nothing.
613
+ if (!args)
614
+ return 'unrelated';
615
+ if (argvIdentifiesBroker(args, hint))
616
+ return 'broker';
617
+ return hint.object || hint.binary ? 'unrelated' : 'unidentified';
618
+ }
619
+ /**
620
+ * Whether any live process still holds this generation's hold file open.
621
+ *
622
+ * `lsof -t` answers from the kernel's open-file table, so this reports the
623
+ * broker child a SIGKILLed supervisor left behind from the instant that child
624
+ * exists — there is no publish step to wait for and nothing to go stale. The
625
+ * exit status is read from an explicit marker rather than from the runner's
626
+ * error shape: `lsof` exits 1 with no output when a file simply has no holders,
627
+ * and that has to be told apart from an `lsof` that could not run at all.
628
+ *
629
+ * An `lsof` that cannot be consulted reads as HELD, matching the rest of this
630
+ * module: `node up` already refuses to start without a usable `lsof` (it is how
631
+ * broker process identity is verified), and a spurious refusal is recoverable
632
+ * with `--force` or `node down` while a wrong "free" verdict is the silent
633
+ * delivery outage the claim exists to prevent.
634
+ *
635
+ * Holders are classified by {@link classifyHolderProcess}, exactly as the
636
+ * connection file's pid is. An inherited descriptor is not close-on-exec, so
637
+ * anything the broker itself spawns inherits it too; a harness left running by
638
+ * a SIGKILLed broker would otherwise pin the node id with no broker anywhere
639
+ * near it. Dropping one is a positive determination and needs BOTH halves of
640
+ * the evidence: the holder identified as some other executable, AND a claim
641
+ * that durably recorded the child it spawned. Without the second half the first
642
+ * cannot be trusted — a process that has `exec`d is not the file the claim
643
+ * recorded — so every holder stays a holder, as one does when the claim cannot
644
+ * classify it at all. Nothing but a Relay start ever passes this descriptor on,
645
+ * and a spurious refusal is recoverable where a wrong "free" verdict is not.
646
+ *
647
+ * `selfPids` are dropped from the holder set: a start re-reading its own
648
+ * reservation still has that generation's descriptor open, and its own
649
+ * descriptor is not evidence that somebody else owns the node id.
650
+ */
651
+ async function inspectClaimHold(claim, env, deps, selfPids) {
652
+ const file = nodeClaimHoldPath(claim.node_id, env, claim.generation ?? 1);
653
+ if (!fs.existsSync(file)) {
654
+ return { held: false, reason: `no start holds ${file}`, pids: [] };
655
+ }
656
+ const execCommand = deps.execCommand;
657
+ if (!execCommand) {
658
+ return { held: true, reason: `open descriptors on ${file} could not be checked`, pids: [] };
659
+ }
660
+ let stdout;
661
+ try {
662
+ ({ stdout } = await execCommand(`LC_ALL=C lsof -t -- ${shellQuote(file)} 2>/dev/null; printf 'rc=%d' "$?"`));
663
+ }
664
+ catch {
665
+ return { held: true, reason: `open descriptors on ${file} could not be checked`, pids: [] };
666
+ }
667
+ const status = /rc=(\d+)/.exec(stdout);
668
+ const pids = stdout
669
+ .replace(/rc=\d+/, '')
670
+ .split(/\s+/)
671
+ .map((value) => value.trim())
672
+ .filter((value) => /^\d+$/.test(value))
673
+ .filter((value) => !selfPids?.has(Number(value)));
674
+ const hint = brokerExecutableHint(claim);
675
+ // Ruling a holder OUT requires the claim to know which process this start
676
+ // actually spawned. Everything recorded before a spawn describes a FILE, and
677
+ // `execve` swaps the file while keeping the descriptor: a launcher that has
678
+ // already exec'd the real broker maps a binary the claim never recorded and
679
+ // no longer carries the launcher anywhere in its argv. `broker_child_pid` is
680
+ // what covers that — but it is published after the spawn, so a supervisor
681
+ // SIGKILLed in between leaves a claim that can never identify its own broker,
682
+ // and the one live process holding the node's fence classifies as
683
+ // `unrelated`. The node id then reads free while that broker is on its way to
684
+ // registering, which is the eviction this module exists to prevent.
685
+ //
686
+ // So until child identity is durably established, a holder of this descriptor
687
+ // keeps the node guarded whatever it is running. Nothing but a Relay start
688
+ // ever passes this descriptor on, and the bias is the module's usual one: a
689
+ // refusal costs one `--force`, a wrong "free" verdict costs deliveries.
690
+ const childEstablished = claim.broker_child_pid !== undefined;
691
+ const brokers = [];
692
+ const unprovable = [];
693
+ for (const pid of pids) {
694
+ const verdict = await classifyHolderProcess(Number(pid), hint, deps);
695
+ if (verdict !== 'unrelated')
696
+ brokers.push(pid);
697
+ else if (!childEstablished)
698
+ unprovable.push(pid);
699
+ }
700
+ if (brokers.length > 0) {
701
+ return {
702
+ held: true,
703
+ reason: `pid ${brokers.join(', ')} still holds ${file} open`,
704
+ pids: brokers.map(Number),
705
+ };
706
+ }
707
+ if (unprovable.length > 0) {
708
+ return {
709
+ held: true,
710
+ reason: `pid ${unprovable.join(', ')} still holds ${file} open, and the start that created it ` +
711
+ 'never recorded the broker child it spawned, so that process cannot be ruled out',
712
+ pids: unprovable.map(Number),
713
+ };
714
+ }
715
+ if (pids.length > 0) {
716
+ return {
717
+ held: false,
718
+ reason: `only unrelated processes (pid ${pids.join(', ')}) hold ${file} open`,
719
+ pids: [],
720
+ };
721
+ }
722
+ // `lsof` exits 1 for "no holders"; anything else means it could not answer.
723
+ if (!status || (status[1] !== '0' && status[1] !== '1')) {
724
+ return { held: true, reason: `open descriptors on ${file} could not be checked`, pids: [] };
725
+ }
726
+ return { held: false, reason: `no process holds ${file} open`, pids: [] };
727
+ }
728
+ /**
729
+ * Whether anything other than `selfPids` still holds this claim's generation
730
+ * fenced, for a caller deciding whether ownership may be dropped.
731
+ *
732
+ * A release has to consult this, not just the pids on the record. The record
733
+ * names a broker only once a start got far enough to capture one: a spawn that
734
+ * rejects before it returns a client — a handshake that never completed, a
735
+ * SIGTERM the child outlived — leaves a broker child that no pid anywhere names,
736
+ * and releasing on "no live pid recorded" tombstones the claim and unlinks the
737
+ * hold file out from under a process that is still fenced by it. The next start
738
+ * then reads the node id as free while that child can still register. The
739
+ * descriptor is the one piece of evidence that exists from the instant the child
740
+ * does, so it is what decides.
741
+ *
742
+ * Holders are filtered exactly as {@link inspectNodeClaim} filters them, so a
743
+ * release never frees a node id that another start would still read as held.
744
+ */
745
+ export async function inspectNodeClaimHold(claim, deps = {}, selfPids) {
746
+ return inspectClaimHold(claim, deps.env ?? process.env, deps, selfPids);
747
+ }
748
+ /**
749
+ * Create and open this generation's hold file, handing the supervising CLI the
750
+ * descriptor it passes to the broker child.
751
+ *
752
+ * Created exclusively (`wx`): only the acquisition that won the generation ever
753
+ * creates it, so a concurrent start can never replace the inode a live child is
754
+ * holding. Opened BEFORE the spawn, because a fence established after `fork`
755
+ * would leave exactly the gap it exists to close.
756
+ *
757
+ * Failing to establish the hold is FATAL to the start, which is why this throws
758
+ * rather than reporting a missing fence. Without the descriptor, a supervisor
759
+ * killed between the fork and the broker's `connection.json` write leaves a
760
+ * claim with only dead pids and no evidence at all — so the next start reads
761
+ * the node id as free and evicts a broker that is about to register. Continuing
762
+ * would mean spawning a broker under a guarantee that is not actually in force,
763
+ * which is worse than not starting: the operator can see and fix a refusal.
764
+ *
765
+ * @throws NodeClaimHoldError when the hold could not be established. Any
766
+ * partially created file and descriptor are cleaned up first, so a retry (or a
767
+ * later start, on the next generation) is not blocked by this one's debris.
768
+ */
769
+ export function openNodeClaimHold(claim, env = process.env) {
770
+ const generation = claim.generation ?? 1;
771
+ const file = nodeClaimHoldPath(claim.node_id, env, generation);
772
+ let fd;
773
+ try {
774
+ fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
775
+ fd = fs.openSync(file, 'wx', 0o600);
776
+ // Contents are operator-facing only; the evidence is the open descriptor.
777
+ fs.writeSync(fd, `${JSON.stringify({ node_id: claim.node_id, generation, supervisor_pid: claim.pid, state_dir: claim.state_dir }, null, 2)}\n`);
778
+ return fd;
779
+ }
780
+ catch (error) {
781
+ // Drop our reference before unlinking: an fd we opened and then abandoned
782
+ // would keep answering `lsof` for as long as this process lives, pinning a
783
+ // node id behind a fence nothing is actually being fenced by.
784
+ closeNodeClaimHold(fd);
785
+ if (fd !== undefined)
786
+ safeUnlinkClaim(file);
787
+ throw new NodeClaimHoldError(claim.node_id, file, error);
788
+ }
789
+ }
790
+ /** Drop this process's reference to a hold file. Other holders keep it alive. */
791
+ export function closeNodeClaimHold(fd) {
792
+ if (fd === undefined)
793
+ return;
794
+ try {
795
+ fs.closeSync(fd);
796
+ }
797
+ catch {
798
+ // Already closed, or never ours.
799
+ }
800
+ }
801
+ /**
802
+ * A live broker serving `stateDir`, discovered from the connection file the
803
+ * broker writes itself.
804
+ *
805
+ * This is the ownership evidence that survives the supervising CLI. The Rust
806
+ * broker writes `<state dir>/connection.json` (with its own pid) as soon as its
807
+ * API listener binds — BEFORE `connect_relay` and before `node.register` is
808
+ * queued (crates/broker/src/runtime/init.rs) — so a broker that was orphaned by
809
+ * a SIGKILLed supervisor before the CLI could record its pid is still
810
+ * discoverable by every other start on the machine. Without it, such an orphan
811
+ * reads as "nobody home" and the next `node up` evicts its delivery socket.
812
+ */
813
+ export async function findLiveStateDirBroker(stateDir, deps = {},
814
+ /** What the claim for this state dir knows about its broker executable. */
815
+ hint = {}) {
816
+ const pid = readStateDirBrokerPid(stateDir);
817
+ if (pid === null || !isProcessAlive(pid, deps)) {
818
+ return null;
819
+ }
820
+ // Unlike the hold descriptor, this pid was read from a file and can have been
821
+ // recycled by any process on the machine, with no relationship to Relay at
822
+ // all. Only a positive identification keeps the node id guarded here.
823
+ if ((await classifyHolderProcess(pid, { stateDir, ...hint }, deps)) !== 'broker') {
824
+ return null;
825
+ }
826
+ const startedAt = await readProcessStartedAt(pid, deps);
827
+ return { pid, ...(startedAt ? { startedAt } : {}) };
828
+ }
829
+ /**
830
+ * Classify a claim for `nodeId` — the read-only view used by preflight guards
831
+ * and diagnostics.
832
+ *
833
+ * A claim is retired only when every process it names is provably gone (the pid
834
+ * no longer exists, or its birth time no longer matches the one recorded) AND
835
+ * no live broker occupies its state dir. Everything else — including a `ps` we
836
+ * cannot run — reads as held, because refusing is recoverable (`--force`,
837
+ * `node down`) while a wrong "free" verdict silently cuts delivery to a live
838
+ * broker.
839
+ */
840
+ export async function inspectNodeClaim(nodeId, deps = {}) {
841
+ const env = deps.env ?? process.env;
842
+ const generations = readClaimGenerations(nodeId, env);
843
+ // A held generation can sit beneath a stale top claim: a `--force` takeover
844
+ // preserves the incumbent's record while its broker lives, and a failed
845
+ // takeover leaves it under the replacement's tombstone. Any live holder is
846
+ // a conflict for a new start, so report the newest held generation rather
847
+ // than only judging the top file. Newest-first keeps the diagnosis pointed
848
+ // at the most recent owner.
849
+ let fallback = null;
850
+ for (let index = generations.length - 1; index >= 0; index -= 1) {
851
+ const entry = generations[index];
852
+ if (!entry.claim)
853
+ continue;
854
+ const status = await classifyNodeClaim(nodeId, entry.claim, env, deps);
855
+ if (status.state === 'held')
856
+ return status;
857
+ fallback ??= status;
858
+ }
859
+ return fallback ?? { state: 'unclaimed' };
860
+ }
861
+ /**
862
+ * Decide whether `claim` still names a live owner.
863
+ *
864
+ * `selfPids` names processes that ARE the caller. They are skipped as recorded
865
+ * holders — a process cannot be a competing broker against itself — but they
866
+ * suppress nothing else: the orphan fences below still run in full, because a
867
+ * pid the caller happens to share (its own, or one recycled from the dead
868
+ * supervisor this claim records) says nothing about the broker that supervisor
869
+ * may have left registered.
870
+ */
871
+ async function classifyNodeClaim(nodeId, claim, env, deps, selfPids) {
872
+ if (claim.node_id.trim() !== nodeId.trim()) {
873
+ // Sanitized filenames can collide. Report it instead of overwriting the
874
+ // other node's claim, which would leave that broker unguarded.
875
+ return {
876
+ state: 'held',
877
+ claim,
878
+ reason: `claim file ${nodeClaimPath(nodeId, env, claim.generation ?? 1)} records node ${claim.node_id}`,
879
+ };
880
+ }
881
+ const reasons = [];
882
+ for (const candidate of claimProcesses(claim)) {
883
+ if (selfPids?.has(candidate.pid)) {
884
+ reasons.push(`${candidate.role} pid ${candidate.pid} is this start itself`);
885
+ continue;
886
+ }
887
+ const status = await isRecordedProcessAlive(candidate.pid, candidate.startedAt, deps);
888
+ if (status.alive) {
889
+ return { state: 'held', claim, reason: `${candidate.role} ${status.reason}` };
890
+ }
891
+ reasons.push(`${candidate.role} ${status.reason}`);
892
+ }
893
+ // Every recorded pid is gone, but a broker this claim started can have
894
+ // outlived them: the supervisor may have been SIGKILLed between the spawn and
895
+ // the moment it could record the broker's pid.
896
+ //
897
+ // The hold descriptor is checked FIRST because it is the only evidence that
898
+ // exists for the whole life of that orphan. A broker child inherits it across
899
+ // `fork`, so it answers even while the child is still paused before binding
900
+ // its API — the window in which `connection.json` does not exist yet and the
901
+ // old "dead supervisor, nothing published" reading let a second broker start.
902
+ const hold = await inspectClaimHold(claim, env, deps, selfPids);
903
+ if (hold.held) {
904
+ return {
905
+ state: 'held',
906
+ // Report the pid that is actually holding the node, not the supervisor
907
+ // the record still names — that one is what was just proven dead, and an
908
+ // operator told to stop it would be chasing a process that is gone.
909
+ claim: {
910
+ ...claim,
911
+ ...(hold.pids[0] !== undefined ? { pid: hold.pids[0] } : {}),
912
+ supervisor_pid: undefined,
913
+ },
914
+ reason: `${reasons.join('; ')}, but ${hold.reason}`,
915
+ };
916
+ }
917
+ // Then the broker's own connection file, which survives a supervisor that
918
+ // died after its child had published but before it could record the pid.
919
+ const orphan = await findLiveStateDirBroker(claim.state_dir, deps, {
920
+ binary: claim.broker_binary,
921
+ object: claim.broker_executable,
922
+ });
923
+ if (orphan) {
924
+ return {
925
+ state: 'held',
926
+ claim: {
927
+ ...claim,
928
+ pid: orphan.pid,
929
+ ...(orphan.startedAt ? { process_started_at: orphan.startedAt } : {}),
930
+ supervisor_pid: undefined,
931
+ status: 'active',
932
+ },
933
+ reason: `${reasons.join('; ')}, but broker pid ${orphan.pid} recorded in ` +
934
+ `${path.join(claim.state_dir, BROKER_CONNECTION_FILENAME)} is still running`,
935
+ };
936
+ }
937
+ return { state: 'stale', claim, reason: reasons.join('; ') };
938
+ }
939
+ /** Live claims on this machine, for locating a broker across state dirs. */
940
+ export async function listHeldNodeClaims(deps = {}) {
941
+ const env = deps.env ?? process.env;
942
+ const held = [];
943
+ for (const claim of listAllNodeClaims(env)) {
944
+ const status = await classifyNodeClaim(claim.node_id, claim, env, deps);
945
+ if (status.state === 'held') {
946
+ held.push(status.claim);
947
+ }
948
+ }
949
+ return held;
950
+ }
951
+ /* ------------------------------------------------------------------ *
952
+ * Acquire / adopt / release
953
+ * ------------------------------------------------------------------ */
954
+ /**
955
+ * How many generations to try before giving up. Each attempt only loses to a
956
+ * start that genuinely won the node id, so more than a couple means the machine
957
+ * is starting brokers for one node id in a tight loop.
958
+ */
959
+ const CLAIM_ACQUIRE_ATTEMPTS = 8;
960
+ /**
961
+ * Create a claim file that does not exist yet, with its full contents already
962
+ * in place.
963
+ *
964
+ * The record is written to a private temp file and hard-linked into its
965
+ * generation path: `link(2)` fails with `EEXIST` rather than clobbering, so the
966
+ * create is both exclusive AND atomic. A reader can therefore never observe a
967
+ * half-written claim — which matters, because a torn read of a live broker's
968
+ * claim would read as "node id free".
969
+ *
970
+ * @returns False when that generation already exists.
971
+ */
972
+ function createClaimGeneration(file, claim) {
973
+ const dir = path.dirname(file);
974
+ fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
975
+ const temporary = path.join(dir, `.${path.basename(file)}.${process.pid}.${crypto.randomUUID()}.tmp`);
976
+ fs.writeFileSync(temporary, `${JSON.stringify(claim, null, 2)}\n`, { mode: 0o600 });
977
+ try {
978
+ fs.linkSync(temporary, file);
979
+ return true;
980
+ }
981
+ catch (error) {
982
+ if (error?.code === 'EEXIST')
983
+ return false;
984
+ throw error;
985
+ }
986
+ finally {
987
+ try {
988
+ fs.unlinkSync(temporary);
989
+ }
990
+ catch {
991
+ // Already gone; the hard link (if any) keeps the contents alive.
992
+ }
993
+ }
994
+ }
995
+ /** Replace a claim file in place, atomically, so no reader sees a partial write. */
996
+ function writeClaimRecordAtomically(file, record) {
997
+ const temporary = `${file}.${process.pid}.${crypto.randomUUID()}.tmp`;
998
+ fs.writeFileSync(temporary, `${JSON.stringify(record, null, 2)}\n`, { mode: 0o600 });
999
+ fs.renameSync(temporary, file);
1000
+ }
1001
+ function safeUnlinkClaim(file) {
1002
+ try {
1003
+ fs.unlinkSync(file);
1004
+ }
1005
+ catch {
1006
+ // Already gone, or not ours to remove.
1007
+ }
1008
+ }
1009
+ /**
1010
+ * Remove a generation this acquisition has superseded, and its hold file.
1011
+ *
1012
+ * Only removed while the file still holds the exact bytes the scan classified.
1013
+ * Generation numbers are never reissued (see {@link NodeClaimTombstone}), so a
1014
+ * path cannot come back as somebody else's claim — but the scan that chose
1015
+ * these files happened before an `await`, and pruning by path alone is what
1016
+ * deleted a live successor's record when a number COULD be reissued. Comparing
1017
+ * the bytes makes the check local instead of resting on that invariant.
1018
+ */
1019
+ function pruneSupersededGeneration(entry, nodeId, env) {
1020
+ if (readClaimBytes(entry.file) !== entry.raw) {
1021
+ return;
1022
+ }
1023
+ safeUnlinkClaim(entry.file);
1024
+ safeUnlinkClaim(nodeClaimHoldPath(nodeId, env, entry.generation));
1025
+ }
1026
+ async function buildClaim(input, generation, deps) {
1027
+ const startedAt = await readProcessStartedAt(input.pid, deps);
1028
+ const supervisorPid = input.supervisorPid ?? input.pid;
1029
+ const supervisorStartedAt = supervisorPid === input.pid ? startedAt : await readProcessStartedAt(supervisorPid, deps);
1030
+ return {
1031
+ version: 1,
1032
+ node_id: input.nodeId.trim(),
1033
+ pid: input.pid,
1034
+ state_dir: normalizeClaimStateDir(input.stateDir),
1035
+ ...describeBrokerExecutable(input.brokerBinary),
1036
+ ...(input.apiPort !== undefined ? { api_port: input.apiPort } : {}),
1037
+ ...(input.brokerName ? { broker_name: input.brokerName } : {}),
1038
+ ...(startedAt ? { process_started_at: startedAt } : {}),
1039
+ supervisor_pid: supervisorPid,
1040
+ ...(supervisorStartedAt ? { supervisor_started_at: supervisorStartedAt } : {}),
1041
+ status: input.status ?? 'active',
1042
+ claimed_at: new Date().toISOString(),
1043
+ generation,
1044
+ owner_token: crypto.randomUUID(),
1045
+ };
1046
+ }
1047
+ /**
1048
+ * Record ownership of `nodeId`, refusing if a live local broker already holds
1049
+ * it.
1050
+ *
1051
+ * Ownership IS the exclusive creation of the next generation file. Generations
1052
+ * only increase and each one is created exactly once — for the whole life of
1053
+ * the stem, not just while a claim is live: {@link releaseNodeClaim} leaves a
1054
+ * tombstone in place of the record it drops, so the highest number on disk
1055
+ * never falls back. Taking a node id over therefore never involves deleting a
1056
+ * file another process might have replaced in the meantime — the failure mode
1057
+ * that makes "validate the holder, then remove its record, then write ours"
1058
+ * unsafe no matter how the validation is fenced.
1059
+ *
1060
+ * Without that, a start suspended between the scan and the create could wake up
1061
+ * after a complete release-and-reacquire cycle, re-create a number that had
1062
+ * been reissued to a live successor, pass the higher-generation check because
1063
+ * its own file was the highest again, and then prune that successor's claim
1064
+ * from its own stale scan. Both would stay alive with one of them recorded —
1065
+ * exactly the double registration this module exists to prevent.
1066
+ *
1067
+ * Two starts that both observe the same stale claim therefore cannot both win:
1068
+ * one creates generation N+1, the other collides (`EEXIST`) and re-reads, now
1069
+ * seeing the winner's live claim. A start that created its generation while a
1070
+ * third one was creating a higher one loses the confirm step below, removes its
1071
+ * own file and refuses. Exactly one — the highest generation NUMBER ever issued
1072
+ * for the stem, live claim or spent tombstone — can own the node id, which is
1073
+ * the invariant the post-create check below enforces.
1074
+ *
1075
+ * Callers reserve with `status: 'reserved'` and their own pid BEFORE spawning a
1076
+ * broker, open the generation's hold descriptor with {@link openNodeClaimHold}
1077
+ * so the child inherits it, and then hand ownership to the verified broker with
1078
+ * {@link adoptNodeClaim}.
1079
+ *
1080
+ * @throws NodeClaimConflictError when a live local broker holds the node id and
1081
+ * `force` was not requested.
1082
+ * @throws NodeClaimContentionError when no generation could be won at all.
1083
+ */
1084
+ export async function acquireNodeClaim(input) {
1085
+ const env = input.env ?? process.env;
1086
+ const deps = { ...input, env };
1087
+ const nodeId = input.nodeId.trim();
1088
+ const selfPids = new Set([input.pid]);
1089
+ for (let attempt = 0; attempt < CLAIM_ACQUIRE_ATTEMPTS; attempt += 1) {
1090
+ const generations = readClaimGenerations(nodeId, env);
1091
+ if (!input.force) {
1092
+ // Our own pid is never evidence that somebody else holds the node id, so
1093
+ // a start may re-take a claim that records it. That exemption is scoped
1094
+ // to the pid AS A RECORDED HOLDER and nothing more: the fence still runs.
1095
+ //
1096
+ // Exempting the whole check on pid equality (as this once did) was
1097
+ // unsound, because a pid is not an identity. A supervisor that died
1098
+ // leaving a registered broker behind records a pid the OS is free to
1099
+ // reissue, and the next start to be handed that number would have walked
1100
+ // straight past the orphan its own claim was pointing at.
1101
+ //
1102
+ // EVERY live claim refuses, not just the current one: a `--force`
1103
+ // takeover preserves a held generation it supersedes (see the prune
1104
+ // below), so more than one generation can record a live broker. Checking
1105
+ // only the highest would let a stale top claim mask a held one beneath.
1106
+ // Scan newest-first so a conflict names the most recent owner.
1107
+ for (let index = generations.length - 1; index >= 0; index -= 1) {
1108
+ const entry = generations[index];
1109
+ if (!entry.claim)
1110
+ continue;
1111
+ const status = await classifyNodeClaim(nodeId, entry.claim, env, deps, selfPids);
1112
+ if (status.state === 'held') {
1113
+ throw new NodeClaimConflictError(nodeId, status.claim);
1114
+ }
1115
+ }
1116
+ }
1117
+ const generation = highestGenerationNumber(generations) + 1;
1118
+ const file = nodeClaimPath(nodeId, env, generation);
1119
+ const claim = await buildClaim(input, generation, deps);
1120
+ if (!createClaimGeneration(file, claim)) {
1121
+ // Another start took this generation between the scan and the create.
1122
+ // Re-read: its claim is what decides whether we may continue at all.
1123
+ continue;
1124
+ }
1125
+ const after = readClaimGenerations(nodeId, env);
1126
+ if (highestGenerationNumber(after) > generation) {
1127
+ // Somebody else has already been ISSUED a higher generation number, so
1128
+ // this one cannot be the owner — whatever that higher file holds now.
1129
+ //
1130
+ // The test is the highest NUMBER on disk and not the highest live claim:
1131
+ // a spent generation is a file too. A start suspended since before a
1132
+ // release cycle could otherwise wake up, re-create a low number in the
1133
+ // gap the cycle's pruning left, see only tombstones above it and declare
1134
+ // itself the owner — while the start that had already read those
1135
+ // tombstones goes on to create `max + 1`. Neither one's prune list names
1136
+ // the other (each scanned before the other's file existed), so both stay
1137
+ // live and both spawn. Only the record with the highest number ever
1138
+ // issued can own the node id.
1139
+ safeUnlinkClaim(file);
1140
+ const winner = currentGeneration(after);
1141
+ if (winner && winner.generation > generation) {
1142
+ // A live claim above ours: that start legitimately won the node id.
1143
+ throw new NodeClaimConflictError(nodeId, winner.claim ?? claim);
1144
+ }
1145
+ // Only spent or unreadable generations sit above ours, so the node id
1146
+ // itself is free — this number just is not ours to hold. Re-scan and take
1147
+ // one above the highest file on disk instead.
1148
+ continue;
1149
+ }
1150
+ for (const superseded of generations) {
1151
+ if (!superseded.claim) {
1152
+ pruneSupersededGeneration(superseded, nodeId, env);
1153
+ continue;
1154
+ }
1155
+ // A generation whose holder is still alive is not garbage: it is the
1156
+ // only record guarding that broker's node id. A `--force` takeover that
1157
+ // pruned it here and then failed to start left the incumbent running and
1158
+ // unguarded — the next plain `node up` took the node id from under a
1159
+ // live, still-registered broker. Held generations stay until their
1160
+ // holder exits (a later acquisition prunes them as stale) or their
1161
+ // broker is stopped (releaseNodeClaimsForBroker retires them); our own
1162
+ // superseded generations are exempted from "held" the same way the
1163
+ // conflict check above exempts them, so a re-acquire still cleans up.
1164
+ const status = await classifyNodeClaim(nodeId, superseded.claim, env, deps, selfPids);
1165
+ if (status.state !== 'held') {
1166
+ pruneSupersededGeneration(superseded, nodeId, env);
1167
+ }
1168
+ }
1169
+ return claim;
1170
+ }
1171
+ throw new NodeClaimContentionError(nodeId, nodeClaimsDir(env));
1172
+ }
1173
+ function isSameAcquisition(current, claim) {
1174
+ if (!current)
1175
+ return false;
1176
+ if (claim.owner_token)
1177
+ return current.owner_token === claim.owner_token;
1178
+ // Claims written before this process (or by hand) carry no token; fall back
1179
+ // to the identity the record does have.
1180
+ return (current.node_id === claim.node_id &&
1181
+ current.pid === claim.pid &&
1182
+ claimNamesStateDir(current, claim.state_dir));
1183
+ }
1184
+ /**
1185
+ * Record the pid of the child this reservation just handed its hold descriptor
1186
+ * to, so a later start can recognise that process no matter what it becomes.
1187
+ *
1188
+ * Deliberately synchronous, and called from the spawn itself rather than from
1189
+ * the adoption that follows it: the whole point is to have the pid on disk
1190
+ * before the child can `exec` into something the recorded executable no longer
1191
+ * describes, and before this supervisor can be killed. It is a plain write of
1192
+ * OUR OWN generation with no `await` between the read and the write, so it
1193
+ * cannot interleave with anything.
1194
+ *
1195
+ * A failure is never fatal. This is additional positive evidence layered on top
1196
+ * of a fence that is already in place — the descriptor was inherited at the
1197
+ * spawn — so losing it costs the launcher/exec case, not the guarantee. The
1198
+ * caller keeps the returned claim, which is unchanged when nothing was written.
1199
+ */
1200
+ export function recordSpawnedBrokerChild(reservation, pid, env = process.env) {
1201
+ if (!Number.isSafeInteger(pid) || pid <= 0)
1202
+ return reservation;
1203
+ const generation = reservation.generation ?? 1;
1204
+ const file = nodeClaimPath(reservation.node_id, env, generation);
1205
+ try {
1206
+ // A `--force` takeover that landed during the spawn owns the node id now.
1207
+ // Rewriting its record would be this module's one forbidden move; the
1208
+ // adoption below will fail this start on the same evidence.
1209
+ if (!isSameAcquisition(readClaimFile(file), reservation))
1210
+ return reservation;
1211
+ const claim = { ...reservation, broker_child_pid: pid };
1212
+ writeClaimRecordAtomically(file, claim);
1213
+ return claim;
1214
+ }
1215
+ catch {
1216
+ return reservation;
1217
+ }
1218
+ }
1219
+ /**
1220
+ * Hand a reservation to the verified broker process that now owns the state
1221
+ * dir.
1222
+ *
1223
+ * Only the acquisition that created a generation ever rewrites it, so this is a
1224
+ * conflict-free in-place update — but it is still conditional on that
1225
+ * generation still being the highest: a `--force` takeover that landed while
1226
+ * this broker was starting has already won the node id, and continuing would
1227
+ * put two brokers back on one delivery socket. Losing here fails startup, which
1228
+ * tears this broker down.
1229
+ *
1230
+ * @throws NodeClaimConflictError when the reservation no longer owns the node id.
1231
+ */
1232
+ export async function adoptNodeClaim(input) {
1233
+ const env = input.env ?? process.env;
1234
+ const deps = { ...input, env };
1235
+ const { reservation } = input;
1236
+ const generation = reservation.generation ?? 1;
1237
+ const file = nodeClaimPath(reservation.node_id, env, generation);
1238
+ const before = readClaimGenerations(reservation.node_id, env);
1239
+ if (highestGenerationNumber(before) > generation || !isSameAcquisition(readClaimFile(file), reservation)) {
1240
+ throw new NodeClaimConflictError(reservation.node_id, currentGeneration(before)?.claim ?? reservation);
1241
+ }
1242
+ const startedAt = await readProcessStartedAt(input.pid, deps);
1243
+ const claim = {
1244
+ ...reservation,
1245
+ pid: input.pid,
1246
+ ...(input.apiPort !== undefined ? { api_port: input.apiPort } : {}),
1247
+ ...(input.brokerName ? { broker_name: input.brokerName } : {}),
1248
+ ...(startedAt ? { process_started_at: startedAt } : {}),
1249
+ supervisor_pid: reservation.pid,
1250
+ ...(reservation.process_started_at ? { supervisor_started_at: reservation.process_started_at } : {}),
1251
+ status: 'active',
1252
+ };
1253
+ if (!startedAt)
1254
+ delete claim.process_started_at;
1255
+ // Atomic replace of OUR generation only: readers see the reservation or the
1256
+ // adopted record, never a partial write, and never another start's file.
1257
+ writeClaimRecordAtomically(file, claim);
1258
+ const after = readClaimGenerations(reservation.node_id, env);
1259
+ if (highestGenerationNumber(after) > generation) {
1260
+ // A takeover landed during the write. It owns the node id; give ours up
1261
+ // rather than leaving a second live-looking record behind.
1262
+ safeUnlinkClaim(file);
1263
+ safeUnlinkClaim(nodeClaimHoldPath(reservation.node_id, env, generation));
1264
+ throw new NodeClaimConflictError(reservation.node_id, currentGeneration(after)?.claim ?? claim);
1265
+ }
1266
+ return claim;
1267
+ }
1268
+ /**
1269
+ * Drop a claim this start owns, leaving its generation number spent.
1270
+ *
1271
+ * The record is REPLACED by a tombstone rather than removed. Removing it made
1272
+ * the number reusable, and a suspended start could then re-create a generation
1273
+ * that had since been handed to somebody else: it would pass the
1274
+ * higher-generation check (its own file was the highest again) and prune the
1275
+ * successor's live claim from its own stale scan, leaving two brokers alive on
1276
+ * one node id with only one of them recorded. A tombstone keeps
1277
+ * `max(generation)` from ever decreasing, so no number is issued twice; it
1278
+ * parses as no claim at all, so the node id reads free immediately; and the
1279
+ * next acquisition prunes it, so at most one spent file per node id is ever on
1280
+ * disk.
1281
+ *
1282
+ * Only the acquisition that created a generation ever rewrites it, proven by
1283
+ * `owner_token`, and the read and the write are adjacent syscalls with no await
1284
+ * in between.
1285
+ */
1286
+ export async function releaseNodeClaim(claim, env = process.env) {
1287
+ const generation = claim.generation ?? 1;
1288
+ const file = nodeClaimPath(claim.node_id, env, generation);
1289
+ if (!isSameAcquisition(readClaimFile(file), claim)) {
1290
+ return false;
1291
+ }
1292
+ try {
1293
+ writeClaimRecordAtomically(file, {
1294
+ version: 1,
1295
+ released: true,
1296
+ node_id: claim.node_id,
1297
+ generation,
1298
+ released_at: new Date().toISOString(),
1299
+ });
1300
+ }
1301
+ catch {
1302
+ return false;
1303
+ }
1304
+ // The hold descriptor protected this generation only; both processes it
1305
+ // fenced are gone by the time a release is allowed to run.
1306
+ safeUnlinkClaim(nodeClaimHoldPath(claim.node_id, env, generation));
1307
+ return true;
1308
+ }
1309
+ /**
1310
+ * Release the claims a broker pid held in a state dir. `node down` stops a
1311
+ * broker it did not start, so it cannot name the node id the claim was written
1312
+ * for — the pid and state dir it verified are what it has.
1313
+ *
1314
+ * A claim for that state dir whose every recorded pid is already dead is
1315
+ * released too: that is the orphan left by a supervisor that was killed before
1316
+ * it could record its broker's pid, and `down` has just proven the state dir's
1317
+ * broker is gone.
1318
+ */
1319
+ export async function releaseNodeClaimsForBroker(input) {
1320
+ const env = input.env ?? process.env;
1321
+ const deps = { ...input, env };
1322
+ const released = [];
1323
+ for (const claim of listAllNodeClaims(env)) {
1324
+ if (!claimNamesStateDir(claim, input.stateDir))
1325
+ continue;
1326
+ const namesThisBroker = claim.pid === input.pid || claim.supervisor_pid === input.pid;
1327
+ if (!namesThisBroker) {
1328
+ const status = await classifyNodeClaim(claim.node_id, claim, env, deps);
1329
+ if (status.state !== 'stale')
1330
+ continue;
1331
+ }
1332
+ if (await releaseNodeClaim(claim, env)) {
1333
+ released.push(claim);
1334
+ }
1335
+ }
1336
+ return released;
1337
+ }
1338
+ /** One-line description of the broker holding a claim, for operator output. */
1339
+ export function describeNodeClaimHolder(claim) {
1340
+ const parts = [`pid ${claim.pid}`, `state dir ${claim.state_dir}`];
1341
+ if (claim.api_port !== undefined)
1342
+ parts.push(`API port ${claim.api_port}`);
1343
+ if (claim.broker_name)
1344
+ parts.push(`broker name ${claim.broker_name}`);
1345
+ if (claim.status === 'reserved')
1346
+ parts.push('starting up');
1347
+ return parts.join(', ');
1348
+ }
1349
+ //# sourceMappingURL=node-claim.js.map