agent-relay 12.2.7 → 12.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1338 @@
1
+ import crypto from 'node:crypto';
2
+ import fs from 'node:fs';
3
+ import os from 'node:os';
4
+ import path from 'node:path';
5
+ /**
6
+ * The machine-global Relay home that holds `fleet-enrollments.json`.
7
+ *
8
+ * Deliberately not imported from `@agent-relay/cloud`: that barrel pulls the
9
+ * Cloud SSH runtime, and a claim check must stay a couple of syscalls that any
10
+ * command can make. `node-claim.test.ts` pins this to the enrollment store's
11
+ * own directory so the two cannot drift apart.
12
+ */
13
+ function relayHome(env) {
14
+ return env.AGENT_RELAY_HOME ?? path.join(os.homedir(), '.agentworkforce/relay');
15
+ }
16
+ /** Filename the broker writes its API endpoint and pid into, inside its state dir. */
17
+ const BROKER_CONNECTION_FILENAME = 'connection.json';
18
+ /** Thrown when a live local broker already serves the requested node id. */
19
+ export class NodeClaimConflictError extends Error {
20
+ nodeId;
21
+ claim;
22
+ constructor(nodeId, claim) {
23
+ super(`node ${nodeId} is already served by a live local broker (pid ${claim.pid}, state dir ${claim.state_dir}). ` +
24
+ 'Stop it with `agent-relay node down --state-dir <dir>`, serve a different enrolled node, or re-run with --force.');
25
+ this.nodeId = nodeId;
26
+ this.claim = claim;
27
+ this.name = 'NodeClaimConflictError';
28
+ }
29
+ }
30
+ /**
31
+ * Thrown when the kernel-level ownership fence could not be established.
32
+ *
33
+ * The hold descriptor is what makes a claim survive its own supervisor, so a
34
+ * start that cannot open one cannot honour the guarantee it is claiming under.
35
+ * It refuses instead of spawning an unfenced broker.
36
+ */
37
+ export class NodeClaimHoldError extends Error {
38
+ nodeId;
39
+ holdPath;
40
+ constructor(nodeId, holdPath, cause) {
41
+ super(`could not establish the ownership fence for node ${nodeId} at ${holdPath}: ` +
42
+ `${cause instanceof Error ? cause.message : String(cause)}. ` +
43
+ 'Startup was stopped rather than run a broker that another start could silently take the node from. ' +
44
+ 'Ensure the claims directory is writable and has free space, then retry.', { cause });
45
+ this.nodeId = nodeId;
46
+ this.holdPath = holdPath;
47
+ this.name = 'NodeClaimHoldError';
48
+ }
49
+ }
50
+ /**
51
+ * Thrown when ownership could not be established because other starts kept
52
+ * winning the exclusive create.
53
+ *
54
+ * Each attempt is a handful of syscalls, so exhausting them means a pathological
55
+ * amount of contention on one node id. Ownership cannot be established in that
56
+ * state, and starting anyway is exactly the double registration this module
57
+ * exists to prevent.
58
+ */
59
+ export class NodeClaimContentionError extends Error {
60
+ nodeId;
61
+ claimsDir;
62
+ constructor(nodeId, claimsDir, cause) {
63
+ super(`could not take ownership of node ${nodeId}: other agent-relay starts kept winning the claim in ${claimsDir}. ` +
64
+ 'Retry, or run `agent-relay node down` on the state dir that should keep the node.', { cause });
65
+ this.nodeId = nodeId;
66
+ this.claimsDir = claimsDir;
67
+ this.name = 'NodeClaimContentionError';
68
+ }
69
+ }
70
+ /**
71
+ * The enrolled node id a start will register as, or `undefined` when it will
72
+ * mint its own identity.
73
+ *
74
+ * `RELAY_NODE_ID` is sent verbatim in `node.register` (`resolve_broker_node_id`,
75
+ * crates/broker/src/runtime/init.rs), so an explicit node id IS the identity
76
+ * that can evict another broker, and it is claimed on that basis alone.
77
+ *
78
+ * This deliberately does not try to predict whether the broker will manage to
79
+ * authenticate as it. An earlier version claimed the id only when a token was
80
+ * in the environment or already in the broker's on-disk cache, and that was a
81
+ * hole: `init.rs` also wires a workspace-key token minter, and
82
+ * `node_control.rs` mints and connects with no cached token at all, so a start
83
+ * carrying nothing but `RELAY_NODE_ID` and workspace credentials walked past a
84
+ * live claim and took the node's delivery socket — the incident this module
85
+ * exists to close. Enumerating the credential routes instead of the identity
86
+ * just moves the hole to the next route that gets added.
87
+ *
88
+ * The cost is a start that could not have registered at all (no token, no
89
+ * workspace key, nothing to mint with) being refused when some other broker
90
+ * genuinely holds the id. That is recoverable in one flag (`--force`) and the
91
+ * start it refuses was headed for local-only operation anyway, while the
92
+ * opposite error is a silent delivery outage. `--local-only` starts register
93
+ * nothing and are excluded by the callers, not here.
94
+ */
95
+ export function enrolledNodeIdForClaim(env) {
96
+ return env.RELAY_NODE_ID?.trim() || undefined;
97
+ }
98
+ /* ------------------------------------------------------------------ *
99
+ * Claim files and generations
100
+ * ------------------------------------------------------------------ */
101
+ /** Zero padding, so claim generations sort lexically as well as numerically. */
102
+ const GENERATION_DIGITS = 6;
103
+ /** Matches `<stem>.<generation>.json` for any node. */
104
+ const CLAIM_FILENAME_PATTERN = new RegExp(`^(.+)\\.(\\d{${GENERATION_DIGITS},})\\.json$`);
105
+ /** Directory holding the claim files for every node id served on this machine. */
106
+ export function nodeClaimsDir(env = process.env) {
107
+ return path.join(relayHome(env), 'node-claims');
108
+ }
109
+ /**
110
+ * Filename stem for a node id's claims. Node ids are engine-issued
111
+ * (`node_<digits>`), but the stem is sanitized anyway so a hand-edited
112
+ * enrollment store cannot write outside the claims directory. Readers verify
113
+ * `node_id` from the file contents, so a sanitized collision is reported as a
114
+ * conflict rather than silently overwriting another node's claim.
115
+ */
116
+ function nodeClaimStem(nodeId) {
117
+ return (nodeId
118
+ .trim()
119
+ .replace(/[^\w.-]/g, '-')
120
+ .slice(0, 96) || 'unnamed');
121
+ }
122
+ /**
123
+ * Path of one generation of a node id's claim.
124
+ *
125
+ * Every write this module makes is the exclusive creation of a NEW generation,
126
+ * never an overwrite of an existing one — that is what makes takeover safe
127
+ * without a lock file (see {@link acquireNodeClaim}).
128
+ */
129
+ export function nodeClaimPath(nodeId, env = process.env, generation = 1) {
130
+ const suffix = String(generation).padStart(GENERATION_DIGITS, '0');
131
+ return path.join(nodeClaimsDir(env), `${nodeClaimStem(nodeId)}.${suffix}.json`);
132
+ }
133
+ /**
134
+ * Sibling of a claim generation whose OPEN DESCRIPTORS are the ownership
135
+ * evidence, held by the kernel rather than written by a process.
136
+ *
137
+ * `<stem>.<generation>.hold` is created and opened by the supervising CLI
138
+ * BEFORE it spawns a broker, and the descriptor is inherited by the child
139
+ * across `fork`. From the instant a broker child exists — long before it binds
140
+ * its API, writes `connection.json` or queues `node.register` — some live
141
+ * process holds this file open, and the kernel drops the last reference the
142
+ * moment both of them die. That is what closes the window a supervisor
143
+ * SIGKILLed between the spawn and the broker's first write used to leave: there
144
+ * is no interval in which a competing start can see "dead supervisor, nothing
145
+ * published" and conclude the node id is free while the orphan is on its way to
146
+ * registering.
147
+ *
148
+ * Not matched by {@link CLAIM_FILENAME_PATTERN} (which requires `.json`), so it
149
+ * never counts as a generation of its own.
150
+ */
151
+ export function nodeClaimHoldPath(nodeId, env = process.env, generation = 1) {
152
+ return `${nodeClaimPath(nodeId, env, generation).slice(0, -'.json'.length)}.hold`;
153
+ }
154
+ /** Single-quote a path for a shell command (`AGENT_RELAY_HOME` is operator input). */
155
+ function shellQuote(value) {
156
+ // Close the quoted run, emit an escaped quote, reopen: ' -> '\''
157
+ return `'${value.split("'").join("'\\''")}'`;
158
+ }
159
+ /** Resolve a state dir to its canonical path so two spellings compare equal. */
160
+ export function normalizeClaimStateDir(stateDir) {
161
+ try {
162
+ return fs.realpathSync(stateDir);
163
+ }
164
+ catch {
165
+ return path.resolve(stateDir);
166
+ }
167
+ }
168
+ function isOptionalString(value) {
169
+ return value === undefined || typeof value === 'string';
170
+ }
171
+ function isOptionalPositiveInteger(value) {
172
+ return value === undefined || (Number.isSafeInteger(value) && value > 0);
173
+ }
174
+ function hasValidOptionalClaimFields(record) {
175
+ return ((record.api_port === undefined || Number.isSafeInteger(record.api_port)) &&
176
+ isOptionalString(record.broker_name) &&
177
+ isOptionalString(record.broker_binary) &&
178
+ isOptionalString(record.broker_executable) &&
179
+ isOptionalString(record.process_started_at) &&
180
+ isOptionalString(record.supervisor_started_at) &&
181
+ isOptionalString(record.owner_token) &&
182
+ isOptionalPositiveInteger(record.supervisor_pid) &&
183
+ isOptionalPositiveInteger(record.broker_child_pid) &&
184
+ isOptionalPositiveInteger(record.generation) &&
185
+ (record.status === undefined || record.status === 'reserved' || record.status === 'active'));
186
+ }
187
+ function isNodeClaim(value) {
188
+ if (typeof value !== 'object' || value === null)
189
+ return false;
190
+ const record = value;
191
+ // A tombstone occupies a generation number so it can never be reissued, but
192
+ // it is evidence of nothing. Rejecting it here is what makes it read as an
193
+ // unclaimed node id everywhere a claim is read.
194
+ if (record.released === true)
195
+ return false;
196
+ return (record.version === 1 &&
197
+ typeof record.node_id === 'string' &&
198
+ record.node_id.trim().length > 0 &&
199
+ Number.isSafeInteger(record.pid) &&
200
+ record.pid > 0 &&
201
+ typeof record.state_dir === 'string' &&
202
+ record.state_dir.length > 0 &&
203
+ typeof record.claimed_at === 'string' &&
204
+ hasValidOptionalClaimFields(record));
205
+ }
206
+ /** Exact bytes of a claim file, or `null` when it cannot be read at all. */
207
+ function readClaimBytes(file) {
208
+ try {
209
+ return fs.readFileSync(file, 'utf-8');
210
+ }
211
+ catch {
212
+ return null;
213
+ }
214
+ }
215
+ function parseClaim(raw) {
216
+ if (raw === null)
217
+ return null;
218
+ try {
219
+ const parsed = JSON.parse(raw);
220
+ return isNodeClaim(parsed) ? parsed : null;
221
+ }
222
+ catch {
223
+ return null;
224
+ }
225
+ }
226
+ function readClaimFile(file) {
227
+ return parseClaim(readClaimBytes(file));
228
+ }
229
+ /**
230
+ * Every generation file present for `nodeId`, ascending.
231
+ *
232
+ * Unreadable files are kept in the list with a `null` claim: they must still
233
+ * raise the next generation number (or an exclusive create would collide with
234
+ * them forever) even though they are no evidence of a live broker.
235
+ */
236
+ function readClaimGenerations(nodeId, env) {
237
+ const dir = nodeClaimsDir(env);
238
+ const stem = nodeClaimStem(nodeId);
239
+ let filenames;
240
+ try {
241
+ filenames = fs.readdirSync(dir);
242
+ }
243
+ catch {
244
+ return [];
245
+ }
246
+ const generations = [];
247
+ for (const filename of filenames) {
248
+ const match = CLAIM_FILENAME_PATTERN.exec(filename);
249
+ if (!match || match[1] !== stem)
250
+ continue;
251
+ const file = path.join(dir, filename);
252
+ const raw = readClaimBytes(file);
253
+ generations.push({ generation: Number.parseInt(match[2], 10), file, raw, claim: parseClaim(raw) });
254
+ }
255
+ return generations.sort((left, right) => left.generation - right.generation);
256
+ }
257
+ /** The generation that currently owns the node id: the highest readable one. */
258
+ function currentGeneration(generations) {
259
+ for (let index = generations.length - 1; index >= 0; index -= 1) {
260
+ if (generations[index].claim)
261
+ return generations[index];
262
+ }
263
+ return undefined;
264
+ }
265
+ function highestGenerationNumber(generations) {
266
+ return generations.length === 0 ? 0 : generations[generations.length - 1].generation;
267
+ }
268
+ /**
269
+ * The claim that currently owns `nodeId`, or `null`.
270
+ *
271
+ * A missing, unreadable, or malformed file reads as `null`: an unparseable
272
+ * claim is no evidence of a live broker, and treating it as one would brick
273
+ * every later `node up` for that node id.
274
+ */
275
+ export function readNodeClaim(nodeId, env = process.env) {
276
+ return currentGeneration(readClaimGenerations(nodeId, env))?.claim ?? null;
277
+ }
278
+ /** The current claim for every node id on this machine, newest first. */
279
+ export function listNodeClaims(env = process.env) {
280
+ const dir = nodeClaimsDir(env);
281
+ let filenames;
282
+ try {
283
+ filenames = fs.readdirSync(dir);
284
+ }
285
+ catch {
286
+ return [];
287
+ }
288
+ const newestByStem = new Map();
289
+ for (const filename of filenames) {
290
+ const match = CLAIM_FILENAME_PATTERN.exec(filename);
291
+ if (!match)
292
+ continue;
293
+ const claim = readClaimFile(path.join(dir, filename));
294
+ if (!claim)
295
+ continue;
296
+ const generation = Number.parseInt(match[2], 10);
297
+ const seen = newestByStem.get(match[1]);
298
+ if (!seen || seen.generation < generation) {
299
+ newestByStem.set(match[1], { generation, claim });
300
+ }
301
+ }
302
+ return [...newestByStem.values()]
303
+ .map((entry) => entry.claim)
304
+ .sort((left, right) => right.claimed_at.localeCompare(left.claimed_at));
305
+ }
306
+ /**
307
+ * Every readable claim on this machine, across ALL generations, newest first.
308
+ *
309
+ * `listNodeClaims` reports only the current generation per node id, but a
310
+ * `--force` takeover can leave a still-held incumbent beneath the
311
+ * replacement's file. Callers that diagnose or release by broker identity must
312
+ * see every generation, or that hidden claim never surfaces.
313
+ */
314
+ function listAllNodeClaims(env) {
315
+ const dir = nodeClaimsDir(env);
316
+ let filenames;
317
+ try {
318
+ filenames = fs.readdirSync(dir);
319
+ }
320
+ catch {
321
+ return [];
322
+ }
323
+ const claims = [];
324
+ for (const filename of filenames) {
325
+ const match = CLAIM_FILENAME_PATTERN.exec(filename);
326
+ if (!match)
327
+ continue;
328
+ const claim = readClaimFile(path.join(dir, filename));
329
+ if (claim)
330
+ claims.push(claim);
331
+ }
332
+ return claims.sort((left, right) => right.claimed_at.localeCompare(left.claimed_at) || (right.generation ?? 0) - (left.generation ?? 0));
333
+ }
334
+ /* ------------------------------------------------------------------ *
335
+ * Liveness
336
+ * ------------------------------------------------------------------ */
337
+ function isProcessAlive(pid, deps) {
338
+ const kill = deps.killProcess ??
339
+ ((target, signal) => {
340
+ process.kill(target, signal);
341
+ });
342
+ try {
343
+ kill(pid, 0);
344
+ return true;
345
+ }
346
+ catch (error) {
347
+ // Permission denial proves the process exists and is not ours to inspect;
348
+ // that must read as live, never as a free node id.
349
+ return error?.code === 'EPERM';
350
+ }
351
+ }
352
+ /**
353
+ * `ps` birth time for a pid, or `null` when it cannot be read. Mirrors the
354
+ * command `broker-process-identity` uses so both agree on the format.
355
+ */
356
+ async function readProcessStartedAt(pid, deps) {
357
+ const execCommand = deps.execCommand;
358
+ if (!execCommand) {
359
+ return null;
360
+ }
361
+ try {
362
+ const { stdout } = await execCommand(`LC_ALL=C TZ=UTC ps -p ${pid} -o lstart=`);
363
+ const normalized = stdout.trim().replace(/\s+/g, ' ');
364
+ return normalized.length > 0 ? normalized : null;
365
+ }
366
+ catch {
367
+ return null;
368
+ }
369
+ }
370
+ /**
371
+ * Whether `pid` is still the process that was recorded with `startedAt`.
372
+ * A pid that is gone, or one whose birth time moved (the number was recycled),
373
+ * no longer holds anything.
374
+ */
375
+ async function isRecordedProcessAlive(pid, startedAt, deps) {
376
+ if (!isProcessAlive(pid, deps)) {
377
+ return { alive: false, reason: `pid ${pid} is no longer running` };
378
+ }
379
+ if (startedAt) {
380
+ const current = await readProcessStartedAt(pid, deps);
381
+ if (current && current !== startedAt) {
382
+ return { alive: false, reason: `pid ${pid} was recycled by a process started ${current}` };
383
+ }
384
+ }
385
+ return { alive: true, reason: `pid ${pid} is running` };
386
+ }
387
+ /** The owning pid plus, when the claim records one, its supervising CLI. */
388
+ function claimProcesses(claim) {
389
+ const processes = [
390
+ {
391
+ pid: claim.pid,
392
+ startedAt: claim.process_started_at,
393
+ role: claim.status === 'reserved' ? 'supervisor' : 'broker',
394
+ },
395
+ ];
396
+ if (claim.supervisor_pid !== undefined && claim.supervisor_pid !== claim.pid) {
397
+ processes.push({
398
+ pid: claim.supervisor_pid,
399
+ startedAt: claim.supervisor_started_at,
400
+ role: 'supervisor',
401
+ });
402
+ }
403
+ return processes;
404
+ }
405
+ /** Pid recorded in `<state dir>/connection.json`, which the broker writes itself. */
406
+ function readStateDirBrokerPid(stateDir) {
407
+ try {
408
+ const parsed = JSON.parse(fs.readFileSync(path.join(stateDir, BROKER_CONNECTION_FILENAME), 'utf-8'));
409
+ if (typeof parsed !== 'object' || parsed === null)
410
+ return null;
411
+ const pid = parsed.pid;
412
+ return Number.isSafeInteger(pid) && pid > 0 ? pid : null;
413
+ }
414
+ catch {
415
+ return null;
416
+ }
417
+ }
418
+ function brokerExecutableHint(claim) {
419
+ return {
420
+ stateDir: claim.state_dir,
421
+ ...(claim.broker_binary ? { binary: claim.broker_binary } : {}),
422
+ ...(claim.broker_executable ? { object: claim.broker_executable } : {}),
423
+ ...(claim.broker_child_pid ? { childPid: claim.broker_child_pid } : {}),
424
+ };
425
+ }
426
+ /**
427
+ * Identity of the executable file a claim's start was going to run, for the
428
+ * record. `realpathSync` first so a claim written through a symlinked install
429
+ * path still names the file a running broker maps.
430
+ */
431
+ /**
432
+ * `<device>:<inode>` of a file on this machine, in the encoding `lsof -d txt`
433
+ * reports and the claim records, or `null` when it cannot be stat'ed.
434
+ */
435
+ function executableObjectOf(file) {
436
+ try {
437
+ const stats = fs.statSync(file, { bigint: true });
438
+ return `0x${stats.dev.toString(16)}:${stats.ino.toString()}`;
439
+ }
440
+ catch {
441
+ return null;
442
+ }
443
+ }
444
+ function describeBrokerExecutable(binary) {
445
+ if (!binary)
446
+ return {};
447
+ try {
448
+ const resolved = fs.realpathSync(binary);
449
+ const object = executableObjectOf(resolved);
450
+ return {
451
+ broker_binary: resolved,
452
+ ...(object ? { broker_executable: object } : {}),
453
+ };
454
+ }
455
+ catch {
456
+ // The path is still worth recording: a process running it is recognisable
457
+ // by argv even when the file cannot be stat'ed from here.
458
+ return path.isAbsolute(binary) ? { broker_binary: binary } : {};
459
+ }
460
+ }
461
+ /**
462
+ * The executable objects `pid` is running, as `<device>:<inode>` pairs, or
463
+ * `null` when they cannot be read.
464
+ *
465
+ * `txt` mappings identify an executable by device and inode on both macOS and
466
+ * Linux, which is the only handle on "what is this process running" that does
467
+ * not go through a filename. Same command and same field encoding as
468
+ * `readBrokerProcessIdentity` in `broker-process-identity.ts`, so the two agree
469
+ * on what an executable's identity is.
470
+ */
471
+ async function readExecutableObjects(pid, deps) {
472
+ const execCommand = deps.execCommand;
473
+ if (!execCommand)
474
+ return null;
475
+ let stdout;
476
+ try {
477
+ ({ stdout } = await execCommand(`LC_ALL=C lsof -nP -a -p ${pid} -d txt -FfDi`));
478
+ }
479
+ catch {
480
+ return null;
481
+ }
482
+ const fields = stdout.trim().split('\n');
483
+ if (fields.shift() !== `p${pid}`)
484
+ return null;
485
+ const objects = new Set();
486
+ for (let index = 0; index < fields.length; index += 3) {
487
+ const [descriptor, device, inode] = fields.slice(index, index + 3);
488
+ if (descriptor !== 'ftxt' || !/^D0x[0-9a-f]+$/i.test(device ?? '') || !/^i[1-9]\d*$/.test(inode ?? '')) {
489
+ return null;
490
+ }
491
+ objects.add(`${device.slice(1).toLowerCase()}:${inode.slice(1)}`);
492
+ }
493
+ return objects.size > 0 ? objects : null;
494
+ }
495
+ /**
496
+ * Whether a command line names the executable the claim recorded, in a
497
+ * position an interpreter would put it.
498
+ *
499
+ * A shebang launcher does not RUN the file the claim recorded — the kernel runs
500
+ * the interpreter from its `#!` line and passes the script as an argument, so
501
+ * `lsof -d txt` reports `/bin/sh` and the recorded device/inode matches
502
+ * nothing. The script path is still right there in argv, and the file it names
503
+ * has exactly the identity the claim recorded. Comparing that file rather than
504
+ * the string is what keeps this working through a symlinked install path, the
505
+ * same reason the claim records an inode instead of a name.
506
+ *
507
+ * Only the first two arguments are considered: `argv[0]` is the program and
508
+ * `argv[1]` is where an interpreter puts its script. Anything further along is
509
+ * a broker's own argument and proves nothing about what it is running.
510
+ */
511
+ function runsRecordedExecutable(args, hint) {
512
+ if (!hint.binary && !hint.object)
513
+ return false;
514
+ for (const token of args.split(/\s+/, 2)) {
515
+ if (!token.startsWith('/'))
516
+ continue;
517
+ if (hint.binary && token === hint.binary)
518
+ return true;
519
+ if (!hint.object)
520
+ continue;
521
+ const object = executableObjectOf(token);
522
+ if (object !== null && object === hint.object)
523
+ return true;
524
+ }
525
+ return false;
526
+ }
527
+ /**
528
+ * Everything a command line can say IN FAVOUR of a process being this claim's
529
+ * broker. Nothing here is ever read the other way round: a command line that
530
+ * matches none of it is not evidence of anything, because the broker binary is
531
+ * operator-selectable and a start on the default state dir passes no
532
+ * `--state-dir` for argv to carry.
533
+ */
534
+ function argvIdentifiesBroker(args, hint) {
535
+ if (hint.binary && (args === hint.binary || args.startsWith(`${hint.binary} `)))
536
+ return true;
537
+ if (runsRecordedExecutable(args, hint))
538
+ return true;
539
+ return args.includes('agent-relay') || args.includes('relay-broker') || args.includes(hint.stateDir);
540
+ }
541
+ /**
542
+ * Whether `pid` maps the executable object the claim recorded.
543
+ *
544
+ * Mappings that cannot be read are NOT a mismatch — the process may simply not
545
+ * be ours to inspect — so they answer yes, like every other unanswerable probe
546
+ * in this module.
547
+ */
548
+ async function mapsRecordedExecutable(pid, object, deps) {
549
+ const objects = await readExecutableObjects(pid, deps);
550
+ return objects === null || objects.has(object);
551
+ }
552
+ /**
553
+ * Whether `pid` is the broker a claim names, an unrelated process that merely
554
+ * inherited its descriptor or its pid number, or something this claim has no
555
+ * way to tell apart.
556
+ *
557
+ * Executable identity decides it: the device and inode of the file the process
558
+ * is running, against the broker executable the claim recorded. Command lines
559
+ * are consulted only as further POSITIVE evidence and never to rule a process
560
+ * out. `AGENT_RELAY_BIN` / `BROKER_BINARY_PATH` let a supported deployment run
561
+ * the broker under any filename, and with the default state dir such a broker
562
+ * carries neither `agent-relay` nor its state dir in argv — so the name test
563
+ * that used to decide this classified a real, live broker as unrelated and
564
+ * handed its node id to the next start.
565
+ *
566
+ * Those same overrides accept a LAUNCHER — a shell script that sets something
567
+ * up and `exec`s the broker is the ordinary shape of a custom install — and a
568
+ * launcher is not the file the running process maps. Before its `exec` the
569
+ * process runs the interpreter from the script's `#!` line; after it, the
570
+ * binary the script chose. Neither is the file recorded before the spawn, so
571
+ * executable identity alone ruled a live, fenced startup launcher out and
572
+ * reopened exactly the eviction this claim exists to prevent. Two further
573
+ * pieces of positive evidence close that: the recorded file appearing in argv
574
+ * where an interpreter puts its script ({@link runsRecordedExecutable}), which
575
+ * needs nothing recorded after the spawn, and the pid of the child this start
576
+ * spawned ({@link NodeClaim.broker_child_pid}), which `execve` preserves
577
+ * whatever the launcher turns into.
578
+ *
579
+ * Anything that cannot be consulted — no runner, an `lsof` or `ps` that fails —
580
+ * reads as `broker`: this only ever decides whether to KEEP guarding a node id,
581
+ * and a spurious refusal is recoverable (`--force`, `node down`) while a wrong
582
+ * "free" verdict is the silent delivery outage.
583
+ */
584
+ async function classifyHolderProcess(pid, hint, deps) {
585
+ // Recorded at the spawn itself, so it is the one piece of evidence that is
586
+ // immune to whatever the child turns into afterwards.
587
+ if (hint.childPid !== undefined && pid === hint.childPid)
588
+ return 'broker';
589
+ const execCommand = deps.execCommand;
590
+ if (!execCommand)
591
+ return 'broker';
592
+ if (hint.object && (await mapsRecordedExecutable(pid, hint.object, deps)))
593
+ return 'broker';
594
+ let args;
595
+ try {
596
+ const { stdout } = await execCommand(`LC_ALL=C ps -p ${pid} -o args=`);
597
+ args = stdout.trim();
598
+ }
599
+ catch {
600
+ return 'broker';
601
+ }
602
+ // `ps` answered with nothing: the pid is gone, so it holds nothing.
603
+ if (!args)
604
+ return 'unrelated';
605
+ if (argvIdentifiesBroker(args, hint))
606
+ return 'broker';
607
+ return hint.object || hint.binary ? 'unrelated' : 'unidentified';
608
+ }
609
+ /**
610
+ * Whether any live process still holds this generation's hold file open.
611
+ *
612
+ * `lsof -t` answers from the kernel's open-file table, so this reports the
613
+ * broker child a SIGKILLed supervisor left behind from the instant that child
614
+ * exists — there is no publish step to wait for and nothing to go stale. The
615
+ * exit status is read from an explicit marker rather than from the runner's
616
+ * error shape: `lsof` exits 1 with no output when a file simply has no holders,
617
+ * and that has to be told apart from an `lsof` that could not run at all.
618
+ *
619
+ * An `lsof` that cannot be consulted reads as HELD, matching the rest of this
620
+ * module: `node up` already refuses to start without a usable `lsof` (it is how
621
+ * broker process identity is verified), and a spurious refusal is recoverable
622
+ * with `--force` or `node down` while a wrong "free" verdict is the silent
623
+ * delivery outage the claim exists to prevent.
624
+ *
625
+ * Holders are classified by {@link classifyHolderProcess}, exactly as the
626
+ * connection file's pid is. An inherited descriptor is not close-on-exec, so
627
+ * anything the broker itself spawns inherits it too; a harness left running by
628
+ * a SIGKILLed broker would otherwise pin the node id with no broker anywhere
629
+ * near it. Dropping one is a positive determination and needs BOTH halves of
630
+ * the evidence: the holder identified as some other executable, AND a claim
631
+ * that durably recorded the child it spawned. Without the second half the first
632
+ * cannot be trusted — a process that has `exec`d is not the file the claim
633
+ * recorded — so every holder stays a holder, as one does when the claim cannot
634
+ * classify it at all. Nothing but a Relay start ever passes this descriptor on,
635
+ * and a spurious refusal is recoverable where a wrong "free" verdict is not.
636
+ *
637
+ * `selfPids` are dropped from the holder set: a start re-reading its own
638
+ * reservation still has that generation's descriptor open, and its own
639
+ * descriptor is not evidence that somebody else owns the node id.
640
+ */
641
+ async function inspectClaimHold(claim, env, deps, selfPids) {
642
+ const file = nodeClaimHoldPath(claim.node_id, env, claim.generation ?? 1);
643
+ if (!fs.existsSync(file)) {
644
+ return { held: false, reason: `no start holds ${file}`, pids: [] };
645
+ }
646
+ const execCommand = deps.execCommand;
647
+ if (!execCommand) {
648
+ return { held: true, reason: `open descriptors on ${file} could not be checked`, pids: [] };
649
+ }
650
+ let stdout;
651
+ try {
652
+ ({ stdout } = await execCommand(`LC_ALL=C lsof -t -- ${shellQuote(file)} 2>/dev/null; printf 'rc=%d' "$?"`));
653
+ }
654
+ catch {
655
+ return { held: true, reason: `open descriptors on ${file} could not be checked`, pids: [] };
656
+ }
657
+ const status = /rc=(\d+)/.exec(stdout);
658
+ const pids = stdout
659
+ .replace(/rc=\d+/, '')
660
+ .split(/\s+/)
661
+ .map((value) => value.trim())
662
+ .filter((value) => /^\d+$/.test(value))
663
+ .filter((value) => !selfPids?.has(Number(value)));
664
+ const hint = brokerExecutableHint(claim);
665
+ // Ruling a holder OUT requires the claim to know which process this start
666
+ // actually spawned. Everything recorded before a spawn describes a FILE, and
667
+ // `execve` swaps the file while keeping the descriptor: a launcher that has
668
+ // already exec'd the real broker maps a binary the claim never recorded and
669
+ // no longer carries the launcher anywhere in its argv. `broker_child_pid` is
670
+ // what covers that — but it is published after the spawn, so a supervisor
671
+ // SIGKILLed in between leaves a claim that can never identify its own broker,
672
+ // and the one live process holding the node's fence classifies as
673
+ // `unrelated`. The node id then reads free while that broker is on its way to
674
+ // registering, which is the eviction this module exists to prevent.
675
+ //
676
+ // So until child identity is durably established, a holder of this descriptor
677
+ // keeps the node guarded whatever it is running. Nothing but a Relay start
678
+ // ever passes this descriptor on, and the bias is the module's usual one: a
679
+ // refusal costs one `--force`, a wrong "free" verdict costs deliveries.
680
+ const childEstablished = claim.broker_child_pid !== undefined;
681
+ const brokers = [];
682
+ const unprovable = [];
683
+ for (const pid of pids) {
684
+ const verdict = await classifyHolderProcess(Number(pid), hint, deps);
685
+ if (verdict !== 'unrelated')
686
+ brokers.push(pid);
687
+ else if (!childEstablished)
688
+ unprovable.push(pid);
689
+ }
690
+ if (brokers.length > 0) {
691
+ return {
692
+ held: true,
693
+ reason: `pid ${brokers.join(', ')} still holds ${file} open`,
694
+ pids: brokers.map(Number),
695
+ };
696
+ }
697
+ if (unprovable.length > 0) {
698
+ return {
699
+ held: true,
700
+ reason: `pid ${unprovable.join(', ')} still holds ${file} open, and the start that created it ` +
701
+ 'never recorded the broker child it spawned, so that process cannot be ruled out',
702
+ pids: unprovable.map(Number),
703
+ };
704
+ }
705
+ if (pids.length > 0) {
706
+ return {
707
+ held: false,
708
+ reason: `only unrelated processes (pid ${pids.join(', ')}) hold ${file} open`,
709
+ pids: [],
710
+ };
711
+ }
712
+ // `lsof` exits 1 for "no holders"; anything else means it could not answer.
713
+ if (!status || (status[1] !== '0' && status[1] !== '1')) {
714
+ return { held: true, reason: `open descriptors on ${file} could not be checked`, pids: [] };
715
+ }
716
+ return { held: false, reason: `no process holds ${file} open`, pids: [] };
717
+ }
718
+ /**
719
+ * Whether anything other than `selfPids` still holds this claim's generation
720
+ * fenced, for a caller deciding whether ownership may be dropped.
721
+ *
722
+ * A release has to consult this, not just the pids on the record. The record
723
+ * names a broker only once a start got far enough to capture one: a spawn that
724
+ * rejects before it returns a client — a handshake that never completed, a
725
+ * SIGTERM the child outlived — leaves a broker child that no pid anywhere names,
726
+ * and releasing on "no live pid recorded" tombstones the claim and unlinks the
727
+ * hold file out from under a process that is still fenced by it. The next start
728
+ * then reads the node id as free while that child can still register. The
729
+ * descriptor is the one piece of evidence that exists from the instant the child
730
+ * does, so it is what decides.
731
+ *
732
+ * Holders are filtered exactly as {@link inspectNodeClaim} filters them, so a
733
+ * release never frees a node id that another start would still read as held.
734
+ */
735
+ export async function inspectNodeClaimHold(claim, deps = {}, selfPids) {
736
+ return inspectClaimHold(claim, deps.env ?? process.env, deps, selfPids);
737
+ }
738
+ /**
739
+ * Create and open this generation's hold file, handing the supervising CLI the
740
+ * descriptor it passes to the broker child.
741
+ *
742
+ * Created exclusively (`wx`): only the acquisition that won the generation ever
743
+ * creates it, so a concurrent start can never replace the inode a live child is
744
+ * holding. Opened BEFORE the spawn, because a fence established after `fork`
745
+ * would leave exactly the gap it exists to close.
746
+ *
747
+ * Failing to establish the hold is FATAL to the start, which is why this throws
748
+ * rather than reporting a missing fence. Without the descriptor, a supervisor
749
+ * killed between the fork and the broker's `connection.json` write leaves a
750
+ * claim with only dead pids and no evidence at all — so the next start reads
751
+ * the node id as free and evicts a broker that is about to register. Continuing
752
+ * would mean spawning a broker under a guarantee that is not actually in force,
753
+ * which is worse than not starting: the operator can see and fix a refusal.
754
+ *
755
+ * @throws NodeClaimHoldError when the hold could not be established. Any
756
+ * partially created file and descriptor are cleaned up first, so a retry (or a
757
+ * later start, on the next generation) is not blocked by this one's debris.
758
+ */
759
+ export function openNodeClaimHold(claim, env = process.env) {
760
+ const generation = claim.generation ?? 1;
761
+ const file = nodeClaimHoldPath(claim.node_id, env, generation);
762
+ let fd;
763
+ try {
764
+ fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
765
+ fd = fs.openSync(file, 'wx', 0o600);
766
+ // Contents are operator-facing only; the evidence is the open descriptor.
767
+ fs.writeSync(fd, `${JSON.stringify({ node_id: claim.node_id, generation, supervisor_pid: claim.pid, state_dir: claim.state_dir }, null, 2)}\n`);
768
+ return fd;
769
+ }
770
+ catch (error) {
771
+ // Drop our reference before unlinking: an fd we opened and then abandoned
772
+ // would keep answering `lsof` for as long as this process lives, pinning a
773
+ // node id behind a fence nothing is actually being fenced by.
774
+ closeNodeClaimHold(fd);
775
+ if (fd !== undefined)
776
+ safeUnlinkClaim(file);
777
+ throw new NodeClaimHoldError(claim.node_id, file, error);
778
+ }
779
+ }
780
+ /** Drop this process's reference to a hold file. Other holders keep it alive. */
781
+ export function closeNodeClaimHold(fd) {
782
+ if (fd === undefined)
783
+ return;
784
+ try {
785
+ fs.closeSync(fd);
786
+ }
787
+ catch {
788
+ // Already closed, or never ours.
789
+ }
790
+ }
791
+ /**
792
+ * A live broker serving `stateDir`, discovered from the connection file the
793
+ * broker writes itself.
794
+ *
795
+ * This is the ownership evidence that survives the supervising CLI. The Rust
796
+ * broker writes `<state dir>/connection.json` (with its own pid) as soon as its
797
+ * API listener binds — BEFORE `connect_relay` and before `node.register` is
798
+ * queued (crates/broker/src/runtime/init.rs) — so a broker that was orphaned by
799
+ * a SIGKILLed supervisor before the CLI could record its pid is still
800
+ * discoverable by every other start on the machine. Without it, such an orphan
801
+ * reads as "nobody home" and the next `node up` evicts its delivery socket.
802
+ */
803
+ export async function findLiveStateDirBroker(stateDir, deps = {},
804
+ /** What the claim for this state dir knows about its broker executable. */
805
+ hint = {}) {
806
+ const pid = readStateDirBrokerPid(stateDir);
807
+ if (pid === null || !isProcessAlive(pid, deps)) {
808
+ return null;
809
+ }
810
+ // Unlike the hold descriptor, this pid was read from a file and can have been
811
+ // recycled by any process on the machine, with no relationship to Relay at
812
+ // all. Only a positive identification keeps the node id guarded here.
813
+ if ((await classifyHolderProcess(pid, { stateDir, ...hint }, deps)) !== 'broker') {
814
+ return null;
815
+ }
816
+ const startedAt = await readProcessStartedAt(pid, deps);
817
+ return { pid, ...(startedAt ? { startedAt } : {}) };
818
+ }
819
+ /**
820
+ * Classify a claim for `nodeId` — the read-only view used by preflight guards
821
+ * and diagnostics.
822
+ *
823
+ * A claim is retired only when every process it names is provably gone (the pid
824
+ * no longer exists, or its birth time no longer matches the one recorded) AND
825
+ * no live broker occupies its state dir. Everything else — including a `ps` we
826
+ * cannot run — reads as held, because refusing is recoverable (`--force`,
827
+ * `node down`) while a wrong "free" verdict silently cuts delivery to a live
828
+ * broker.
829
+ */
830
+ export async function inspectNodeClaim(nodeId, deps = {}) {
831
+ const env = deps.env ?? process.env;
832
+ const generations = readClaimGenerations(nodeId, env);
833
+ // A held generation can sit beneath a stale top claim: a `--force` takeover
834
+ // preserves the incumbent's record while its broker lives, and a failed
835
+ // takeover leaves it under the replacement's tombstone. Any live holder is
836
+ // a conflict for a new start, so report the newest held generation rather
837
+ // than only judging the top file. Newest-first keeps the diagnosis pointed
838
+ // at the most recent owner.
839
+ let fallback = null;
840
+ for (let index = generations.length - 1; index >= 0; index -= 1) {
841
+ const entry = generations[index];
842
+ if (!entry.claim)
843
+ continue;
844
+ const status = await classifyNodeClaim(nodeId, entry.claim, env, deps);
845
+ if (status.state === 'held')
846
+ return status;
847
+ fallback ??= status;
848
+ }
849
+ return fallback ?? { state: 'unclaimed' };
850
+ }
851
+ /**
852
+ * Decide whether `claim` still names a live owner.
853
+ *
854
+ * `selfPids` names processes that ARE the caller. They are skipped as recorded
855
+ * holders — a process cannot be a competing broker against itself — but they
856
+ * suppress nothing else: the orphan fences below still run in full, because a
857
+ * pid the caller happens to share (its own, or one recycled from the dead
858
+ * supervisor this claim records) says nothing about the broker that supervisor
859
+ * may have left registered.
860
+ */
861
+ async function classifyNodeClaim(nodeId, claim, env, deps, selfPids) {
862
+ if (claim.node_id.trim() !== nodeId.trim()) {
863
+ // Sanitized filenames can collide. Report it instead of overwriting the
864
+ // other node's claim, which would leave that broker unguarded.
865
+ return {
866
+ state: 'held',
867
+ claim,
868
+ reason: `claim file ${nodeClaimPath(nodeId, env, claim.generation ?? 1)} records node ${claim.node_id}`,
869
+ };
870
+ }
871
+ const reasons = [];
872
+ for (const candidate of claimProcesses(claim)) {
873
+ if (selfPids?.has(candidate.pid)) {
874
+ reasons.push(`${candidate.role} pid ${candidate.pid} is this start itself`);
875
+ continue;
876
+ }
877
+ const status = await isRecordedProcessAlive(candidate.pid, candidate.startedAt, deps);
878
+ if (status.alive) {
879
+ return { state: 'held', claim, reason: `${candidate.role} ${status.reason}` };
880
+ }
881
+ reasons.push(`${candidate.role} ${status.reason}`);
882
+ }
883
+ // Every recorded pid is gone, but a broker this claim started can have
884
+ // outlived them: the supervisor may have been SIGKILLed between the spawn and
885
+ // the moment it could record the broker's pid.
886
+ //
887
+ // The hold descriptor is checked FIRST because it is the only evidence that
888
+ // exists for the whole life of that orphan. A broker child inherits it across
889
+ // `fork`, so it answers even while the child is still paused before binding
890
+ // its API — the window in which `connection.json` does not exist yet and the
891
+ // old "dead supervisor, nothing published" reading let a second broker start.
892
+ const hold = await inspectClaimHold(claim, env, deps, selfPids);
893
+ if (hold.held) {
894
+ return {
895
+ state: 'held',
896
+ // Report the pid that is actually holding the node, not the supervisor
897
+ // the record still names — that one is what was just proven dead, and an
898
+ // operator told to stop it would be chasing a process that is gone.
899
+ claim: {
900
+ ...claim,
901
+ ...(hold.pids[0] !== undefined ? { pid: hold.pids[0] } : {}),
902
+ supervisor_pid: undefined,
903
+ },
904
+ reason: `${reasons.join('; ')}, but ${hold.reason}`,
905
+ };
906
+ }
907
+ // Then the broker's own connection file, which survives a supervisor that
908
+ // died after its child had published but before it could record the pid.
909
+ const orphan = await findLiveStateDirBroker(claim.state_dir, deps, {
910
+ binary: claim.broker_binary,
911
+ object: claim.broker_executable,
912
+ });
913
+ if (orphan) {
914
+ return {
915
+ state: 'held',
916
+ claim: {
917
+ ...claim,
918
+ pid: orphan.pid,
919
+ ...(orphan.startedAt ? { process_started_at: orphan.startedAt } : {}),
920
+ supervisor_pid: undefined,
921
+ status: 'active',
922
+ },
923
+ reason: `${reasons.join('; ')}, but broker pid ${orphan.pid} recorded in ` +
924
+ `${path.join(claim.state_dir, BROKER_CONNECTION_FILENAME)} is still running`,
925
+ };
926
+ }
927
+ return { state: 'stale', claim, reason: reasons.join('; ') };
928
+ }
929
+ /** Live claims on this machine, for locating a broker across state dirs. */
930
+ export async function listHeldNodeClaims(deps = {}) {
931
+ const env = deps.env ?? process.env;
932
+ const held = [];
933
+ for (const claim of listAllNodeClaims(env)) {
934
+ const status = await classifyNodeClaim(claim.node_id, claim, env, deps);
935
+ if (status.state === 'held') {
936
+ held.push(status.claim);
937
+ }
938
+ }
939
+ return held;
940
+ }
941
+ /* ------------------------------------------------------------------ *
942
+ * Acquire / adopt / release
943
+ * ------------------------------------------------------------------ */
944
+ /**
945
+ * How many generations to try before giving up. Each attempt only loses to a
946
+ * start that genuinely won the node id, so more than a couple means the machine
947
+ * is starting brokers for one node id in a tight loop.
948
+ */
949
+ const CLAIM_ACQUIRE_ATTEMPTS = 8;
950
+ /**
951
+ * Create a claim file that does not exist yet, with its full contents already
952
+ * in place.
953
+ *
954
+ * The record is written to a private temp file and hard-linked into its
955
+ * generation path: `link(2)` fails with `EEXIST` rather than clobbering, so the
956
+ * create is both exclusive AND atomic. A reader can therefore never observe a
957
+ * half-written claim — which matters, because a torn read of a live broker's
958
+ * claim would read as "node id free".
959
+ *
960
+ * @returns False when that generation already exists.
961
+ */
962
+ function createClaimGeneration(file, claim) {
963
+ const dir = path.dirname(file);
964
+ fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
965
+ const temporary = path.join(dir, `.${path.basename(file)}.${process.pid}.${crypto.randomUUID()}.tmp`);
966
+ fs.writeFileSync(temporary, `${JSON.stringify(claim, null, 2)}\n`, { mode: 0o600 });
967
+ try {
968
+ fs.linkSync(temporary, file);
969
+ return true;
970
+ }
971
+ catch (error) {
972
+ if (error?.code === 'EEXIST')
973
+ return false;
974
+ throw error;
975
+ }
976
+ finally {
977
+ try {
978
+ fs.unlinkSync(temporary);
979
+ }
980
+ catch {
981
+ // Already gone; the hard link (if any) keeps the contents alive.
982
+ }
983
+ }
984
+ }
985
+ /** Replace a claim file in place, atomically, so no reader sees a partial write. */
986
+ function writeClaimRecordAtomically(file, record) {
987
+ const temporary = `${file}.${process.pid}.${crypto.randomUUID()}.tmp`;
988
+ fs.writeFileSync(temporary, `${JSON.stringify(record, null, 2)}\n`, { mode: 0o600 });
989
+ fs.renameSync(temporary, file);
990
+ }
991
+ function safeUnlinkClaim(file) {
992
+ try {
993
+ fs.unlinkSync(file);
994
+ }
995
+ catch {
996
+ // Already gone, or not ours to remove.
997
+ }
998
+ }
999
+ /**
1000
+ * Remove a generation this acquisition has superseded, and its hold file.
1001
+ *
1002
+ * Only removed while the file still holds the exact bytes the scan classified.
1003
+ * Generation numbers are never reissued (see {@link NodeClaimTombstone}), so a
1004
+ * path cannot come back as somebody else's claim — but the scan that chose
1005
+ * these files happened before an `await`, and pruning by path alone is what
1006
+ * deleted a live successor's record when a number COULD be reissued. Comparing
1007
+ * the bytes makes the check local instead of resting on that invariant.
1008
+ */
1009
+ function pruneSupersededGeneration(entry, nodeId, env) {
1010
+ if (readClaimBytes(entry.file) !== entry.raw) {
1011
+ return;
1012
+ }
1013
+ safeUnlinkClaim(entry.file);
1014
+ safeUnlinkClaim(nodeClaimHoldPath(nodeId, env, entry.generation));
1015
+ }
1016
+ async function buildClaim(input, generation, deps) {
1017
+ const startedAt = await readProcessStartedAt(input.pid, deps);
1018
+ const supervisorPid = input.supervisorPid ?? input.pid;
1019
+ const supervisorStartedAt = supervisorPid === input.pid ? startedAt : await readProcessStartedAt(supervisorPid, deps);
1020
+ return {
1021
+ version: 1,
1022
+ node_id: input.nodeId.trim(),
1023
+ pid: input.pid,
1024
+ state_dir: normalizeClaimStateDir(input.stateDir),
1025
+ ...describeBrokerExecutable(input.brokerBinary),
1026
+ ...(input.apiPort !== undefined ? { api_port: input.apiPort } : {}),
1027
+ ...(input.brokerName ? { broker_name: input.brokerName } : {}),
1028
+ ...(startedAt ? { process_started_at: startedAt } : {}),
1029
+ supervisor_pid: supervisorPid,
1030
+ ...(supervisorStartedAt ? { supervisor_started_at: supervisorStartedAt } : {}),
1031
+ status: input.status ?? 'active',
1032
+ claimed_at: new Date().toISOString(),
1033
+ generation,
1034
+ owner_token: crypto.randomUUID(),
1035
+ };
1036
+ }
1037
+ /**
1038
+ * Record ownership of `nodeId`, refusing if a live local broker already holds
1039
+ * it.
1040
+ *
1041
+ * Ownership IS the exclusive creation of the next generation file. Generations
1042
+ * only increase and each one is created exactly once — for the whole life of
1043
+ * the stem, not just while a claim is live: {@link releaseNodeClaim} leaves a
1044
+ * tombstone in place of the record it drops, so the highest number on disk
1045
+ * never falls back. Taking a node id over therefore never involves deleting a
1046
+ * file another process might have replaced in the meantime — the failure mode
1047
+ * that makes "validate the holder, then remove its record, then write ours"
1048
+ * unsafe no matter how the validation is fenced.
1049
+ *
1050
+ * Without that, a start suspended between the scan and the create could wake up
1051
+ * after a complete release-and-reacquire cycle, re-create a number that had
1052
+ * been reissued to a live successor, pass the higher-generation check because
1053
+ * its own file was the highest again, and then prune that successor's claim
1054
+ * from its own stale scan. Both would stay alive with one of them recorded —
1055
+ * exactly the double registration this module exists to prevent.
1056
+ *
1057
+ * Two starts that both observe the same stale claim therefore cannot both win:
1058
+ * one creates generation N+1, the other collides (`EEXIST`) and re-reads, now
1059
+ * seeing the winner's live claim. A start that created its generation while a
1060
+ * third one was creating a higher one loses the confirm step below, removes its
1061
+ * own file and refuses. Exactly one — the highest generation NUMBER ever issued
1062
+ * for the stem, live claim or spent tombstone — can own the node id, which is
1063
+ * the invariant the post-create check below enforces.
1064
+ *
1065
+ * Callers reserve with `status: 'reserved'` and their own pid BEFORE spawning a
1066
+ * broker, open the generation's hold descriptor with {@link openNodeClaimHold}
1067
+ * so the child inherits it, and then hand ownership to the verified broker with
1068
+ * {@link adoptNodeClaim}.
1069
+ *
1070
+ * @throws NodeClaimConflictError when a live local broker holds the node id and
1071
+ * `force` was not requested.
1072
+ * @throws NodeClaimContentionError when no generation could be won at all.
1073
+ */
1074
+ export async function acquireNodeClaim(input) {
1075
+ const env = input.env ?? process.env;
1076
+ const deps = { ...input, env };
1077
+ const nodeId = input.nodeId.trim();
1078
+ const selfPids = new Set([input.pid]);
1079
+ for (let attempt = 0; attempt < CLAIM_ACQUIRE_ATTEMPTS; attempt += 1) {
1080
+ const generations = readClaimGenerations(nodeId, env);
1081
+ if (!input.force) {
1082
+ // Our own pid is never evidence that somebody else holds the node id, so
1083
+ // a start may re-take a claim that records it. That exemption is scoped
1084
+ // to the pid AS A RECORDED HOLDER and nothing more: the fence still runs.
1085
+ //
1086
+ // Exempting the whole check on pid equality (as this once did) was
1087
+ // unsound, because a pid is not an identity. A supervisor that died
1088
+ // leaving a registered broker behind records a pid the OS is free to
1089
+ // reissue, and the next start to be handed that number would have walked
1090
+ // straight past the orphan its own claim was pointing at.
1091
+ //
1092
+ // EVERY live claim refuses, not just the current one: a `--force`
1093
+ // takeover preserves a held generation it supersedes (see the prune
1094
+ // below), so more than one generation can record a live broker. Checking
1095
+ // only the highest would let a stale top claim mask a held one beneath.
1096
+ // Scan newest-first so a conflict names the most recent owner.
1097
+ for (let index = generations.length - 1; index >= 0; index -= 1) {
1098
+ const entry = generations[index];
1099
+ if (!entry.claim)
1100
+ continue;
1101
+ const status = await classifyNodeClaim(nodeId, entry.claim, env, deps, selfPids);
1102
+ if (status.state === 'held') {
1103
+ throw new NodeClaimConflictError(nodeId, status.claim);
1104
+ }
1105
+ }
1106
+ }
1107
+ const generation = highestGenerationNumber(generations) + 1;
1108
+ const file = nodeClaimPath(nodeId, env, generation);
1109
+ const claim = await buildClaim(input, generation, deps);
1110
+ if (!createClaimGeneration(file, claim)) {
1111
+ // Another start took this generation between the scan and the create.
1112
+ // Re-read: its claim is what decides whether we may continue at all.
1113
+ continue;
1114
+ }
1115
+ const after = readClaimGenerations(nodeId, env);
1116
+ if (highestGenerationNumber(after) > generation) {
1117
+ // Somebody else has already been ISSUED a higher generation number, so
1118
+ // this one cannot be the owner — whatever that higher file holds now.
1119
+ //
1120
+ // The test is the highest NUMBER on disk and not the highest live claim:
1121
+ // a spent generation is a file too. A start suspended since before a
1122
+ // release cycle could otherwise wake up, re-create a low number in the
1123
+ // gap the cycle's pruning left, see only tombstones above it and declare
1124
+ // itself the owner — while the start that had already read those
1125
+ // tombstones goes on to create `max + 1`. Neither one's prune list names
1126
+ // the other (each scanned before the other's file existed), so both stay
1127
+ // live and both spawn. Only the record with the highest number ever
1128
+ // issued can own the node id.
1129
+ safeUnlinkClaim(file);
1130
+ const winner = currentGeneration(after);
1131
+ if (winner && winner.generation > generation) {
1132
+ // A live claim above ours: that start legitimately won the node id.
1133
+ throw new NodeClaimConflictError(nodeId, winner.claim ?? claim);
1134
+ }
1135
+ // Only spent or unreadable generations sit above ours, so the node id
1136
+ // itself is free — this number just is not ours to hold. Re-scan and take
1137
+ // one above the highest file on disk instead.
1138
+ continue;
1139
+ }
1140
+ for (const superseded of generations) {
1141
+ if (!superseded.claim) {
1142
+ pruneSupersededGeneration(superseded, nodeId, env);
1143
+ continue;
1144
+ }
1145
+ // A generation whose holder is still alive is not garbage: it is the
1146
+ // only record guarding that broker's node id. A `--force` takeover that
1147
+ // pruned it here and then failed to start left the incumbent running and
1148
+ // unguarded — the next plain `node up` took the node id from under a
1149
+ // live, still-registered broker. Held generations stay until their
1150
+ // holder exits (a later acquisition prunes them as stale) or their
1151
+ // broker is stopped (releaseNodeClaimsForBroker retires them); our own
1152
+ // superseded generations are exempted from "held" the same way the
1153
+ // conflict check above exempts them, so a re-acquire still cleans up.
1154
+ const status = await classifyNodeClaim(nodeId, superseded.claim, env, deps, selfPids);
1155
+ if (status.state !== 'held') {
1156
+ pruneSupersededGeneration(superseded, nodeId, env);
1157
+ }
1158
+ }
1159
+ return claim;
1160
+ }
1161
+ throw new NodeClaimContentionError(nodeId, nodeClaimsDir(env));
1162
+ }
1163
+ function isSameAcquisition(current, claim) {
1164
+ if (!current)
1165
+ return false;
1166
+ if (claim.owner_token)
1167
+ return current.owner_token === claim.owner_token;
1168
+ // Claims written before this process (or by hand) carry no token; fall back
1169
+ // to the identity the record does have.
1170
+ return (current.node_id === claim.node_id && current.pid === claim.pid && current.state_dir === claim.state_dir);
1171
+ }
1172
+ /**
1173
+ * Record the pid of the child this reservation just handed its hold descriptor
1174
+ * to, so a later start can recognise that process no matter what it becomes.
1175
+ *
1176
+ * Deliberately synchronous, and called from the spawn itself rather than from
1177
+ * the adoption that follows it: the whole point is to have the pid on disk
1178
+ * before the child can `exec` into something the recorded executable no longer
1179
+ * describes, and before this supervisor can be killed. It is a plain write of
1180
+ * OUR OWN generation with no `await` between the read and the write, so it
1181
+ * cannot interleave with anything.
1182
+ *
1183
+ * A failure is never fatal. This is additional positive evidence layered on top
1184
+ * of a fence that is already in place — the descriptor was inherited at the
1185
+ * spawn — so losing it costs the launcher/exec case, not the guarantee. The
1186
+ * caller keeps the returned claim, which is unchanged when nothing was written.
1187
+ */
1188
+ export function recordSpawnedBrokerChild(reservation, pid, env = process.env) {
1189
+ if (!Number.isSafeInteger(pid) || pid <= 0)
1190
+ return reservation;
1191
+ const generation = reservation.generation ?? 1;
1192
+ const file = nodeClaimPath(reservation.node_id, env, generation);
1193
+ try {
1194
+ // A `--force` takeover that landed during the spawn owns the node id now.
1195
+ // Rewriting its record would be this module's one forbidden move; the
1196
+ // adoption below will fail this start on the same evidence.
1197
+ if (!isSameAcquisition(readClaimFile(file), reservation))
1198
+ return reservation;
1199
+ const claim = { ...reservation, broker_child_pid: pid };
1200
+ writeClaimRecordAtomically(file, claim);
1201
+ return claim;
1202
+ }
1203
+ catch {
1204
+ return reservation;
1205
+ }
1206
+ }
1207
+ /**
1208
+ * Hand a reservation to the verified broker process that now owns the state
1209
+ * dir.
1210
+ *
1211
+ * Only the acquisition that created a generation ever rewrites it, so this is a
1212
+ * conflict-free in-place update — but it is still conditional on that
1213
+ * generation still being the highest: a `--force` takeover that landed while
1214
+ * this broker was starting has already won the node id, and continuing would
1215
+ * put two brokers back on one delivery socket. Losing here fails startup, which
1216
+ * tears this broker down.
1217
+ *
1218
+ * @throws NodeClaimConflictError when the reservation no longer owns the node id.
1219
+ */
1220
+ export async function adoptNodeClaim(input) {
1221
+ const env = input.env ?? process.env;
1222
+ const deps = { ...input, env };
1223
+ const { reservation } = input;
1224
+ const generation = reservation.generation ?? 1;
1225
+ const file = nodeClaimPath(reservation.node_id, env, generation);
1226
+ const before = readClaimGenerations(reservation.node_id, env);
1227
+ if (highestGenerationNumber(before) > generation || !isSameAcquisition(readClaimFile(file), reservation)) {
1228
+ throw new NodeClaimConflictError(reservation.node_id, currentGeneration(before)?.claim ?? reservation);
1229
+ }
1230
+ const startedAt = await readProcessStartedAt(input.pid, deps);
1231
+ const claim = {
1232
+ ...reservation,
1233
+ pid: input.pid,
1234
+ ...(input.apiPort !== undefined ? { api_port: input.apiPort } : {}),
1235
+ ...(input.brokerName ? { broker_name: input.brokerName } : {}),
1236
+ ...(startedAt ? { process_started_at: startedAt } : {}),
1237
+ supervisor_pid: reservation.pid,
1238
+ ...(reservation.process_started_at ? { supervisor_started_at: reservation.process_started_at } : {}),
1239
+ status: 'active',
1240
+ };
1241
+ if (!startedAt)
1242
+ delete claim.process_started_at;
1243
+ // Atomic replace of OUR generation only: readers see the reservation or the
1244
+ // adopted record, never a partial write, and never another start's file.
1245
+ writeClaimRecordAtomically(file, claim);
1246
+ const after = readClaimGenerations(reservation.node_id, env);
1247
+ if (highestGenerationNumber(after) > generation) {
1248
+ // A takeover landed during the write. It owns the node id; give ours up
1249
+ // rather than leaving a second live-looking record behind.
1250
+ safeUnlinkClaim(file);
1251
+ safeUnlinkClaim(nodeClaimHoldPath(reservation.node_id, env, generation));
1252
+ throw new NodeClaimConflictError(reservation.node_id, currentGeneration(after)?.claim ?? claim);
1253
+ }
1254
+ return claim;
1255
+ }
1256
+ /**
1257
+ * Drop a claim this start owns, leaving its generation number spent.
1258
+ *
1259
+ * The record is REPLACED by a tombstone rather than removed. Removing it made
1260
+ * the number reusable, and a suspended start could then re-create a generation
1261
+ * that had since been handed to somebody else: it would pass the
1262
+ * higher-generation check (its own file was the highest again) and prune the
1263
+ * successor's live claim from its own stale scan, leaving two brokers alive on
1264
+ * one node id with only one of them recorded. A tombstone keeps
1265
+ * `max(generation)` from ever decreasing, so no number is issued twice; it
1266
+ * parses as no claim at all, so the node id reads free immediately; and the
1267
+ * next acquisition prunes it, so at most one spent file per node id is ever on
1268
+ * disk.
1269
+ *
1270
+ * Only the acquisition that created a generation ever rewrites it, proven by
1271
+ * `owner_token`, and the read and the write are adjacent syscalls with no await
1272
+ * in between.
1273
+ */
1274
+ export async function releaseNodeClaim(claim, env = process.env) {
1275
+ const generation = claim.generation ?? 1;
1276
+ const file = nodeClaimPath(claim.node_id, env, generation);
1277
+ if (!isSameAcquisition(readClaimFile(file), claim)) {
1278
+ return false;
1279
+ }
1280
+ try {
1281
+ writeClaimRecordAtomically(file, {
1282
+ version: 1,
1283
+ released: true,
1284
+ node_id: claim.node_id,
1285
+ generation,
1286
+ released_at: new Date().toISOString(),
1287
+ });
1288
+ }
1289
+ catch {
1290
+ return false;
1291
+ }
1292
+ // The hold descriptor protected this generation only; both processes it
1293
+ // fenced are gone by the time a release is allowed to run.
1294
+ safeUnlinkClaim(nodeClaimHoldPath(claim.node_id, env, generation));
1295
+ return true;
1296
+ }
1297
+ /**
1298
+ * Release the claims a broker pid held in a state dir. `node down` stops a
1299
+ * broker it did not start, so it cannot name the node id the claim was written
1300
+ * for — the pid and state dir it verified are what it has.
1301
+ *
1302
+ * A claim for that state dir whose every recorded pid is already dead is
1303
+ * released too: that is the orphan left by a supervisor that was killed before
1304
+ * it could record its broker's pid, and `down` has just proven the state dir's
1305
+ * broker is gone.
1306
+ */
1307
+ export async function releaseNodeClaimsForBroker(input) {
1308
+ const env = input.env ?? process.env;
1309
+ const deps = { ...input, env };
1310
+ const stateDir = normalizeClaimStateDir(input.stateDir);
1311
+ const released = [];
1312
+ for (const claim of listAllNodeClaims(env)) {
1313
+ if (claim.state_dir !== stateDir)
1314
+ continue;
1315
+ const namesThisBroker = claim.pid === input.pid || claim.supervisor_pid === input.pid;
1316
+ if (!namesThisBroker) {
1317
+ const status = await classifyNodeClaim(claim.node_id, claim, env, deps);
1318
+ if (status.state !== 'stale')
1319
+ continue;
1320
+ }
1321
+ if (await releaseNodeClaim(claim, env)) {
1322
+ released.push(claim);
1323
+ }
1324
+ }
1325
+ return released;
1326
+ }
1327
+ /** One-line description of the broker holding a claim, for operator output. */
1328
+ export function describeNodeClaimHolder(claim) {
1329
+ const parts = [`pid ${claim.pid}`, `state dir ${claim.state_dir}`];
1330
+ if (claim.api_port !== undefined)
1331
+ parts.push(`API port ${claim.api_port}`);
1332
+ if (claim.broker_name)
1333
+ parts.push(`broker name ${claim.broker_name}`);
1334
+ if (claim.status === 'reserved')
1335
+ parts.push('starting up');
1336
+ return parts.join(', ');
1337
+ }
1338
+ //# sourceMappingURL=node-claim.js.map