agent-relay 12.2.7 → 12.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -0
- package/dist/cli/commands/core.d.ts +13 -1
- package/dist/cli/commands/core.d.ts.map +1 -1
- package/dist/cli/commands/core.js +3 -1
- package/dist/cli/commands/core.js.map +1 -1
- package/dist/cli/commands/node.d.ts +6 -0
- package/dist/cli/commands/node.d.ts.map +1 -1
- package/dist/cli/commands/node.js +72 -0
- package/dist/cli/commands/node.js.map +1 -1
- package/dist/cli/lib/broker-lifecycle.d.ts +21 -1
- package/dist/cli/lib/broker-lifecycle.d.ts.map +1 -1
- package/dist/cli/lib/broker-lifecycle.js +354 -4
- package/dist/cli/lib/broker-lifecycle.js.map +1 -1
- package/dist/cli/lib/client-factory.d.ts +4 -0
- package/dist/cli/lib/client-factory.d.ts.map +1 -1
- package/dist/cli/lib/client-factory.js +3 -1
- package/dist/cli/lib/client-factory.js.map +1 -1
- package/dist/cli/lib/node-claim.d.ts +460 -0
- package/dist/cli/lib/node-claim.d.ts.map +1 -0
- package/dist/cli/lib/node-claim.js +1338 -0
- package/dist/cli/lib/node-claim.js.map +1 -0
- package/dist/index.cjs +1 -1
- package/package.json +10 -10
|
@@ -0,0 +1,1338 @@
|
|
|
1
|
+
import crypto from 'node:crypto';
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import os from 'node:os';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
/**
|
|
6
|
+
* The machine-global Relay home that holds `fleet-enrollments.json`.
|
|
7
|
+
*
|
|
8
|
+
* Deliberately not imported from `@agent-relay/cloud`: that barrel pulls the
|
|
9
|
+
* Cloud SSH runtime, and a claim check must stay a couple of syscalls that any
|
|
10
|
+
* command can make. `node-claim.test.ts` pins this to the enrollment store's
|
|
11
|
+
* own directory so the two cannot drift apart.
|
|
12
|
+
*/
|
|
13
|
+
function relayHome(env) {
|
|
14
|
+
return env.AGENT_RELAY_HOME ?? path.join(os.homedir(), '.agentworkforce/relay');
|
|
15
|
+
}
|
|
16
|
+
/** Filename the broker writes its API endpoint and pid into, inside its state dir. */
|
|
17
|
+
const BROKER_CONNECTION_FILENAME = 'connection.json';
|
|
18
|
+
/** Thrown when a live local broker already serves the requested node id. */
|
|
19
|
+
export class NodeClaimConflictError extends Error {
|
|
20
|
+
nodeId;
|
|
21
|
+
claim;
|
|
22
|
+
constructor(nodeId, claim) {
|
|
23
|
+
super(`node ${nodeId} is already served by a live local broker (pid ${claim.pid}, state dir ${claim.state_dir}). ` +
|
|
24
|
+
'Stop it with `agent-relay node down --state-dir <dir>`, serve a different enrolled node, or re-run with --force.');
|
|
25
|
+
this.nodeId = nodeId;
|
|
26
|
+
this.claim = claim;
|
|
27
|
+
this.name = 'NodeClaimConflictError';
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Thrown when the kernel-level ownership fence could not be established.
|
|
32
|
+
*
|
|
33
|
+
* The hold descriptor is what makes a claim survive its own supervisor, so a
|
|
34
|
+
* start that cannot open one cannot honour the guarantee it is claiming under.
|
|
35
|
+
* It refuses instead of spawning an unfenced broker.
|
|
36
|
+
*/
|
|
37
|
+
export class NodeClaimHoldError extends Error {
|
|
38
|
+
nodeId;
|
|
39
|
+
holdPath;
|
|
40
|
+
constructor(nodeId, holdPath, cause) {
|
|
41
|
+
super(`could not establish the ownership fence for node ${nodeId} at ${holdPath}: ` +
|
|
42
|
+
`${cause instanceof Error ? cause.message : String(cause)}. ` +
|
|
43
|
+
'Startup was stopped rather than run a broker that another start could silently take the node from. ' +
|
|
44
|
+
'Ensure the claims directory is writable and has free space, then retry.', { cause });
|
|
45
|
+
this.nodeId = nodeId;
|
|
46
|
+
this.holdPath = holdPath;
|
|
47
|
+
this.name = 'NodeClaimHoldError';
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Thrown when ownership could not be established because other starts kept
|
|
52
|
+
* winning the exclusive create.
|
|
53
|
+
*
|
|
54
|
+
* Each attempt is a handful of syscalls, so exhausting them means a pathological
|
|
55
|
+
* amount of contention on one node id. Ownership cannot be established in that
|
|
56
|
+
* state, and starting anyway is exactly the double registration this module
|
|
57
|
+
* exists to prevent.
|
|
58
|
+
*/
|
|
59
|
+
export class NodeClaimContentionError extends Error {
|
|
60
|
+
nodeId;
|
|
61
|
+
claimsDir;
|
|
62
|
+
constructor(nodeId, claimsDir, cause) {
|
|
63
|
+
super(`could not take ownership of node ${nodeId}: other agent-relay starts kept winning the claim in ${claimsDir}. ` +
|
|
64
|
+
'Retry, or run `agent-relay node down` on the state dir that should keep the node.', { cause });
|
|
65
|
+
this.nodeId = nodeId;
|
|
66
|
+
this.claimsDir = claimsDir;
|
|
67
|
+
this.name = 'NodeClaimContentionError';
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* The enrolled node id a start will register as, or `undefined` when it will
|
|
72
|
+
* mint its own identity.
|
|
73
|
+
*
|
|
74
|
+
* `RELAY_NODE_ID` is sent verbatim in `node.register` (`resolve_broker_node_id`,
|
|
75
|
+
* crates/broker/src/runtime/init.rs), so an explicit node id IS the identity
|
|
76
|
+
* that can evict another broker, and it is claimed on that basis alone.
|
|
77
|
+
*
|
|
78
|
+
* This deliberately does not try to predict whether the broker will manage to
|
|
79
|
+
* authenticate as it. An earlier version claimed the id only when a token was
|
|
80
|
+
* in the environment or already in the broker's on-disk cache, and that was a
|
|
81
|
+
* hole: `init.rs` also wires a workspace-key token minter, and
|
|
82
|
+
* `node_control.rs` mints and connects with no cached token at all, so a start
|
|
83
|
+
* carrying nothing but `RELAY_NODE_ID` and workspace credentials walked past a
|
|
84
|
+
* live claim and took the node's delivery socket — the incident this module
|
|
85
|
+
* exists to close. Enumerating the credential routes instead of the identity
|
|
86
|
+
* just moves the hole to the next route that gets added.
|
|
87
|
+
*
|
|
88
|
+
* The cost is a start that could not have registered at all (no token, no
|
|
89
|
+
* workspace key, nothing to mint with) being refused when some other broker
|
|
90
|
+
* genuinely holds the id. That is recoverable in one flag (`--force`) and the
|
|
91
|
+
* start it refuses was headed for local-only operation anyway, while the
|
|
92
|
+
* opposite error is a silent delivery outage. `--local-only` starts register
|
|
93
|
+
* nothing and are excluded by the callers, not here.
|
|
94
|
+
*/
|
|
95
|
+
export function enrolledNodeIdForClaim(env) {
|
|
96
|
+
return env.RELAY_NODE_ID?.trim() || undefined;
|
|
97
|
+
}
|
|
98
|
+
/* ------------------------------------------------------------------ *
|
|
99
|
+
* Claim files and generations
|
|
100
|
+
* ------------------------------------------------------------------ */
|
|
101
|
+
/** Zero padding, so claim generations sort lexically as well as numerically. */
|
|
102
|
+
const GENERATION_DIGITS = 6;
|
|
103
|
+
/** Matches `<stem>.<generation>.json` for any node. */
|
|
104
|
+
const CLAIM_FILENAME_PATTERN = new RegExp(`^(.+)\\.(\\d{${GENERATION_DIGITS},})\\.json$`);
|
|
105
|
+
/** Directory holding the claim files for every node id served on this machine. */
|
|
106
|
+
export function nodeClaimsDir(env = process.env) {
|
|
107
|
+
return path.join(relayHome(env), 'node-claims');
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Filename stem for a node id's claims. Node ids are engine-issued
|
|
111
|
+
* (`node_<digits>`), but the stem is sanitized anyway so a hand-edited
|
|
112
|
+
* enrollment store cannot write outside the claims directory. Readers verify
|
|
113
|
+
* `node_id` from the file contents, so a sanitized collision is reported as a
|
|
114
|
+
* conflict rather than silently overwriting another node's claim.
|
|
115
|
+
*/
|
|
116
|
+
function nodeClaimStem(nodeId) {
|
|
117
|
+
return (nodeId
|
|
118
|
+
.trim()
|
|
119
|
+
.replace(/[^\w.-]/g, '-')
|
|
120
|
+
.slice(0, 96) || 'unnamed');
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Path of one generation of a node id's claim.
|
|
124
|
+
*
|
|
125
|
+
* Every write this module makes is the exclusive creation of a NEW generation,
|
|
126
|
+
* never an overwrite of an existing one — that is what makes takeover safe
|
|
127
|
+
* without a lock file (see {@link acquireNodeClaim}).
|
|
128
|
+
*/
|
|
129
|
+
export function nodeClaimPath(nodeId, env = process.env, generation = 1) {
|
|
130
|
+
const suffix = String(generation).padStart(GENERATION_DIGITS, '0');
|
|
131
|
+
return path.join(nodeClaimsDir(env), `${nodeClaimStem(nodeId)}.${suffix}.json`);
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Sibling of a claim generation whose OPEN DESCRIPTORS are the ownership
|
|
135
|
+
* evidence, held by the kernel rather than written by a process.
|
|
136
|
+
*
|
|
137
|
+
* `<stem>.<generation>.hold` is created and opened by the supervising CLI
|
|
138
|
+
* BEFORE it spawns a broker, and the descriptor is inherited by the child
|
|
139
|
+
* across `fork`. From the instant a broker child exists — long before it binds
|
|
140
|
+
* its API, writes `connection.json` or queues `node.register` — some live
|
|
141
|
+
* process holds this file open, and the kernel drops the last reference the
|
|
142
|
+
* moment both of them die. That is what closes the window a supervisor
|
|
143
|
+
* SIGKILLed between the spawn and the broker's first write used to leave: there
|
|
144
|
+
* is no interval in which a competing start can see "dead supervisor, nothing
|
|
145
|
+
* published" and conclude the node id is free while the orphan is on its way to
|
|
146
|
+
* registering.
|
|
147
|
+
*
|
|
148
|
+
* Not matched by {@link CLAIM_FILENAME_PATTERN} (which requires `.json`), so it
|
|
149
|
+
* never counts as a generation of its own.
|
|
150
|
+
*/
|
|
151
|
+
export function nodeClaimHoldPath(nodeId, env = process.env, generation = 1) {
|
|
152
|
+
return `${nodeClaimPath(nodeId, env, generation).slice(0, -'.json'.length)}.hold`;
|
|
153
|
+
}
|
|
154
|
+
/** Single-quote a path for a shell command (`AGENT_RELAY_HOME` is operator input). */
|
|
155
|
+
function shellQuote(value) {
|
|
156
|
+
// Close the quoted run, emit an escaped quote, reopen: ' -> '\''
|
|
157
|
+
return `'${value.split("'").join("'\\''")}'`;
|
|
158
|
+
}
|
|
159
|
+
/** Resolve a state dir to its canonical path so two spellings compare equal. */
|
|
160
|
+
export function normalizeClaimStateDir(stateDir) {
|
|
161
|
+
try {
|
|
162
|
+
return fs.realpathSync(stateDir);
|
|
163
|
+
}
|
|
164
|
+
catch {
|
|
165
|
+
return path.resolve(stateDir);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
function isOptionalString(value) {
|
|
169
|
+
return value === undefined || typeof value === 'string';
|
|
170
|
+
}
|
|
171
|
+
function isOptionalPositiveInteger(value) {
|
|
172
|
+
return value === undefined || (Number.isSafeInteger(value) && value > 0);
|
|
173
|
+
}
|
|
174
|
+
function hasValidOptionalClaimFields(record) {
|
|
175
|
+
return ((record.api_port === undefined || Number.isSafeInteger(record.api_port)) &&
|
|
176
|
+
isOptionalString(record.broker_name) &&
|
|
177
|
+
isOptionalString(record.broker_binary) &&
|
|
178
|
+
isOptionalString(record.broker_executable) &&
|
|
179
|
+
isOptionalString(record.process_started_at) &&
|
|
180
|
+
isOptionalString(record.supervisor_started_at) &&
|
|
181
|
+
isOptionalString(record.owner_token) &&
|
|
182
|
+
isOptionalPositiveInteger(record.supervisor_pid) &&
|
|
183
|
+
isOptionalPositiveInteger(record.broker_child_pid) &&
|
|
184
|
+
isOptionalPositiveInteger(record.generation) &&
|
|
185
|
+
(record.status === undefined || record.status === 'reserved' || record.status === 'active'));
|
|
186
|
+
}
|
|
187
|
+
function isNodeClaim(value) {
|
|
188
|
+
if (typeof value !== 'object' || value === null)
|
|
189
|
+
return false;
|
|
190
|
+
const record = value;
|
|
191
|
+
// A tombstone occupies a generation number so it can never be reissued, but
|
|
192
|
+
// it is evidence of nothing. Rejecting it here is what makes it read as an
|
|
193
|
+
// unclaimed node id everywhere a claim is read.
|
|
194
|
+
if (record.released === true)
|
|
195
|
+
return false;
|
|
196
|
+
return (record.version === 1 &&
|
|
197
|
+
typeof record.node_id === 'string' &&
|
|
198
|
+
record.node_id.trim().length > 0 &&
|
|
199
|
+
Number.isSafeInteger(record.pid) &&
|
|
200
|
+
record.pid > 0 &&
|
|
201
|
+
typeof record.state_dir === 'string' &&
|
|
202
|
+
record.state_dir.length > 0 &&
|
|
203
|
+
typeof record.claimed_at === 'string' &&
|
|
204
|
+
hasValidOptionalClaimFields(record));
|
|
205
|
+
}
|
|
206
|
+
/** Exact bytes of a claim file, or `null` when it cannot be read at all. */
|
|
207
|
+
function readClaimBytes(file) {
|
|
208
|
+
try {
|
|
209
|
+
return fs.readFileSync(file, 'utf-8');
|
|
210
|
+
}
|
|
211
|
+
catch {
|
|
212
|
+
return null;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
function parseClaim(raw) {
|
|
216
|
+
if (raw === null)
|
|
217
|
+
return null;
|
|
218
|
+
try {
|
|
219
|
+
const parsed = JSON.parse(raw);
|
|
220
|
+
return isNodeClaim(parsed) ? parsed : null;
|
|
221
|
+
}
|
|
222
|
+
catch {
|
|
223
|
+
return null;
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
function readClaimFile(file) {
|
|
227
|
+
return parseClaim(readClaimBytes(file));
|
|
228
|
+
}
|
|
229
|
+
/**
|
|
230
|
+
* Every generation file present for `nodeId`, ascending.
|
|
231
|
+
*
|
|
232
|
+
* Unreadable files are kept in the list with a `null` claim: they must still
|
|
233
|
+
* raise the next generation number (or an exclusive create would collide with
|
|
234
|
+
* them forever) even though they are no evidence of a live broker.
|
|
235
|
+
*/
|
|
236
|
+
function readClaimGenerations(nodeId, env) {
|
|
237
|
+
const dir = nodeClaimsDir(env);
|
|
238
|
+
const stem = nodeClaimStem(nodeId);
|
|
239
|
+
let filenames;
|
|
240
|
+
try {
|
|
241
|
+
filenames = fs.readdirSync(dir);
|
|
242
|
+
}
|
|
243
|
+
catch {
|
|
244
|
+
return [];
|
|
245
|
+
}
|
|
246
|
+
const generations = [];
|
|
247
|
+
for (const filename of filenames) {
|
|
248
|
+
const match = CLAIM_FILENAME_PATTERN.exec(filename);
|
|
249
|
+
if (!match || match[1] !== stem)
|
|
250
|
+
continue;
|
|
251
|
+
const file = path.join(dir, filename);
|
|
252
|
+
const raw = readClaimBytes(file);
|
|
253
|
+
generations.push({ generation: Number.parseInt(match[2], 10), file, raw, claim: parseClaim(raw) });
|
|
254
|
+
}
|
|
255
|
+
return generations.sort((left, right) => left.generation - right.generation);
|
|
256
|
+
}
|
|
257
|
+
/** The generation that currently owns the node id: the highest readable one. */
|
|
258
|
+
function currentGeneration(generations) {
|
|
259
|
+
for (let index = generations.length - 1; index >= 0; index -= 1) {
|
|
260
|
+
if (generations[index].claim)
|
|
261
|
+
return generations[index];
|
|
262
|
+
}
|
|
263
|
+
return undefined;
|
|
264
|
+
}
|
|
265
|
+
function highestGenerationNumber(generations) {
|
|
266
|
+
return generations.length === 0 ? 0 : generations[generations.length - 1].generation;
|
|
267
|
+
}
|
|
268
|
+
/**
|
|
269
|
+
* The claim that currently owns `nodeId`, or `null`.
|
|
270
|
+
*
|
|
271
|
+
* A missing, unreadable, or malformed file reads as `null`: an unparseable
|
|
272
|
+
* claim is no evidence of a live broker, and treating it as one would brick
|
|
273
|
+
* every later `node up` for that node id.
|
|
274
|
+
*/
|
|
275
|
+
export function readNodeClaim(nodeId, env = process.env) {
|
|
276
|
+
return currentGeneration(readClaimGenerations(nodeId, env))?.claim ?? null;
|
|
277
|
+
}
|
|
278
|
+
/** The current claim for every node id on this machine, newest first. */
|
|
279
|
+
export function listNodeClaims(env = process.env) {
|
|
280
|
+
const dir = nodeClaimsDir(env);
|
|
281
|
+
let filenames;
|
|
282
|
+
try {
|
|
283
|
+
filenames = fs.readdirSync(dir);
|
|
284
|
+
}
|
|
285
|
+
catch {
|
|
286
|
+
return [];
|
|
287
|
+
}
|
|
288
|
+
const newestByStem = new Map();
|
|
289
|
+
for (const filename of filenames) {
|
|
290
|
+
const match = CLAIM_FILENAME_PATTERN.exec(filename);
|
|
291
|
+
if (!match)
|
|
292
|
+
continue;
|
|
293
|
+
const claim = readClaimFile(path.join(dir, filename));
|
|
294
|
+
if (!claim)
|
|
295
|
+
continue;
|
|
296
|
+
const generation = Number.parseInt(match[2], 10);
|
|
297
|
+
const seen = newestByStem.get(match[1]);
|
|
298
|
+
if (!seen || seen.generation < generation) {
|
|
299
|
+
newestByStem.set(match[1], { generation, claim });
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
return [...newestByStem.values()]
|
|
303
|
+
.map((entry) => entry.claim)
|
|
304
|
+
.sort((left, right) => right.claimed_at.localeCompare(left.claimed_at));
|
|
305
|
+
}
|
|
306
|
+
/**
|
|
307
|
+
* Every readable claim on this machine, across ALL generations, newest first.
|
|
308
|
+
*
|
|
309
|
+
* `listNodeClaims` reports only the current generation per node id, but a
|
|
310
|
+
* `--force` takeover can leave a still-held incumbent beneath the
|
|
311
|
+
* replacement's file. Callers that diagnose or release by broker identity must
|
|
312
|
+
* see every generation, or that hidden claim never surfaces.
|
|
313
|
+
*/
|
|
314
|
+
function listAllNodeClaims(env) {
|
|
315
|
+
const dir = nodeClaimsDir(env);
|
|
316
|
+
let filenames;
|
|
317
|
+
try {
|
|
318
|
+
filenames = fs.readdirSync(dir);
|
|
319
|
+
}
|
|
320
|
+
catch {
|
|
321
|
+
return [];
|
|
322
|
+
}
|
|
323
|
+
const claims = [];
|
|
324
|
+
for (const filename of filenames) {
|
|
325
|
+
const match = CLAIM_FILENAME_PATTERN.exec(filename);
|
|
326
|
+
if (!match)
|
|
327
|
+
continue;
|
|
328
|
+
const claim = readClaimFile(path.join(dir, filename));
|
|
329
|
+
if (claim)
|
|
330
|
+
claims.push(claim);
|
|
331
|
+
}
|
|
332
|
+
return claims.sort((left, right) => right.claimed_at.localeCompare(left.claimed_at) || (right.generation ?? 0) - (left.generation ?? 0));
|
|
333
|
+
}
|
|
334
|
+
/* ------------------------------------------------------------------ *
|
|
335
|
+
* Liveness
|
|
336
|
+
* ------------------------------------------------------------------ */
|
|
337
|
+
function isProcessAlive(pid, deps) {
|
|
338
|
+
const kill = deps.killProcess ??
|
|
339
|
+
((target, signal) => {
|
|
340
|
+
process.kill(target, signal);
|
|
341
|
+
});
|
|
342
|
+
try {
|
|
343
|
+
kill(pid, 0);
|
|
344
|
+
return true;
|
|
345
|
+
}
|
|
346
|
+
catch (error) {
|
|
347
|
+
// Permission denial proves the process exists and is not ours to inspect;
|
|
348
|
+
// that must read as live, never as a free node id.
|
|
349
|
+
return error?.code === 'EPERM';
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
/**
|
|
353
|
+
* `ps` birth time for a pid, or `null` when it cannot be read. Mirrors the
|
|
354
|
+
* command `broker-process-identity` uses so both agree on the format.
|
|
355
|
+
*/
|
|
356
|
+
async function readProcessStartedAt(pid, deps) {
|
|
357
|
+
const execCommand = deps.execCommand;
|
|
358
|
+
if (!execCommand) {
|
|
359
|
+
return null;
|
|
360
|
+
}
|
|
361
|
+
try {
|
|
362
|
+
const { stdout } = await execCommand(`LC_ALL=C TZ=UTC ps -p ${pid} -o lstart=`);
|
|
363
|
+
const normalized = stdout.trim().replace(/\s+/g, ' ');
|
|
364
|
+
return normalized.length > 0 ? normalized : null;
|
|
365
|
+
}
|
|
366
|
+
catch {
|
|
367
|
+
return null;
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
/**
|
|
371
|
+
* Whether `pid` is still the process that was recorded with `startedAt`.
|
|
372
|
+
* A pid that is gone, or one whose birth time moved (the number was recycled),
|
|
373
|
+
* no longer holds anything.
|
|
374
|
+
*/
|
|
375
|
+
async function isRecordedProcessAlive(pid, startedAt, deps) {
|
|
376
|
+
if (!isProcessAlive(pid, deps)) {
|
|
377
|
+
return { alive: false, reason: `pid ${pid} is no longer running` };
|
|
378
|
+
}
|
|
379
|
+
if (startedAt) {
|
|
380
|
+
const current = await readProcessStartedAt(pid, deps);
|
|
381
|
+
if (current && current !== startedAt) {
|
|
382
|
+
return { alive: false, reason: `pid ${pid} was recycled by a process started ${current}` };
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
return { alive: true, reason: `pid ${pid} is running` };
|
|
386
|
+
}
|
|
387
|
+
/** The owning pid plus, when the claim records one, its supervising CLI. */
|
|
388
|
+
function claimProcesses(claim) {
|
|
389
|
+
const processes = [
|
|
390
|
+
{
|
|
391
|
+
pid: claim.pid,
|
|
392
|
+
startedAt: claim.process_started_at,
|
|
393
|
+
role: claim.status === 'reserved' ? 'supervisor' : 'broker',
|
|
394
|
+
},
|
|
395
|
+
];
|
|
396
|
+
if (claim.supervisor_pid !== undefined && claim.supervisor_pid !== claim.pid) {
|
|
397
|
+
processes.push({
|
|
398
|
+
pid: claim.supervisor_pid,
|
|
399
|
+
startedAt: claim.supervisor_started_at,
|
|
400
|
+
role: 'supervisor',
|
|
401
|
+
});
|
|
402
|
+
}
|
|
403
|
+
return processes;
|
|
404
|
+
}
|
|
405
|
+
/** Pid recorded in `<state dir>/connection.json`, which the broker writes itself. */
|
|
406
|
+
function readStateDirBrokerPid(stateDir) {
|
|
407
|
+
try {
|
|
408
|
+
const parsed = JSON.parse(fs.readFileSync(path.join(stateDir, BROKER_CONNECTION_FILENAME), 'utf-8'));
|
|
409
|
+
if (typeof parsed !== 'object' || parsed === null)
|
|
410
|
+
return null;
|
|
411
|
+
const pid = parsed.pid;
|
|
412
|
+
return Number.isSafeInteger(pid) && pid > 0 ? pid : null;
|
|
413
|
+
}
|
|
414
|
+
catch {
|
|
415
|
+
return null;
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
function brokerExecutableHint(claim) {
|
|
419
|
+
return {
|
|
420
|
+
stateDir: claim.state_dir,
|
|
421
|
+
...(claim.broker_binary ? { binary: claim.broker_binary } : {}),
|
|
422
|
+
...(claim.broker_executable ? { object: claim.broker_executable } : {}),
|
|
423
|
+
...(claim.broker_child_pid ? { childPid: claim.broker_child_pid } : {}),
|
|
424
|
+
};
|
|
425
|
+
}
|
|
426
|
+
/**
|
|
427
|
+
* Identity of the executable file a claim's start was going to run, for the
|
|
428
|
+
* record. `realpathSync` first so a claim written through a symlinked install
|
|
429
|
+
* path still names the file a running broker maps.
|
|
430
|
+
*/
|
|
431
|
+
/**
|
|
432
|
+
* `<device>:<inode>` of a file on this machine, in the encoding `lsof -d txt`
|
|
433
|
+
* reports and the claim records, or `null` when it cannot be stat'ed.
|
|
434
|
+
*/
|
|
435
|
+
function executableObjectOf(file) {
|
|
436
|
+
try {
|
|
437
|
+
const stats = fs.statSync(file, { bigint: true });
|
|
438
|
+
return `0x${stats.dev.toString(16)}:${stats.ino.toString()}`;
|
|
439
|
+
}
|
|
440
|
+
catch {
|
|
441
|
+
return null;
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
function describeBrokerExecutable(binary) {
|
|
445
|
+
if (!binary)
|
|
446
|
+
return {};
|
|
447
|
+
try {
|
|
448
|
+
const resolved = fs.realpathSync(binary);
|
|
449
|
+
const object = executableObjectOf(resolved);
|
|
450
|
+
return {
|
|
451
|
+
broker_binary: resolved,
|
|
452
|
+
...(object ? { broker_executable: object } : {}),
|
|
453
|
+
};
|
|
454
|
+
}
|
|
455
|
+
catch {
|
|
456
|
+
// The path is still worth recording: a process running it is recognisable
|
|
457
|
+
// by argv even when the file cannot be stat'ed from here.
|
|
458
|
+
return path.isAbsolute(binary) ? { broker_binary: binary } : {};
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
/**
|
|
462
|
+
* The executable objects `pid` is running, as `<device>:<inode>` pairs, or
|
|
463
|
+
* `null` when they cannot be read.
|
|
464
|
+
*
|
|
465
|
+
* `txt` mappings identify an executable by device and inode on both macOS and
|
|
466
|
+
* Linux, which is the only handle on "what is this process running" that does
|
|
467
|
+
* not go through a filename. Same command and same field encoding as
|
|
468
|
+
* `readBrokerProcessIdentity` in `broker-process-identity.ts`, so the two agree
|
|
469
|
+
* on what an executable's identity is.
|
|
470
|
+
*/
|
|
471
|
+
async function readExecutableObjects(pid, deps) {
|
|
472
|
+
const execCommand = deps.execCommand;
|
|
473
|
+
if (!execCommand)
|
|
474
|
+
return null;
|
|
475
|
+
let stdout;
|
|
476
|
+
try {
|
|
477
|
+
({ stdout } = await execCommand(`LC_ALL=C lsof -nP -a -p ${pid} -d txt -FfDi`));
|
|
478
|
+
}
|
|
479
|
+
catch {
|
|
480
|
+
return null;
|
|
481
|
+
}
|
|
482
|
+
const fields = stdout.trim().split('\n');
|
|
483
|
+
if (fields.shift() !== `p${pid}`)
|
|
484
|
+
return null;
|
|
485
|
+
const objects = new Set();
|
|
486
|
+
for (let index = 0; index < fields.length; index += 3) {
|
|
487
|
+
const [descriptor, device, inode] = fields.slice(index, index + 3);
|
|
488
|
+
if (descriptor !== 'ftxt' || !/^D0x[0-9a-f]+$/i.test(device ?? '') || !/^i[1-9]\d*$/.test(inode ?? '')) {
|
|
489
|
+
return null;
|
|
490
|
+
}
|
|
491
|
+
objects.add(`${device.slice(1).toLowerCase()}:${inode.slice(1)}`);
|
|
492
|
+
}
|
|
493
|
+
return objects.size > 0 ? objects : null;
|
|
494
|
+
}
|
|
495
|
+
/**
|
|
496
|
+
* Whether a command line names the executable the claim recorded, in a
|
|
497
|
+
* position an interpreter would put it.
|
|
498
|
+
*
|
|
499
|
+
* A shebang launcher does not RUN the file the claim recorded — the kernel runs
|
|
500
|
+
* the interpreter from its `#!` line and passes the script as an argument, so
|
|
501
|
+
* `lsof -d txt` reports `/bin/sh` and the recorded device/inode matches
|
|
502
|
+
* nothing. The script path is still right there in argv, and the file it names
|
|
503
|
+
* has exactly the identity the claim recorded. Comparing that file rather than
|
|
504
|
+
* the string is what keeps this working through a symlinked install path, the
|
|
505
|
+
* same reason the claim records an inode instead of a name.
|
|
506
|
+
*
|
|
507
|
+
* Only the first two arguments are considered: `argv[0]` is the program and
|
|
508
|
+
* `argv[1]` is where an interpreter puts its script. Anything further along is
|
|
509
|
+
* a broker's own argument and proves nothing about what it is running.
|
|
510
|
+
*/
|
|
511
|
+
function runsRecordedExecutable(args, hint) {
|
|
512
|
+
if (!hint.binary && !hint.object)
|
|
513
|
+
return false;
|
|
514
|
+
for (const token of args.split(/\s+/, 2)) {
|
|
515
|
+
if (!token.startsWith('/'))
|
|
516
|
+
continue;
|
|
517
|
+
if (hint.binary && token === hint.binary)
|
|
518
|
+
return true;
|
|
519
|
+
if (!hint.object)
|
|
520
|
+
continue;
|
|
521
|
+
const object = executableObjectOf(token);
|
|
522
|
+
if (object !== null && object === hint.object)
|
|
523
|
+
return true;
|
|
524
|
+
}
|
|
525
|
+
return false;
|
|
526
|
+
}
|
|
527
|
+
/**
|
|
528
|
+
* Everything a command line can say IN FAVOUR of a process being this claim's
|
|
529
|
+
* broker. Nothing here is ever read the other way round: a command line that
|
|
530
|
+
* matches none of it is not evidence of anything, because the broker binary is
|
|
531
|
+
* operator-selectable and a start on the default state dir passes no
|
|
532
|
+
* `--state-dir` for argv to carry.
|
|
533
|
+
*/
|
|
534
|
+
function argvIdentifiesBroker(args, hint) {
|
|
535
|
+
if (hint.binary && (args === hint.binary || args.startsWith(`${hint.binary} `)))
|
|
536
|
+
return true;
|
|
537
|
+
if (runsRecordedExecutable(args, hint))
|
|
538
|
+
return true;
|
|
539
|
+
return args.includes('agent-relay') || args.includes('relay-broker') || args.includes(hint.stateDir);
|
|
540
|
+
}
|
|
541
|
+
/**
|
|
542
|
+
* Whether `pid` maps the executable object the claim recorded.
|
|
543
|
+
*
|
|
544
|
+
* Mappings that cannot be read are NOT a mismatch — the process may simply not
|
|
545
|
+
* be ours to inspect — so they answer yes, like every other unanswerable probe
|
|
546
|
+
* in this module.
|
|
547
|
+
*/
|
|
548
|
+
async function mapsRecordedExecutable(pid, object, deps) {
|
|
549
|
+
const objects = await readExecutableObjects(pid, deps);
|
|
550
|
+
return objects === null || objects.has(object);
|
|
551
|
+
}
|
|
552
|
+
/**
|
|
553
|
+
* Whether `pid` is the broker a claim names, an unrelated process that merely
|
|
554
|
+
* inherited its descriptor or its pid number, or something this claim has no
|
|
555
|
+
* way to tell apart.
|
|
556
|
+
*
|
|
557
|
+
* Executable identity decides it: the device and inode of the file the process
|
|
558
|
+
* is running, against the broker executable the claim recorded. Command lines
|
|
559
|
+
* are consulted only as further POSITIVE evidence and never to rule a process
|
|
560
|
+
* out. `AGENT_RELAY_BIN` / `BROKER_BINARY_PATH` let a supported deployment run
|
|
561
|
+
* the broker under any filename, and with the default state dir such a broker
|
|
562
|
+
* carries neither `agent-relay` nor its state dir in argv — so the name test
|
|
563
|
+
* that used to decide this classified a real, live broker as unrelated and
|
|
564
|
+
* handed its node id to the next start.
|
|
565
|
+
*
|
|
566
|
+
* Those same overrides accept a LAUNCHER — a shell script that sets something
|
|
567
|
+
* up and `exec`s the broker is the ordinary shape of a custom install — and a
|
|
568
|
+
* launcher is not the file the running process maps. Before its `exec` the
|
|
569
|
+
* process runs the interpreter from the script's `#!` line; after it, the
|
|
570
|
+
* binary the script chose. Neither is the file recorded before the spawn, so
|
|
571
|
+
* executable identity alone ruled a live, fenced startup launcher out and
|
|
572
|
+
* reopened exactly the eviction this claim exists to prevent. Two further
|
|
573
|
+
* pieces of positive evidence close that: the recorded file appearing in argv
|
|
574
|
+
* where an interpreter puts its script ({@link runsRecordedExecutable}), which
|
|
575
|
+
* needs nothing recorded after the spawn, and the pid of the child this start
|
|
576
|
+
* spawned ({@link NodeClaim.broker_child_pid}), which `execve` preserves
|
|
577
|
+
* whatever the launcher turns into.
|
|
578
|
+
*
|
|
579
|
+
* Anything that cannot be consulted — no runner, an `lsof` or `ps` that fails —
|
|
580
|
+
* reads as `broker`: this only ever decides whether to KEEP guarding a node id,
|
|
581
|
+
* and a spurious refusal is recoverable (`--force`, `node down`) while a wrong
|
|
582
|
+
* "free" verdict is the silent delivery outage.
|
|
583
|
+
*/
|
|
584
|
+
async function classifyHolderProcess(pid, hint, deps) {
|
|
585
|
+
// Recorded at the spawn itself, so it is the one piece of evidence that is
|
|
586
|
+
// immune to whatever the child turns into afterwards.
|
|
587
|
+
if (hint.childPid !== undefined && pid === hint.childPid)
|
|
588
|
+
return 'broker';
|
|
589
|
+
const execCommand = deps.execCommand;
|
|
590
|
+
if (!execCommand)
|
|
591
|
+
return 'broker';
|
|
592
|
+
if (hint.object && (await mapsRecordedExecutable(pid, hint.object, deps)))
|
|
593
|
+
return 'broker';
|
|
594
|
+
let args;
|
|
595
|
+
try {
|
|
596
|
+
const { stdout } = await execCommand(`LC_ALL=C ps -p ${pid} -o args=`);
|
|
597
|
+
args = stdout.trim();
|
|
598
|
+
}
|
|
599
|
+
catch {
|
|
600
|
+
return 'broker';
|
|
601
|
+
}
|
|
602
|
+
// `ps` answered with nothing: the pid is gone, so it holds nothing.
|
|
603
|
+
if (!args)
|
|
604
|
+
return 'unrelated';
|
|
605
|
+
if (argvIdentifiesBroker(args, hint))
|
|
606
|
+
return 'broker';
|
|
607
|
+
return hint.object || hint.binary ? 'unrelated' : 'unidentified';
|
|
608
|
+
}
|
|
609
|
+
/**
|
|
610
|
+
* Whether any live process still holds this generation's hold file open.
|
|
611
|
+
*
|
|
612
|
+
* `lsof -t` answers from the kernel's open-file table, so this reports the
|
|
613
|
+
* broker child a SIGKILLed supervisor left behind from the instant that child
|
|
614
|
+
* exists — there is no publish step to wait for and nothing to go stale. The
|
|
615
|
+
* exit status is read from an explicit marker rather than from the runner's
|
|
616
|
+
* error shape: `lsof` exits 1 with no output when a file simply has no holders,
|
|
617
|
+
* and that has to be told apart from an `lsof` that could not run at all.
|
|
618
|
+
*
|
|
619
|
+
* An `lsof` that cannot be consulted reads as HELD, matching the rest of this
|
|
620
|
+
* module: `node up` already refuses to start without a usable `lsof` (it is how
|
|
621
|
+
* broker process identity is verified), and a spurious refusal is recoverable
|
|
622
|
+
* with `--force` or `node down` while a wrong "free" verdict is the silent
|
|
623
|
+
* delivery outage the claim exists to prevent.
|
|
624
|
+
*
|
|
625
|
+
* Holders are classified by {@link classifyHolderProcess}, exactly as the
|
|
626
|
+
* connection file's pid is. An inherited descriptor is not close-on-exec, so
|
|
627
|
+
* anything the broker itself spawns inherits it too; a harness left running by
|
|
628
|
+
* a SIGKILLed broker would otherwise pin the node id with no broker anywhere
|
|
629
|
+
* near it. Dropping one is a positive determination and needs BOTH halves of
|
|
630
|
+
* the evidence: the holder identified as some other executable, AND a claim
|
|
631
|
+
* that durably recorded the child it spawned. Without the second half the first
|
|
632
|
+
* cannot be trusted — a process that has `exec`d is not the file the claim
|
|
633
|
+
* recorded — so every holder stays a holder, as one does when the claim cannot
|
|
634
|
+
* classify it at all. Nothing but a Relay start ever passes this descriptor on,
|
|
635
|
+
* and a spurious refusal is recoverable where a wrong "free" verdict is not.
|
|
636
|
+
*
|
|
637
|
+
* `selfPids` are dropped from the holder set: a start re-reading its own
|
|
638
|
+
* reservation still has that generation's descriptor open, and its own
|
|
639
|
+
* descriptor is not evidence that somebody else owns the node id.
|
|
640
|
+
*/
|
|
641
|
+
async function inspectClaimHold(claim, env, deps, selfPids) {
|
|
642
|
+
const file = nodeClaimHoldPath(claim.node_id, env, claim.generation ?? 1);
|
|
643
|
+
if (!fs.existsSync(file)) {
|
|
644
|
+
return { held: false, reason: `no start holds ${file}`, pids: [] };
|
|
645
|
+
}
|
|
646
|
+
const execCommand = deps.execCommand;
|
|
647
|
+
if (!execCommand) {
|
|
648
|
+
return { held: true, reason: `open descriptors on ${file} could not be checked`, pids: [] };
|
|
649
|
+
}
|
|
650
|
+
let stdout;
|
|
651
|
+
try {
|
|
652
|
+
({ stdout } = await execCommand(`LC_ALL=C lsof -t -- ${shellQuote(file)} 2>/dev/null; printf 'rc=%d' "$?"`));
|
|
653
|
+
}
|
|
654
|
+
catch {
|
|
655
|
+
return { held: true, reason: `open descriptors on ${file} could not be checked`, pids: [] };
|
|
656
|
+
}
|
|
657
|
+
const status = /rc=(\d+)/.exec(stdout);
|
|
658
|
+
const pids = stdout
|
|
659
|
+
.replace(/rc=\d+/, '')
|
|
660
|
+
.split(/\s+/)
|
|
661
|
+
.map((value) => value.trim())
|
|
662
|
+
.filter((value) => /^\d+$/.test(value))
|
|
663
|
+
.filter((value) => !selfPids?.has(Number(value)));
|
|
664
|
+
const hint = brokerExecutableHint(claim);
|
|
665
|
+
// Ruling a holder OUT requires the claim to know which process this start
|
|
666
|
+
// actually spawned. Everything recorded before a spawn describes a FILE, and
|
|
667
|
+
// `execve` swaps the file while keeping the descriptor: a launcher that has
|
|
668
|
+
// already exec'd the real broker maps a binary the claim never recorded and
|
|
669
|
+
// no longer carries the launcher anywhere in its argv. `broker_child_pid` is
|
|
670
|
+
// what covers that — but it is published after the spawn, so a supervisor
|
|
671
|
+
// SIGKILLed in between leaves a claim that can never identify its own broker,
|
|
672
|
+
// and the one live process holding the node's fence classifies as
|
|
673
|
+
// `unrelated`. The node id then reads free while that broker is on its way to
|
|
674
|
+
// registering, which is the eviction this module exists to prevent.
|
|
675
|
+
//
|
|
676
|
+
// So until child identity is durably established, a holder of this descriptor
|
|
677
|
+
// keeps the node guarded whatever it is running. Nothing but a Relay start
|
|
678
|
+
// ever passes this descriptor on, and the bias is the module's usual one: a
|
|
679
|
+
// refusal costs one `--force`, a wrong "free" verdict costs deliveries.
|
|
680
|
+
const childEstablished = claim.broker_child_pid !== undefined;
|
|
681
|
+
const brokers = [];
|
|
682
|
+
const unprovable = [];
|
|
683
|
+
for (const pid of pids) {
|
|
684
|
+
const verdict = await classifyHolderProcess(Number(pid), hint, deps);
|
|
685
|
+
if (verdict !== 'unrelated')
|
|
686
|
+
brokers.push(pid);
|
|
687
|
+
else if (!childEstablished)
|
|
688
|
+
unprovable.push(pid);
|
|
689
|
+
}
|
|
690
|
+
if (brokers.length > 0) {
|
|
691
|
+
return {
|
|
692
|
+
held: true,
|
|
693
|
+
reason: `pid ${brokers.join(', ')} still holds ${file} open`,
|
|
694
|
+
pids: brokers.map(Number),
|
|
695
|
+
};
|
|
696
|
+
}
|
|
697
|
+
if (unprovable.length > 0) {
|
|
698
|
+
return {
|
|
699
|
+
held: true,
|
|
700
|
+
reason: `pid ${unprovable.join(', ')} still holds ${file} open, and the start that created it ` +
|
|
701
|
+
'never recorded the broker child it spawned, so that process cannot be ruled out',
|
|
702
|
+
pids: unprovable.map(Number),
|
|
703
|
+
};
|
|
704
|
+
}
|
|
705
|
+
if (pids.length > 0) {
|
|
706
|
+
return {
|
|
707
|
+
held: false,
|
|
708
|
+
reason: `only unrelated processes (pid ${pids.join(', ')}) hold ${file} open`,
|
|
709
|
+
pids: [],
|
|
710
|
+
};
|
|
711
|
+
}
|
|
712
|
+
// `lsof` exits 1 for "no holders"; anything else means it could not answer.
|
|
713
|
+
if (!status || (status[1] !== '0' && status[1] !== '1')) {
|
|
714
|
+
return { held: true, reason: `open descriptors on ${file} could not be checked`, pids: [] };
|
|
715
|
+
}
|
|
716
|
+
return { held: false, reason: `no process holds ${file} open`, pids: [] };
|
|
717
|
+
}
|
|
718
|
+
/**
|
|
719
|
+
* Whether anything other than `selfPids` still holds this claim's generation
|
|
720
|
+
* fenced, for a caller deciding whether ownership may be dropped.
|
|
721
|
+
*
|
|
722
|
+
* A release has to consult this, not just the pids on the record. The record
|
|
723
|
+
* names a broker only once a start got far enough to capture one: a spawn that
|
|
724
|
+
* rejects before it returns a client — a handshake that never completed, a
|
|
725
|
+
* SIGTERM the child outlived — leaves a broker child that no pid anywhere names,
|
|
726
|
+
* and releasing on "no live pid recorded" tombstones the claim and unlinks the
|
|
727
|
+
* hold file out from under a process that is still fenced by it. The next start
|
|
728
|
+
* then reads the node id as free while that child can still register. The
|
|
729
|
+
* descriptor is the one piece of evidence that exists from the instant the child
|
|
730
|
+
* does, so it is what decides.
|
|
731
|
+
*
|
|
732
|
+
* Holders are filtered exactly as {@link inspectNodeClaim} filters them, so a
|
|
733
|
+
* release never frees a node id that another start would still read as held.
|
|
734
|
+
*/
|
|
735
|
+
export async function inspectNodeClaimHold(claim, deps = {}, selfPids) {
|
|
736
|
+
return inspectClaimHold(claim, deps.env ?? process.env, deps, selfPids);
|
|
737
|
+
}
|
|
738
|
+
/**
|
|
739
|
+
* Create and open this generation's hold file, handing the supervising CLI the
|
|
740
|
+
* descriptor it passes to the broker child.
|
|
741
|
+
*
|
|
742
|
+
* Created exclusively (`wx`): only the acquisition that won the generation ever
|
|
743
|
+
* creates it, so a concurrent start can never replace the inode a live child is
|
|
744
|
+
* holding. Opened BEFORE the spawn, because a fence established after `fork`
|
|
745
|
+
* would leave exactly the gap it exists to close.
|
|
746
|
+
*
|
|
747
|
+
* Failing to establish the hold is FATAL to the start, which is why this throws
|
|
748
|
+
* rather than reporting a missing fence. Without the descriptor, a supervisor
|
|
749
|
+
* killed between the fork and the broker's `connection.json` write leaves a
|
|
750
|
+
* claim with only dead pids and no evidence at all — so the next start reads
|
|
751
|
+
* the node id as free and evicts a broker that is about to register. Continuing
|
|
752
|
+
* would mean spawning a broker under a guarantee that is not actually in force,
|
|
753
|
+
* which is worse than not starting: the operator can see and fix a refusal.
|
|
754
|
+
*
|
|
755
|
+
* @throws NodeClaimHoldError when the hold could not be established. Any
|
|
756
|
+
* partially created file and descriptor are cleaned up first, so a retry (or a
|
|
757
|
+
* later start, on the next generation) is not blocked by this one's debris.
|
|
758
|
+
*/
|
|
759
|
+
export function openNodeClaimHold(claim, env = process.env) {
|
|
760
|
+
const generation = claim.generation ?? 1;
|
|
761
|
+
const file = nodeClaimHoldPath(claim.node_id, env, generation);
|
|
762
|
+
let fd;
|
|
763
|
+
try {
|
|
764
|
+
fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
|
|
765
|
+
fd = fs.openSync(file, 'wx', 0o600);
|
|
766
|
+
// Contents are operator-facing only; the evidence is the open descriptor.
|
|
767
|
+
fs.writeSync(fd, `${JSON.stringify({ node_id: claim.node_id, generation, supervisor_pid: claim.pid, state_dir: claim.state_dir }, null, 2)}\n`);
|
|
768
|
+
return fd;
|
|
769
|
+
}
|
|
770
|
+
catch (error) {
|
|
771
|
+
// Drop our reference before unlinking: an fd we opened and then abandoned
|
|
772
|
+
// would keep answering `lsof` for as long as this process lives, pinning a
|
|
773
|
+
// node id behind a fence nothing is actually being fenced by.
|
|
774
|
+
closeNodeClaimHold(fd);
|
|
775
|
+
if (fd !== undefined)
|
|
776
|
+
safeUnlinkClaim(file);
|
|
777
|
+
throw new NodeClaimHoldError(claim.node_id, file, error);
|
|
778
|
+
}
|
|
779
|
+
}
|
|
780
|
+
/** Drop this process's reference to a hold file. Other holders keep it alive. */
|
|
781
|
+
export function closeNodeClaimHold(fd) {
|
|
782
|
+
if (fd === undefined)
|
|
783
|
+
return;
|
|
784
|
+
try {
|
|
785
|
+
fs.closeSync(fd);
|
|
786
|
+
}
|
|
787
|
+
catch {
|
|
788
|
+
// Already closed, or never ours.
|
|
789
|
+
}
|
|
790
|
+
}
|
|
791
|
+
/**
|
|
792
|
+
* A live broker serving `stateDir`, discovered from the connection file the
|
|
793
|
+
* broker writes itself.
|
|
794
|
+
*
|
|
795
|
+
* This is the ownership evidence that survives the supervising CLI. The Rust
|
|
796
|
+
* broker writes `<state dir>/connection.json` (with its own pid) as soon as its
|
|
797
|
+
* API listener binds — BEFORE `connect_relay` and before `node.register` is
|
|
798
|
+
* queued (crates/broker/src/runtime/init.rs) — so a broker that was orphaned by
|
|
799
|
+
* a SIGKILLed supervisor before the CLI could record its pid is still
|
|
800
|
+
* discoverable by every other start on the machine. Without it, such an orphan
|
|
801
|
+
* reads as "nobody home" and the next `node up` evicts its delivery socket.
|
|
802
|
+
*/
|
|
803
|
+
export async function findLiveStateDirBroker(stateDir, deps = {},
|
|
804
|
+
/** What the claim for this state dir knows about its broker executable. */
|
|
805
|
+
hint = {}) {
|
|
806
|
+
const pid = readStateDirBrokerPid(stateDir);
|
|
807
|
+
if (pid === null || !isProcessAlive(pid, deps)) {
|
|
808
|
+
return null;
|
|
809
|
+
}
|
|
810
|
+
// Unlike the hold descriptor, this pid was read from a file and can have been
|
|
811
|
+
// recycled by any process on the machine, with no relationship to Relay at
|
|
812
|
+
// all. Only a positive identification keeps the node id guarded here.
|
|
813
|
+
if ((await classifyHolderProcess(pid, { stateDir, ...hint }, deps)) !== 'broker') {
|
|
814
|
+
return null;
|
|
815
|
+
}
|
|
816
|
+
const startedAt = await readProcessStartedAt(pid, deps);
|
|
817
|
+
return { pid, ...(startedAt ? { startedAt } : {}) };
|
|
818
|
+
}
|
|
819
|
+
/**
|
|
820
|
+
* Classify a claim for `nodeId` — the read-only view used by preflight guards
|
|
821
|
+
* and diagnostics.
|
|
822
|
+
*
|
|
823
|
+
* A claim is retired only when every process it names is provably gone (the pid
|
|
824
|
+
* no longer exists, or its birth time no longer matches the one recorded) AND
|
|
825
|
+
* no live broker occupies its state dir. Everything else — including a `ps` we
|
|
826
|
+
* cannot run — reads as held, because refusing is recoverable (`--force`,
|
|
827
|
+
* `node down`) while a wrong "free" verdict silently cuts delivery to a live
|
|
828
|
+
* broker.
|
|
829
|
+
*/
|
|
830
|
+
export async function inspectNodeClaim(nodeId, deps = {}) {
|
|
831
|
+
const env = deps.env ?? process.env;
|
|
832
|
+
const generations = readClaimGenerations(nodeId, env);
|
|
833
|
+
// A held generation can sit beneath a stale top claim: a `--force` takeover
|
|
834
|
+
// preserves the incumbent's record while its broker lives, and a failed
|
|
835
|
+
// takeover leaves it under the replacement's tombstone. Any live holder is
|
|
836
|
+
// a conflict for a new start, so report the newest held generation rather
|
|
837
|
+
// than only judging the top file. Newest-first keeps the diagnosis pointed
|
|
838
|
+
// at the most recent owner.
|
|
839
|
+
let fallback = null;
|
|
840
|
+
for (let index = generations.length - 1; index >= 0; index -= 1) {
|
|
841
|
+
const entry = generations[index];
|
|
842
|
+
if (!entry.claim)
|
|
843
|
+
continue;
|
|
844
|
+
const status = await classifyNodeClaim(nodeId, entry.claim, env, deps);
|
|
845
|
+
if (status.state === 'held')
|
|
846
|
+
return status;
|
|
847
|
+
fallback ??= status;
|
|
848
|
+
}
|
|
849
|
+
return fallback ?? { state: 'unclaimed' };
|
|
850
|
+
}
|
|
851
|
+
/**
|
|
852
|
+
* Decide whether `claim` still names a live owner.
|
|
853
|
+
*
|
|
854
|
+
* `selfPids` names processes that ARE the caller. They are skipped as recorded
|
|
855
|
+
* holders — a process cannot be a competing broker against itself — but they
|
|
856
|
+
* suppress nothing else: the orphan fences below still run in full, because a
|
|
857
|
+
* pid the caller happens to share (its own, or one recycled from the dead
|
|
858
|
+
* supervisor this claim records) says nothing about the broker that supervisor
|
|
859
|
+
* may have left registered.
|
|
860
|
+
*/
|
|
861
|
+
async function classifyNodeClaim(nodeId, claim, env, deps, selfPids) {
|
|
862
|
+
if (claim.node_id.trim() !== nodeId.trim()) {
|
|
863
|
+
// Sanitized filenames can collide. Report it instead of overwriting the
|
|
864
|
+
// other node's claim, which would leave that broker unguarded.
|
|
865
|
+
return {
|
|
866
|
+
state: 'held',
|
|
867
|
+
claim,
|
|
868
|
+
reason: `claim file ${nodeClaimPath(nodeId, env, claim.generation ?? 1)} records node ${claim.node_id}`,
|
|
869
|
+
};
|
|
870
|
+
}
|
|
871
|
+
const reasons = [];
|
|
872
|
+
for (const candidate of claimProcesses(claim)) {
|
|
873
|
+
if (selfPids?.has(candidate.pid)) {
|
|
874
|
+
reasons.push(`${candidate.role} pid ${candidate.pid} is this start itself`);
|
|
875
|
+
continue;
|
|
876
|
+
}
|
|
877
|
+
const status = await isRecordedProcessAlive(candidate.pid, candidate.startedAt, deps);
|
|
878
|
+
if (status.alive) {
|
|
879
|
+
return { state: 'held', claim, reason: `${candidate.role} ${status.reason}` };
|
|
880
|
+
}
|
|
881
|
+
reasons.push(`${candidate.role} ${status.reason}`);
|
|
882
|
+
}
|
|
883
|
+
// Every recorded pid is gone, but a broker this claim started can have
|
|
884
|
+
// outlived them: the supervisor may have been SIGKILLed between the spawn and
|
|
885
|
+
// the moment it could record the broker's pid.
|
|
886
|
+
//
|
|
887
|
+
// The hold descriptor is checked FIRST because it is the only evidence that
|
|
888
|
+
// exists for the whole life of that orphan. A broker child inherits it across
|
|
889
|
+
// `fork`, so it answers even while the child is still paused before binding
|
|
890
|
+
// its API — the window in which `connection.json` does not exist yet and the
|
|
891
|
+
// old "dead supervisor, nothing published" reading let a second broker start.
|
|
892
|
+
const hold = await inspectClaimHold(claim, env, deps, selfPids);
|
|
893
|
+
if (hold.held) {
|
|
894
|
+
return {
|
|
895
|
+
state: 'held',
|
|
896
|
+
// Report the pid that is actually holding the node, not the supervisor
|
|
897
|
+
// the record still names — that one is what was just proven dead, and an
|
|
898
|
+
// operator told to stop it would be chasing a process that is gone.
|
|
899
|
+
claim: {
|
|
900
|
+
...claim,
|
|
901
|
+
...(hold.pids[0] !== undefined ? { pid: hold.pids[0] } : {}),
|
|
902
|
+
supervisor_pid: undefined,
|
|
903
|
+
},
|
|
904
|
+
reason: `${reasons.join('; ')}, but ${hold.reason}`,
|
|
905
|
+
};
|
|
906
|
+
}
|
|
907
|
+
// Then the broker's own connection file, which survives a supervisor that
|
|
908
|
+
// died after its child had published but before it could record the pid.
|
|
909
|
+
const orphan = await findLiveStateDirBroker(claim.state_dir, deps, {
|
|
910
|
+
binary: claim.broker_binary,
|
|
911
|
+
object: claim.broker_executable,
|
|
912
|
+
});
|
|
913
|
+
if (orphan) {
|
|
914
|
+
return {
|
|
915
|
+
state: 'held',
|
|
916
|
+
claim: {
|
|
917
|
+
...claim,
|
|
918
|
+
pid: orphan.pid,
|
|
919
|
+
...(orphan.startedAt ? { process_started_at: orphan.startedAt } : {}),
|
|
920
|
+
supervisor_pid: undefined,
|
|
921
|
+
status: 'active',
|
|
922
|
+
},
|
|
923
|
+
reason: `${reasons.join('; ')}, but broker pid ${orphan.pid} recorded in ` +
|
|
924
|
+
`${path.join(claim.state_dir, BROKER_CONNECTION_FILENAME)} is still running`,
|
|
925
|
+
};
|
|
926
|
+
}
|
|
927
|
+
return { state: 'stale', claim, reason: reasons.join('; ') };
|
|
928
|
+
}
|
|
929
|
+
/** Live claims on this machine, for locating a broker across state dirs. */
|
|
930
|
+
export async function listHeldNodeClaims(deps = {}) {
|
|
931
|
+
const env = deps.env ?? process.env;
|
|
932
|
+
const held = [];
|
|
933
|
+
for (const claim of listAllNodeClaims(env)) {
|
|
934
|
+
const status = await classifyNodeClaim(claim.node_id, claim, env, deps);
|
|
935
|
+
if (status.state === 'held') {
|
|
936
|
+
held.push(status.claim);
|
|
937
|
+
}
|
|
938
|
+
}
|
|
939
|
+
return held;
|
|
940
|
+
}
|
|
941
|
+
/* ------------------------------------------------------------------ *
|
|
942
|
+
* Acquire / adopt / release
|
|
943
|
+
* ------------------------------------------------------------------ */
|
|
944
|
+
/**
|
|
945
|
+
* How many generations to try before giving up. Each attempt only loses to a
|
|
946
|
+
* start that genuinely won the node id, so more than a couple means the machine
|
|
947
|
+
* is starting brokers for one node id in a tight loop.
|
|
948
|
+
*/
|
|
949
|
+
const CLAIM_ACQUIRE_ATTEMPTS = 8;
|
|
950
|
+
/**
|
|
951
|
+
* Create a claim file that does not exist yet, with its full contents already
|
|
952
|
+
* in place.
|
|
953
|
+
*
|
|
954
|
+
* The record is written to a private temp file and hard-linked into its
|
|
955
|
+
* generation path: `link(2)` fails with `EEXIST` rather than clobbering, so the
|
|
956
|
+
* create is both exclusive AND atomic. A reader can therefore never observe a
|
|
957
|
+
* half-written claim — which matters, because a torn read of a live broker's
|
|
958
|
+
* claim would read as "node id free".
|
|
959
|
+
*
|
|
960
|
+
* @returns False when that generation already exists.
|
|
961
|
+
*/
|
|
962
|
+
function createClaimGeneration(file, claim) {
|
|
963
|
+
const dir = path.dirname(file);
|
|
964
|
+
fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
965
|
+
const temporary = path.join(dir, `.${path.basename(file)}.${process.pid}.${crypto.randomUUID()}.tmp`);
|
|
966
|
+
fs.writeFileSync(temporary, `${JSON.stringify(claim, null, 2)}\n`, { mode: 0o600 });
|
|
967
|
+
try {
|
|
968
|
+
fs.linkSync(temporary, file);
|
|
969
|
+
return true;
|
|
970
|
+
}
|
|
971
|
+
catch (error) {
|
|
972
|
+
if (error?.code === 'EEXIST')
|
|
973
|
+
return false;
|
|
974
|
+
throw error;
|
|
975
|
+
}
|
|
976
|
+
finally {
|
|
977
|
+
try {
|
|
978
|
+
fs.unlinkSync(temporary);
|
|
979
|
+
}
|
|
980
|
+
catch {
|
|
981
|
+
// Already gone; the hard link (if any) keeps the contents alive.
|
|
982
|
+
}
|
|
983
|
+
}
|
|
984
|
+
}
|
|
985
|
+
/** Replace a claim file in place, atomically, so no reader sees a partial write. */
|
|
986
|
+
function writeClaimRecordAtomically(file, record) {
|
|
987
|
+
const temporary = `${file}.${process.pid}.${crypto.randomUUID()}.tmp`;
|
|
988
|
+
fs.writeFileSync(temporary, `${JSON.stringify(record, null, 2)}\n`, { mode: 0o600 });
|
|
989
|
+
fs.renameSync(temporary, file);
|
|
990
|
+
}
|
|
991
|
+
function safeUnlinkClaim(file) {
|
|
992
|
+
try {
|
|
993
|
+
fs.unlinkSync(file);
|
|
994
|
+
}
|
|
995
|
+
catch {
|
|
996
|
+
// Already gone, or not ours to remove.
|
|
997
|
+
}
|
|
998
|
+
}
|
|
999
|
+
/**
|
|
1000
|
+
* Remove a generation this acquisition has superseded, and its hold file.
|
|
1001
|
+
*
|
|
1002
|
+
* Only removed while the file still holds the exact bytes the scan classified.
|
|
1003
|
+
* Generation numbers are never reissued (see {@link NodeClaimTombstone}), so a
|
|
1004
|
+
* path cannot come back as somebody else's claim — but the scan that chose
|
|
1005
|
+
* these files happened before an `await`, and pruning by path alone is what
|
|
1006
|
+
* deleted a live successor's record when a number COULD be reissued. Comparing
|
|
1007
|
+
* the bytes makes the check local instead of resting on that invariant.
|
|
1008
|
+
*/
|
|
1009
|
+
function pruneSupersededGeneration(entry, nodeId, env) {
|
|
1010
|
+
if (readClaimBytes(entry.file) !== entry.raw) {
|
|
1011
|
+
return;
|
|
1012
|
+
}
|
|
1013
|
+
safeUnlinkClaim(entry.file);
|
|
1014
|
+
safeUnlinkClaim(nodeClaimHoldPath(nodeId, env, entry.generation));
|
|
1015
|
+
}
|
|
1016
|
+
async function buildClaim(input, generation, deps) {
|
|
1017
|
+
const startedAt = await readProcessStartedAt(input.pid, deps);
|
|
1018
|
+
const supervisorPid = input.supervisorPid ?? input.pid;
|
|
1019
|
+
const supervisorStartedAt = supervisorPid === input.pid ? startedAt : await readProcessStartedAt(supervisorPid, deps);
|
|
1020
|
+
return {
|
|
1021
|
+
version: 1,
|
|
1022
|
+
node_id: input.nodeId.trim(),
|
|
1023
|
+
pid: input.pid,
|
|
1024
|
+
state_dir: normalizeClaimStateDir(input.stateDir),
|
|
1025
|
+
...describeBrokerExecutable(input.brokerBinary),
|
|
1026
|
+
...(input.apiPort !== undefined ? { api_port: input.apiPort } : {}),
|
|
1027
|
+
...(input.brokerName ? { broker_name: input.brokerName } : {}),
|
|
1028
|
+
...(startedAt ? { process_started_at: startedAt } : {}),
|
|
1029
|
+
supervisor_pid: supervisorPid,
|
|
1030
|
+
...(supervisorStartedAt ? { supervisor_started_at: supervisorStartedAt } : {}),
|
|
1031
|
+
status: input.status ?? 'active',
|
|
1032
|
+
claimed_at: new Date().toISOString(),
|
|
1033
|
+
generation,
|
|
1034
|
+
owner_token: crypto.randomUUID(),
|
|
1035
|
+
};
|
|
1036
|
+
}
|
|
1037
|
+
/**
|
|
1038
|
+
* Record ownership of `nodeId`, refusing if a live local broker already holds
|
|
1039
|
+
* it.
|
|
1040
|
+
*
|
|
1041
|
+
* Ownership IS the exclusive creation of the next generation file. Generations
|
|
1042
|
+
* only increase and each one is created exactly once — for the whole life of
|
|
1043
|
+
* the stem, not just while a claim is live: {@link releaseNodeClaim} leaves a
|
|
1044
|
+
* tombstone in place of the record it drops, so the highest number on disk
|
|
1045
|
+
* never falls back. Taking a node id over therefore never involves deleting a
|
|
1046
|
+
* file another process might have replaced in the meantime — the failure mode
|
|
1047
|
+
* that makes "validate the holder, then remove its record, then write ours"
|
|
1048
|
+
* unsafe no matter how the validation is fenced.
|
|
1049
|
+
*
|
|
1050
|
+
* Without that, a start suspended between the scan and the create could wake up
|
|
1051
|
+
* after a complete release-and-reacquire cycle, re-create a number that had
|
|
1052
|
+
* been reissued to a live successor, pass the higher-generation check because
|
|
1053
|
+
* its own file was the highest again, and then prune that successor's claim
|
|
1054
|
+
* from its own stale scan. Both would stay alive with one of them recorded —
|
|
1055
|
+
* exactly the double registration this module exists to prevent.
|
|
1056
|
+
*
|
|
1057
|
+
* Two starts that both observe the same stale claim therefore cannot both win:
|
|
1058
|
+
* one creates generation N+1, the other collides (`EEXIST`) and re-reads, now
|
|
1059
|
+
* seeing the winner's live claim. A start that created its generation while a
|
|
1060
|
+
* third one was creating a higher one loses the confirm step below, removes its
|
|
1061
|
+
* own file and refuses. Exactly one — the highest generation NUMBER ever issued
|
|
1062
|
+
* for the stem, live claim or spent tombstone — can own the node id, which is
|
|
1063
|
+
* the invariant the post-create check below enforces.
|
|
1064
|
+
*
|
|
1065
|
+
* Callers reserve with `status: 'reserved'` and their own pid BEFORE spawning a
|
|
1066
|
+
* broker, open the generation's hold descriptor with {@link openNodeClaimHold}
|
|
1067
|
+
* so the child inherits it, and then hand ownership to the verified broker with
|
|
1068
|
+
* {@link adoptNodeClaim}.
|
|
1069
|
+
*
|
|
1070
|
+
* @throws NodeClaimConflictError when a live local broker holds the node id and
|
|
1071
|
+
* `force` was not requested.
|
|
1072
|
+
* @throws NodeClaimContentionError when no generation could be won at all.
|
|
1073
|
+
*/
|
|
1074
|
+
export async function acquireNodeClaim(input) {
|
|
1075
|
+
const env = input.env ?? process.env;
|
|
1076
|
+
const deps = { ...input, env };
|
|
1077
|
+
const nodeId = input.nodeId.trim();
|
|
1078
|
+
const selfPids = new Set([input.pid]);
|
|
1079
|
+
for (let attempt = 0; attempt < CLAIM_ACQUIRE_ATTEMPTS; attempt += 1) {
|
|
1080
|
+
const generations = readClaimGenerations(nodeId, env);
|
|
1081
|
+
if (!input.force) {
|
|
1082
|
+
// Our own pid is never evidence that somebody else holds the node id, so
|
|
1083
|
+
// a start may re-take a claim that records it. That exemption is scoped
|
|
1084
|
+
// to the pid AS A RECORDED HOLDER and nothing more: the fence still runs.
|
|
1085
|
+
//
|
|
1086
|
+
// Exempting the whole check on pid equality (as this once did) was
|
|
1087
|
+
// unsound, because a pid is not an identity. A supervisor that died
|
|
1088
|
+
// leaving a registered broker behind records a pid the OS is free to
|
|
1089
|
+
// reissue, and the next start to be handed that number would have walked
|
|
1090
|
+
// straight past the orphan its own claim was pointing at.
|
|
1091
|
+
//
|
|
1092
|
+
// EVERY live claim refuses, not just the current one: a `--force`
|
|
1093
|
+
// takeover preserves a held generation it supersedes (see the prune
|
|
1094
|
+
// below), so more than one generation can record a live broker. Checking
|
|
1095
|
+
// only the highest would let a stale top claim mask a held one beneath.
|
|
1096
|
+
// Scan newest-first so a conflict names the most recent owner.
|
|
1097
|
+
for (let index = generations.length - 1; index >= 0; index -= 1) {
|
|
1098
|
+
const entry = generations[index];
|
|
1099
|
+
if (!entry.claim)
|
|
1100
|
+
continue;
|
|
1101
|
+
const status = await classifyNodeClaim(nodeId, entry.claim, env, deps, selfPids);
|
|
1102
|
+
if (status.state === 'held') {
|
|
1103
|
+
throw new NodeClaimConflictError(nodeId, status.claim);
|
|
1104
|
+
}
|
|
1105
|
+
}
|
|
1106
|
+
}
|
|
1107
|
+
const generation = highestGenerationNumber(generations) + 1;
|
|
1108
|
+
const file = nodeClaimPath(nodeId, env, generation);
|
|
1109
|
+
const claim = await buildClaim(input, generation, deps);
|
|
1110
|
+
if (!createClaimGeneration(file, claim)) {
|
|
1111
|
+
// Another start took this generation between the scan and the create.
|
|
1112
|
+
// Re-read: its claim is what decides whether we may continue at all.
|
|
1113
|
+
continue;
|
|
1114
|
+
}
|
|
1115
|
+
const after = readClaimGenerations(nodeId, env);
|
|
1116
|
+
if (highestGenerationNumber(after) > generation) {
|
|
1117
|
+
// Somebody else has already been ISSUED a higher generation number, so
|
|
1118
|
+
// this one cannot be the owner — whatever that higher file holds now.
|
|
1119
|
+
//
|
|
1120
|
+
// The test is the highest NUMBER on disk and not the highest live claim:
|
|
1121
|
+
// a spent generation is a file too. A start suspended since before a
|
|
1122
|
+
// release cycle could otherwise wake up, re-create a low number in the
|
|
1123
|
+
// gap the cycle's pruning left, see only tombstones above it and declare
|
|
1124
|
+
// itself the owner — while the start that had already read those
|
|
1125
|
+
// tombstones goes on to create `max + 1`. Neither one's prune list names
|
|
1126
|
+
// the other (each scanned before the other's file existed), so both stay
|
|
1127
|
+
// live and both spawn. Only the record with the highest number ever
|
|
1128
|
+
// issued can own the node id.
|
|
1129
|
+
safeUnlinkClaim(file);
|
|
1130
|
+
const winner = currentGeneration(after);
|
|
1131
|
+
if (winner && winner.generation > generation) {
|
|
1132
|
+
// A live claim above ours: that start legitimately won the node id.
|
|
1133
|
+
throw new NodeClaimConflictError(nodeId, winner.claim ?? claim);
|
|
1134
|
+
}
|
|
1135
|
+
// Only spent or unreadable generations sit above ours, so the node id
|
|
1136
|
+
// itself is free — this number just is not ours to hold. Re-scan and take
|
|
1137
|
+
// one above the highest file on disk instead.
|
|
1138
|
+
continue;
|
|
1139
|
+
}
|
|
1140
|
+
for (const superseded of generations) {
|
|
1141
|
+
if (!superseded.claim) {
|
|
1142
|
+
pruneSupersededGeneration(superseded, nodeId, env);
|
|
1143
|
+
continue;
|
|
1144
|
+
}
|
|
1145
|
+
// A generation whose holder is still alive is not garbage: it is the
|
|
1146
|
+
// only record guarding that broker's node id. A `--force` takeover that
|
|
1147
|
+
// pruned it here and then failed to start left the incumbent running and
|
|
1148
|
+
// unguarded — the next plain `node up` took the node id from under a
|
|
1149
|
+
// live, still-registered broker. Held generations stay until their
|
|
1150
|
+
// holder exits (a later acquisition prunes them as stale) or their
|
|
1151
|
+
// broker is stopped (releaseNodeClaimsForBroker retires them); our own
|
|
1152
|
+
// superseded generations are exempted from "held" the same way the
|
|
1153
|
+
// conflict check above exempts them, so a re-acquire still cleans up.
|
|
1154
|
+
const status = await classifyNodeClaim(nodeId, superseded.claim, env, deps, selfPids);
|
|
1155
|
+
if (status.state !== 'held') {
|
|
1156
|
+
pruneSupersededGeneration(superseded, nodeId, env);
|
|
1157
|
+
}
|
|
1158
|
+
}
|
|
1159
|
+
return claim;
|
|
1160
|
+
}
|
|
1161
|
+
throw new NodeClaimContentionError(nodeId, nodeClaimsDir(env));
|
|
1162
|
+
}
|
|
1163
|
+
function isSameAcquisition(current, claim) {
|
|
1164
|
+
if (!current)
|
|
1165
|
+
return false;
|
|
1166
|
+
if (claim.owner_token)
|
|
1167
|
+
return current.owner_token === claim.owner_token;
|
|
1168
|
+
// Claims written before this process (or by hand) carry no token; fall back
|
|
1169
|
+
// to the identity the record does have.
|
|
1170
|
+
return (current.node_id === claim.node_id && current.pid === claim.pid && current.state_dir === claim.state_dir);
|
|
1171
|
+
}
|
|
1172
|
+
/**
|
|
1173
|
+
* Record the pid of the child this reservation just handed its hold descriptor
|
|
1174
|
+
* to, so a later start can recognise that process no matter what it becomes.
|
|
1175
|
+
*
|
|
1176
|
+
* Deliberately synchronous, and called from the spawn itself rather than from
|
|
1177
|
+
* the adoption that follows it: the whole point is to have the pid on disk
|
|
1178
|
+
* before the child can `exec` into something the recorded executable no longer
|
|
1179
|
+
* describes, and before this supervisor can be killed. It is a plain write of
|
|
1180
|
+
* OUR OWN generation with no `await` between the read and the write, so it
|
|
1181
|
+
* cannot interleave with anything.
|
|
1182
|
+
*
|
|
1183
|
+
* A failure is never fatal. This is additional positive evidence layered on top
|
|
1184
|
+
* of a fence that is already in place — the descriptor was inherited at the
|
|
1185
|
+
* spawn — so losing it costs the launcher/exec case, not the guarantee. The
|
|
1186
|
+
* caller keeps the returned claim, which is unchanged when nothing was written.
|
|
1187
|
+
*/
|
|
1188
|
+
export function recordSpawnedBrokerChild(reservation, pid, env = process.env) {
|
|
1189
|
+
if (!Number.isSafeInteger(pid) || pid <= 0)
|
|
1190
|
+
return reservation;
|
|
1191
|
+
const generation = reservation.generation ?? 1;
|
|
1192
|
+
const file = nodeClaimPath(reservation.node_id, env, generation);
|
|
1193
|
+
try {
|
|
1194
|
+
// A `--force` takeover that landed during the spawn owns the node id now.
|
|
1195
|
+
// Rewriting its record would be this module's one forbidden move; the
|
|
1196
|
+
// adoption below will fail this start on the same evidence.
|
|
1197
|
+
if (!isSameAcquisition(readClaimFile(file), reservation))
|
|
1198
|
+
return reservation;
|
|
1199
|
+
const claim = { ...reservation, broker_child_pid: pid };
|
|
1200
|
+
writeClaimRecordAtomically(file, claim);
|
|
1201
|
+
return claim;
|
|
1202
|
+
}
|
|
1203
|
+
catch {
|
|
1204
|
+
return reservation;
|
|
1205
|
+
}
|
|
1206
|
+
}
|
|
1207
|
+
/**
|
|
1208
|
+
* Hand a reservation to the verified broker process that now owns the state
|
|
1209
|
+
* dir.
|
|
1210
|
+
*
|
|
1211
|
+
* Only the acquisition that created a generation ever rewrites it, so this is a
|
|
1212
|
+
* conflict-free in-place update — but it is still conditional on that
|
|
1213
|
+
* generation still being the highest: a `--force` takeover that landed while
|
|
1214
|
+
* this broker was starting has already won the node id, and continuing would
|
|
1215
|
+
* put two brokers back on one delivery socket. Losing here fails startup, which
|
|
1216
|
+
* tears this broker down.
|
|
1217
|
+
*
|
|
1218
|
+
* @throws NodeClaimConflictError when the reservation no longer owns the node id.
|
|
1219
|
+
*/
|
|
1220
|
+
export async function adoptNodeClaim(input) {
|
|
1221
|
+
const env = input.env ?? process.env;
|
|
1222
|
+
const deps = { ...input, env };
|
|
1223
|
+
const { reservation } = input;
|
|
1224
|
+
const generation = reservation.generation ?? 1;
|
|
1225
|
+
const file = nodeClaimPath(reservation.node_id, env, generation);
|
|
1226
|
+
const before = readClaimGenerations(reservation.node_id, env);
|
|
1227
|
+
if (highestGenerationNumber(before) > generation || !isSameAcquisition(readClaimFile(file), reservation)) {
|
|
1228
|
+
throw new NodeClaimConflictError(reservation.node_id, currentGeneration(before)?.claim ?? reservation);
|
|
1229
|
+
}
|
|
1230
|
+
const startedAt = await readProcessStartedAt(input.pid, deps);
|
|
1231
|
+
const claim = {
|
|
1232
|
+
...reservation,
|
|
1233
|
+
pid: input.pid,
|
|
1234
|
+
...(input.apiPort !== undefined ? { api_port: input.apiPort } : {}),
|
|
1235
|
+
...(input.brokerName ? { broker_name: input.brokerName } : {}),
|
|
1236
|
+
...(startedAt ? { process_started_at: startedAt } : {}),
|
|
1237
|
+
supervisor_pid: reservation.pid,
|
|
1238
|
+
...(reservation.process_started_at ? { supervisor_started_at: reservation.process_started_at } : {}),
|
|
1239
|
+
status: 'active',
|
|
1240
|
+
};
|
|
1241
|
+
if (!startedAt)
|
|
1242
|
+
delete claim.process_started_at;
|
|
1243
|
+
// Atomic replace of OUR generation only: readers see the reservation or the
|
|
1244
|
+
// adopted record, never a partial write, and never another start's file.
|
|
1245
|
+
writeClaimRecordAtomically(file, claim);
|
|
1246
|
+
const after = readClaimGenerations(reservation.node_id, env);
|
|
1247
|
+
if (highestGenerationNumber(after) > generation) {
|
|
1248
|
+
// A takeover landed during the write. It owns the node id; give ours up
|
|
1249
|
+
// rather than leaving a second live-looking record behind.
|
|
1250
|
+
safeUnlinkClaim(file);
|
|
1251
|
+
safeUnlinkClaim(nodeClaimHoldPath(reservation.node_id, env, generation));
|
|
1252
|
+
throw new NodeClaimConflictError(reservation.node_id, currentGeneration(after)?.claim ?? claim);
|
|
1253
|
+
}
|
|
1254
|
+
return claim;
|
|
1255
|
+
}
|
|
1256
|
+
/**
|
|
1257
|
+
* Drop a claim this start owns, leaving its generation number spent.
|
|
1258
|
+
*
|
|
1259
|
+
* The record is REPLACED by a tombstone rather than removed. Removing it made
|
|
1260
|
+
* the number reusable, and a suspended start could then re-create a generation
|
|
1261
|
+
* that had since been handed to somebody else: it would pass the
|
|
1262
|
+
* higher-generation check (its own file was the highest again) and prune the
|
|
1263
|
+
* successor's live claim from its own stale scan, leaving two brokers alive on
|
|
1264
|
+
* one node id with only one of them recorded. A tombstone keeps
|
|
1265
|
+
* `max(generation)` from ever decreasing, so no number is issued twice; it
|
|
1266
|
+
* parses as no claim at all, so the node id reads free immediately; and the
|
|
1267
|
+
* next acquisition prunes it, so at most one spent file per node id is ever on
|
|
1268
|
+
* disk.
|
|
1269
|
+
*
|
|
1270
|
+
* Only the acquisition that created a generation ever rewrites it, proven by
|
|
1271
|
+
* `owner_token`, and the read and the write are adjacent syscalls with no await
|
|
1272
|
+
* in between.
|
|
1273
|
+
*/
|
|
1274
|
+
export async function releaseNodeClaim(claim, env = process.env) {
|
|
1275
|
+
const generation = claim.generation ?? 1;
|
|
1276
|
+
const file = nodeClaimPath(claim.node_id, env, generation);
|
|
1277
|
+
if (!isSameAcquisition(readClaimFile(file), claim)) {
|
|
1278
|
+
return false;
|
|
1279
|
+
}
|
|
1280
|
+
try {
|
|
1281
|
+
writeClaimRecordAtomically(file, {
|
|
1282
|
+
version: 1,
|
|
1283
|
+
released: true,
|
|
1284
|
+
node_id: claim.node_id,
|
|
1285
|
+
generation,
|
|
1286
|
+
released_at: new Date().toISOString(),
|
|
1287
|
+
});
|
|
1288
|
+
}
|
|
1289
|
+
catch {
|
|
1290
|
+
return false;
|
|
1291
|
+
}
|
|
1292
|
+
// The hold descriptor protected this generation only; both processes it
|
|
1293
|
+
// fenced are gone by the time a release is allowed to run.
|
|
1294
|
+
safeUnlinkClaim(nodeClaimHoldPath(claim.node_id, env, generation));
|
|
1295
|
+
return true;
|
|
1296
|
+
}
|
|
1297
|
+
/**
|
|
1298
|
+
* Release the claims a broker pid held in a state dir. `node down` stops a
|
|
1299
|
+
* broker it did not start, so it cannot name the node id the claim was written
|
|
1300
|
+
* for — the pid and state dir it verified are what it has.
|
|
1301
|
+
*
|
|
1302
|
+
* A claim for that state dir whose every recorded pid is already dead is
|
|
1303
|
+
* released too: that is the orphan left by a supervisor that was killed before
|
|
1304
|
+
* it could record its broker's pid, and `down` has just proven the state dir's
|
|
1305
|
+
* broker is gone.
|
|
1306
|
+
*/
|
|
1307
|
+
export async function releaseNodeClaimsForBroker(input) {
|
|
1308
|
+
const env = input.env ?? process.env;
|
|
1309
|
+
const deps = { ...input, env };
|
|
1310
|
+
const stateDir = normalizeClaimStateDir(input.stateDir);
|
|
1311
|
+
const released = [];
|
|
1312
|
+
for (const claim of listAllNodeClaims(env)) {
|
|
1313
|
+
if (claim.state_dir !== stateDir)
|
|
1314
|
+
continue;
|
|
1315
|
+
const namesThisBroker = claim.pid === input.pid || claim.supervisor_pid === input.pid;
|
|
1316
|
+
if (!namesThisBroker) {
|
|
1317
|
+
const status = await classifyNodeClaim(claim.node_id, claim, env, deps);
|
|
1318
|
+
if (status.state !== 'stale')
|
|
1319
|
+
continue;
|
|
1320
|
+
}
|
|
1321
|
+
if (await releaseNodeClaim(claim, env)) {
|
|
1322
|
+
released.push(claim);
|
|
1323
|
+
}
|
|
1324
|
+
}
|
|
1325
|
+
return released;
|
|
1326
|
+
}
|
|
1327
|
+
/** One-line description of the broker holding a claim, for operator output. */
|
|
1328
|
+
export function describeNodeClaimHolder(claim) {
|
|
1329
|
+
const parts = [`pid ${claim.pid}`, `state dir ${claim.state_dir}`];
|
|
1330
|
+
if (claim.api_port !== undefined)
|
|
1331
|
+
parts.push(`API port ${claim.api_port}`);
|
|
1332
|
+
if (claim.broker_name)
|
|
1333
|
+
parts.push(`broker name ${claim.broker_name}`);
|
|
1334
|
+
if (claim.status === 'reserved')
|
|
1335
|
+
parts.push('starting up');
|
|
1336
|
+
return parts.join(', ');
|
|
1337
|
+
}
|
|
1338
|
+
//# sourceMappingURL=node-claim.js.map
|