@gadicc/loom 2.4.0 → 2.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,543 @@
1
+ import { spawnSync } from "node:child_process";
2
+ /**
3
+ * Machine-wide CPU slots (`loom-slots-v1`, docs/machine-slots.md): one
4
+ * per-user pool of CPUs shared by every Loom consumer on the machine.
5
+ * Holding a slot is holding an exclusive `flock` on `<dir>/cpu-<id>`; only the
6
+ * wrapper holds it, and the job runs pinned to its grant.
7
+ */
8
+ import { randomBytes } from "node:crypto";
9
+ import {
10
+ closeSync,
11
+ ftruncateSync,
12
+ readdirSync,
13
+ readSync,
14
+ renameSync,
15
+ statSync,
16
+ unlinkSync,
17
+ writeSync,
18
+ } from "node:fs";
19
+ import { join } from "node:path";
20
+ import process from "node:process";
21
+ import {
22
+ ancestorKeys,
23
+ openPoolFile,
24
+ readProcessStat,
25
+ } from "./machineSlotsHost.ts";
26
+ import { parseCpuList } from "./testResourcePolicy.ts";
27
+
28
+ /** Spec identifier recorded in labels and shown by `loom slots status`. */
29
+ export const SLOTS_SPEC = "loom-slots-v1";
30
+
31
+ /** Exit code `flock(1)` is told to use when the lock is already held. */
32
+ const FLOCK_BUSY = 75;
33
+
34
+ /** A ticket older than this that nobody holds belongs to a dead waiter. */
35
+ const TICKET_GRACE_MS = 2_000;
36
+
37
+ /** Lock operations the grant loop needs; real ones use `flock(1)`. */
38
+ export interface SlotLocks {
39
+ /** Tries to take a slot without waiting; true if it is now held. */
40
+ tryTake(cpu: number): Promise<boolean>;
41
+ /** Gives a held slot back. */
42
+ release(cpu: number): Promise<void>;
43
+ }
44
+
45
+ /** A job's request: at least `min` CPUs, at most `max`. */
46
+ export interface SlotRequest {
47
+ min: number;
48
+ max: number;
49
+ }
50
+
51
+ /**
52
+ * Parses and clamps `--min`/`--max` against the pool and any known quota.
53
+ * The floor defaults to 2 and the ceiling to `defaultMax` (never below the
54
+ * floor); both are clamped to what the pool and quota allow.
55
+ */
56
+ export function resolveSlotRequest(
57
+ pool: number,
58
+ min: string | undefined,
59
+ max: string | undefined,
60
+ options: { defaultMax?: number; quota?: number } = {},
61
+ ): SlotRequest {
62
+ const parse = (value: string | undefined, name: string) => {
63
+ if (value === undefined) return undefined;
64
+ if (!/^\d+$/.test(value) || Number(value) < 1)
65
+ throw new Error(`--${name} must be a positive whole number`);
66
+ return Number(value);
67
+ };
68
+ const limit = Math.min(pool, options.quota ?? pool);
69
+ if (limit < 1) throw new Error("The slot pool is empty");
70
+ const requestedMax = parse(max, "max");
71
+ // Without --min, the default floor never exceeds a given --max.
72
+ const floor = Math.min(
73
+ parse(min, "min") ?? Math.min(2, requestedMax ?? 2),
74
+ limit,
75
+ );
76
+ if (requestedMax !== undefined && requestedMax < floor)
77
+ throw new Error("--max must not be below --min");
78
+ const ceiling = Math.min(
79
+ requestedMax ?? Math.max(floor, options.defaultMax ?? Math.ceil(pool / 2)),
80
+ limit,
81
+ );
82
+ return { min: floor, max: ceiling };
83
+ }
84
+
85
+ /**
86
+ * Orders the pool for scanning. With hybrid core types it interleaves them in
87
+ * proportion (one performance core to about three efficiency cores here), so
88
+ * each grant gets its share of fast cores; otherwise ascending CPU id. The
89
+ * order never affects which CPUs are shared, only which a job gets first.
90
+ */
91
+ export function scanOrder(
92
+ pool: readonly number[],
93
+ coreTypes: readonly string[],
94
+ ): number[] {
95
+ const sorted = [...pool].sort((a, b) => a - b);
96
+ let groups: number[][];
97
+ try {
98
+ const inPool = new Set(sorted);
99
+ groups = coreTypes
100
+ .map((text) => parseCpuList(text).filter((cpu) => inPool.has(cpu)))
101
+ .filter((group) => group.length > 0);
102
+ } catch {
103
+ return sorted;
104
+ }
105
+ const covered = new Set(groups.flat());
106
+ if (groups.length < 2 || covered.size !== sorted.length) return sorted;
107
+ const taken = groups.map(() => 0);
108
+ const order: number[] = [];
109
+ while (order.length < sorted.length) {
110
+ // Next from the type furthest behind its share; ties go to the first type.
111
+ let best = -1;
112
+ for (let index = 0; index < groups.length; index++) {
113
+ if (taken[index] >= groups[index].length) continue;
114
+ if (
115
+ best < 0 ||
116
+ taken[index] / groups[index].length < taken[best] / groups[best].length
117
+ )
118
+ best = index;
119
+ }
120
+ order.push(groups[best][taken[best]++]);
121
+ }
122
+ return order;
123
+ }
124
+
125
+ /**
126
+ * One pass over the pool in scan order, taking free slots until `max` are
127
+ * held. Fewer than `min` gives every slot back at once, so a job never holds
128
+ * slots while it waits; enough is topped up by one more pass for slots freed
129
+ * meanwhile.
130
+ */
131
+ export async function scanForGrant(
132
+ pool: readonly number[],
133
+ request: SlotRequest,
134
+ locks: SlotLocks,
135
+ ): Promise<number[] | null> {
136
+ const held: number[] = [];
137
+ const pass = async () => {
138
+ for (const cpu of pool) {
139
+ if (held.length >= request.max) return;
140
+ if (!held.includes(cpu) && (await locks.tryTake(cpu))) held.push(cpu);
141
+ }
142
+ };
143
+ await pass();
144
+ if (held.length < request.min) {
145
+ for (const cpu of held) await locks.release(cpu);
146
+ return null;
147
+ }
148
+ if (held.length < request.max) await pass();
149
+ return held;
150
+ }
151
+
152
+ /**
153
+ * Takes a non-blocking exclusive `flock` on an open descriptor through
154
+ * util-linux `flock(1)`, which receives it as fd 3. The lock belongs to the
155
+ * open file description, so it stays with `fd` after `flock(1)` exits.
156
+ */
157
+ export function flockDescriptor(
158
+ fd: number,
159
+ flockPath = "flock",
160
+ ): "taken" | "busy" | "interrupted" | "unavailable" {
161
+ const result = spawnSync(
162
+ flockPath,
163
+ [
164
+ "--nonblock",
165
+ "--exclusive",
166
+ "--conflict-exit-code",
167
+ String(FLOCK_BUSY),
168
+ "3",
169
+ ],
170
+ { stdio: ["ignore", "ignore", "ignore", fd] },
171
+ );
172
+ if (result.error) return "unavailable";
173
+ // A terminal's Ctrl-C reaches flock(1) too; that is an interrupt, not a fault.
174
+ if (result.signal) return "interrupted";
175
+ if (result.status === 0) return "taken";
176
+ if (result.status === FLOCK_BUSY) return "busy";
177
+ return "unavailable";
178
+ }
179
+
180
+ /** What a slot or ticket file says about its holder while it is locked. */
181
+ export interface SlotLabel {
182
+ spec: typeof SLOTS_SPEC;
183
+ kind: "slot" | "ticket";
184
+ lease: string;
185
+ pid: number;
186
+ /** The holder's start time from `/proc/<pid>/stat`, so a reused pid is not taken for it. */
187
+ pidStart: string | null;
188
+ label: string;
189
+ startedAt: string;
190
+ request?: SlotRequest;
191
+ }
192
+
193
+ /** Writes a label over a held file's contents. */
194
+ export function writeLabel(fd: number, label: SlotLabel): void {
195
+ ftruncateSync(fd, 0);
196
+ writeSync(fd, JSON.stringify(label), 0);
197
+ }
198
+
199
+ /** Reads a label from an open descriptor; anything unexpected is null. */
200
+ export function readLabel(fd: number): SlotLabel | null {
201
+ const buffer = Buffer.alloc(4096);
202
+ const length = readSync(fd, buffer, 0, buffer.length, 0);
203
+ try {
204
+ const value = JSON.parse(buffer.subarray(0, length).toString("utf8"));
205
+ if (
206
+ value?.spec === SLOTS_SPEC &&
207
+ (value.kind === "slot" || value.kind === "ticket") &&
208
+ typeof value.lease === "string" &&
209
+ Number.isSafeInteger(value.pid) &&
210
+ typeof value.label === "string" &&
211
+ typeof value.startedAt === "string"
212
+ )
213
+ return value as SlotLabel;
214
+ } catch {
215
+ /* An empty or partly written file has no label. */
216
+ }
217
+ return null;
218
+ }
219
+
220
+ /** Whether a label's holder is still the process that wrote it. */
221
+ export function holderAlive(label: SlotLabel): boolean {
222
+ const stat = readProcessStat(label.pid);
223
+ return Boolean(
224
+ stat && (label.pidStart === null || stat.start === label.pidStart),
225
+ );
226
+ }
227
+
228
+ /** A new label for this process. */
229
+ export function ownLabel(
230
+ kind: SlotLabel["kind"],
231
+ lease: string,
232
+ text: string,
233
+ request?: SlotRequest,
234
+ ): SlotLabel {
235
+ return {
236
+ spec: SLOTS_SPEC,
237
+ kind,
238
+ lease,
239
+ pid: process.pid,
240
+ pidStart: readProcessStat(process.pid)?.start ?? null,
241
+ label: text,
242
+ startedAt: new Date().toISOString(),
243
+ ...(request ? { request } : {}),
244
+ };
245
+ }
246
+
247
+ /** A fresh random lease id. */
248
+ export function newLease(): string {
249
+ return randomBytes(9).toString("base64url");
250
+ }
251
+
252
+ /** Raised when a signal ends a lock attempt: the caller is being interrupted. */
253
+ export class SlotsInterrupted extends Error {
254
+ constructor() {
255
+ super("interrupted");
256
+ }
257
+ }
258
+
259
+ /** Slot locks on `<dir>/cpu-<id>` files; this process alone holds them. */
260
+ export class FlockSlotLocks implements SlotLocks {
261
+ readonly #dir: string;
262
+ readonly #flock: string;
263
+ readonly #held = new Map<number, number>();
264
+
265
+ constructor(dir: string, flockPath = "flock") {
266
+ this.#dir = dir;
267
+ this.#flock = flockPath;
268
+ }
269
+
270
+ tryTake(cpu: number): Promise<boolean> {
271
+ if (this.#held.has(cpu)) return Promise.resolve(true);
272
+ try {
273
+ const fd = openPoolFile(join(this.#dir, `cpu-${cpu}`)) as number;
274
+ const outcome = flockDescriptor(fd, this.#flock);
275
+ if (outcome === "taken") {
276
+ this.#held.set(cpu, fd);
277
+ return Promise.resolve(true);
278
+ }
279
+ closeSync(fd);
280
+ if (outcome === "unavailable")
281
+ return Promise.reject(new Error("flock(1) is unavailable"));
282
+ if (outcome === "interrupted")
283
+ return Promise.reject(new SlotsInterrupted());
284
+ return Promise.resolve(false);
285
+ } catch (error) {
286
+ return Promise.reject(error);
287
+ }
288
+ }
289
+
290
+ release(cpu: number): Promise<void> {
291
+ const fd = this.#held.get(cpu);
292
+ if (fd !== undefined) {
293
+ this.#held.delete(cpu);
294
+ closeSync(fd);
295
+ }
296
+ return Promise.resolve();
297
+ }
298
+
299
+ /** Labels every held slot. */
300
+ label(label: SlotLabel): void {
301
+ for (const fd of this.#held.values()) writeLabel(fd, label);
302
+ }
303
+
304
+ /** Releases every held slot. */
305
+ releaseAll(): void {
306
+ for (const fd of this.#held.values()) closeSync(fd);
307
+ this.#held.clear();
308
+ }
309
+ }
310
+
311
+ /** One pool file's state: free, or held with the label its holder wrote. */
312
+ export type FileState =
313
+ | { state: "missing" }
314
+ | { state: "free" }
315
+ | { state: "held"; label: SlotLabel | null };
316
+
317
+ /**
318
+ * Probes a pool file without changing who holds it: a free file is locked for
319
+ * a moment and released (a concurrent taker may then see it busy and skip it);
320
+ * a held one has its label read. `whenFree` runs while the probe holds a free
321
+ * file, so a dead ticket can be removed under its own lock.
322
+ */
323
+ export function probeFile(
324
+ path: string,
325
+ flockPath = "flock",
326
+ whenFree?: () => void,
327
+ ): FileState {
328
+ const fd = openPoolFile(path, { existingOnly: true });
329
+ if (fd === null) return { state: "missing" };
330
+ try {
331
+ const outcome = flockDescriptor(fd, flockPath);
332
+ if (outcome === "unavailable") throw new Error("flock(1) is unavailable");
333
+ if (outcome === "interrupted") throw new SlotsInterrupted();
334
+ if (outcome === "taken") {
335
+ whenFree?.();
336
+ return { state: "free" };
337
+ }
338
+ return { state: "held", label: readLabel(fd) };
339
+ } finally {
340
+ closeSync(fd);
341
+ }
342
+ }
343
+
344
+ /** A waiter's place in line, held for as long as it waits. */
345
+ export interface Ticket {
346
+ name: string;
347
+ fd: number;
348
+ }
349
+
350
+ /**
351
+ * Joins the line: creates a ticket under a `pending-` name, locks and labels
352
+ * it, then renames it into the line, so no prober ever sees it unlocked.
353
+ */
354
+ export function takeTicket(
355
+ dir: string,
356
+ label: SlotLabel,
357
+ flockPath = "flock",
358
+ ): Ticket {
359
+ const stem = `${String(Date.now()).padStart(15, "0")}-${process.pid}-${randomBytes(4).toString("hex")}`;
360
+ const pending = join(dir, `pending-${stem}`);
361
+ const fd = openPoolFile(pending, { exclusive: true }) as number;
362
+ const outcome = flockDescriptor(fd, flockPath);
363
+ if (outcome !== "taken") {
364
+ unlinkSync(pending);
365
+ closeSync(fd);
366
+ if (outcome === "interrupted") throw new SlotsInterrupted();
367
+ throw new Error("Could not lock a new ticket");
368
+ }
369
+ writeLabel(fd, label);
370
+ const name = `ticket-${stem}`;
371
+ renameSync(pending, join(dir, name));
372
+ return { name, fd };
373
+ }
374
+
375
+ /** Leaves the line. */
376
+ export function dropTicket(dir: string, ticket: Ticket): void {
377
+ try {
378
+ unlinkSync(join(dir, ticket.name));
379
+ } catch {
380
+ /* Already removed by someone who found it unheld; that cannot happen while held. */
381
+ }
382
+ closeSync(ticket.fd);
383
+ }
384
+
385
+ /** A live waiter, for the head decision and for `status`. */
386
+ export interface Waiter {
387
+ name: string;
388
+ label: SlotLabel | null;
389
+ stopped: boolean;
390
+ }
391
+
392
+ /**
393
+ * Lists live tickets in arrival order. A ticket nobody holds, once past its
394
+ * short creation grace, belongs to a dead waiter and is removed under its own
395
+ * lock. Stopped holders (Ctrl-Z) are marked so they do not block the line.
396
+ */
397
+ export function liveWaiters(
398
+ dir: string,
399
+ own: string | null,
400
+ flockPath = "flock",
401
+ now = Date.now(),
402
+ ): Waiter[] {
403
+ const waiters: Waiter[] = [];
404
+ const all = readdirSync(dir);
405
+ // A pending ticket left by a waiter that died before renaming it.
406
+ for (const name of all.filter((one) => one.startsWith("pending-"))) {
407
+ const path = join(dir, name);
408
+ try {
409
+ if (now - statSync(path).mtimeMs > TICKET_GRACE_MS)
410
+ probeFile(path, flockPath, () => unlinkSync(path));
411
+ } catch {
412
+ /* Gone already, or not ours to judge. */
413
+ }
414
+ }
415
+ const names = all.filter((name) => name.startsWith("ticket-")).sort();
416
+ for (const name of names) {
417
+ const path = join(dir, name);
418
+ if (name === own) {
419
+ waiters.push({ name, label: null, stopped: false });
420
+ continue;
421
+ }
422
+ let old = false;
423
+ try {
424
+ old = now - statSync(path).mtimeMs > TICKET_GRACE_MS;
425
+ } catch {
426
+ continue;
427
+ }
428
+ let state: FileState;
429
+ try {
430
+ state = probeFile(path, flockPath, () => {
431
+ if (old) unlinkSync(path);
432
+ });
433
+ } catch {
434
+ continue;
435
+ }
436
+ if (state.state !== "held") continue;
437
+ const stat = state.label ? readProcessStat(state.label.pid) : null;
438
+ waiters.push({
439
+ name,
440
+ label: state.label,
441
+ stopped: Boolean(stat && (stat.state === "T" || stat.state === "t")),
442
+ });
443
+ }
444
+ return waiters;
445
+ }
446
+
447
+ /** Whether `own` is first in line among waiters that are not stopped. */
448
+ export function isHead(waiters: readonly Waiter[], own: string): boolean {
449
+ return waiters.find((waiter) => !waiter.stopped)?.name === own;
450
+ }
451
+
452
+ /**
453
+ * Verifies an inherited grant for nesting: every CPU it names is locked, by a
454
+ * live holder whose label carries the same lease, and this process's own
455
+ * affinity lies within it. Returns the grant's CPUs, or null when the
456
+ * variables are absent, stale or forged.
457
+ */
458
+ export function verifyLease(
459
+ dir: string,
460
+ env: Readonly<Record<string, string | undefined>>,
461
+ inherited: readonly number[],
462
+ flockPath = "flock",
463
+ ): number[] | null {
464
+ const lease = env.LOOM_SLOTS_LEASE;
465
+ const listed = env.LOOM_SLOT_CPUS;
466
+ if (!lease || !listed) return null;
467
+ let cpus: number[];
468
+ try {
469
+ cpus = parseCpuList(listed);
470
+ } catch {
471
+ return null;
472
+ }
473
+ if (cpus.length === 0 || !inherited.every((cpu) => cpus.includes(cpu)))
474
+ return null;
475
+ for (const cpu of cpus) {
476
+ let state: FileState;
477
+ try {
478
+ state = probeFile(join(dir, `cpu-${cpu}`), flockPath);
479
+ } catch {
480
+ return null;
481
+ }
482
+ if (
483
+ state.state !== "held" ||
484
+ state.label?.lease !== lease ||
485
+ !holderAlive(state.label)
486
+ )
487
+ return null;
488
+ }
489
+ return cpus;
490
+ }
491
+
492
+ /** `status` view of one CPU in the pool. */
493
+ export interface SlotStatus {
494
+ cpu: number;
495
+ state: "free" | "held" | "unknown";
496
+ label: SlotLabel | null;
497
+ }
498
+
499
+ /** Reads every pool CPU's state for `loom slots status`. */
500
+ export function readSlots(
501
+ dir: string,
502
+ pool: readonly number[],
503
+ flockPath = "flock",
504
+ ): SlotStatus[] {
505
+ return pool.map((cpu): SlotStatus => {
506
+ let state: FileState;
507
+ try {
508
+ state = probeFile(join(dir, `cpu-${cpu}`), flockPath);
509
+ } catch {
510
+ return { cpu, state: "unknown", label: null };
511
+ }
512
+ return state.state === "held"
513
+ ? { cpu, state: "held", label: state.label }
514
+ : { cpu, state: "free", label: null };
515
+ });
516
+ }
517
+
518
+ /**
519
+ * Finds a grant held by an ancestor of this process among the pool's slots,
520
+ * by matching each holder's pid and start time against this process's
521
+ * `/proc` parent chain. This recognises nesting even when an intermediary
522
+ * scrubbed the grant variables; without it an inner run would queue for CPUs
523
+ * its own ancestor holds and wait forever.
524
+ */
525
+ export function ancestorGrant(
526
+ dir: string,
527
+ pool: readonly number[],
528
+ flockPath = "flock",
529
+ ancestors: Set<string> = ancestorKeys(),
530
+ ): { lease: string; cpus: number[] } | null {
531
+ const byLease = new Map<string, number[]>();
532
+ for (const slot of readSlots(dir, pool, flockPath)) {
533
+ const label = slot.label;
534
+ if (
535
+ slot.state === "held" &&
536
+ label?.kind === "slot" &&
537
+ ancestors.has(`${label.pid}:${label.pidStart}`)
538
+ )
539
+ byLease.set(label.lease, [...(byLease.get(label.lease) ?? []), slot.cpu]);
540
+ }
541
+ const [first] = byLease;
542
+ return first ? { lease: first[0], cpus: first[1] } : null;
543
+ }