@smoothbricks/nx-plugin 0.7.3 → 0.7.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -38,8 +38,17 @@ import type { HangDiagnostics } from './hang-diagnostics.js';
38
38
  import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
39
39
  import type { CpuWaitRow } from './lease-wait.js';
40
40
  import type { LockWait } from './lock-wait.js';
41
- import { MBX_GRANT_FD, mbxIdleGated, mbxLeaf, mbxPool, withCountedReservation, withMbxLeaf } from './mbx-pool.js';
41
+ import {
42
+ askedCores,
43
+ MBX_GRANT_FD,
44
+ mbxIdleGated,
45
+ mbxLeaf,
46
+ mbxPool,
47
+ withCountedReservation,
48
+ withMbxLeaf,
49
+ } from './mbx-pool.js';
42
50
  import { MBX_QUEUE_POLL_MS, type MbxQueuedRequest, snapshotMbxQueue } from './mbx-queue.js';
51
+ import { RESERVATIONS_ENV, type Reservation, reserveCores } from './mbx-reservations.js';
43
52
  import { createNoSpaceWatch } from './no-space.js';
44
53
  import {
45
54
  COWSHED_EXECUTABLE_ENV,
@@ -568,8 +577,11 @@ export async function runBoundedExec(
568
577
  sized = sizeRunner(demand, requested, demand.want);
569
578
  }
570
579
  // The run's sub-pool and grant are given back exactly once, however the run ends. The sub-pool closes first, so no
571
- // descendant still draws on tokens the grant has returned.
580
+ // descendant still draws on tokens the grant has returned. A reserved leaf's mbx grant covers the leaf's conclusion,
581
+ // its subtree's retirement included (11_shell.md), so it goes back after `releaseLease`.
572
582
  let pool: CpuPool | null = null;
583
+ let reservation: Reservation | null = null;
584
+ const releaseReservation = (): void => reservation?.release();
573
585
  let cpuReleased: Promise<void> | null = null;
574
586
  const releaseCpu = (): Promise<void> => {
575
587
  cpuReleased ??= (async () => {
@@ -596,8 +608,9 @@ export async function runBoundedExec(
596
608
  let unconfirmedAdmission: string | null = null;
597
609
  /** When the last successful snapshot completed (`Date.now()`): credit runs to it and no further. */
598
610
  let confirmedAdmissionAt: number | null = null;
611
+ // A reserved leaf's own request on its run's endpoint is its admission wait; a Cargo leaf's comes from mbx's report.
599
612
  const queueWait = (): AdmissionQueueWait | null =>
600
- mbxAdmitted || lastQueueRequests.length === 0
613
+ mbxAdmitted || (lastQueueRequests.length === 0 && reservation === null)
601
614
  ? null
602
615
  : {
603
616
  waitedMs: admissionWaitMs(admissionWait, performance.now()),
@@ -679,6 +692,14 @@ export async function runBoundedExec(
679
692
  if (parentRefusal !== null) {
680
693
  return refuse(`cowshed: cpu-pool parent pool refused: ${parentRefusal}; refusing to run the command\n`);
681
694
  }
695
+ // A reserved leaf takes its cores from its Nx run's mbx reservation endpoint (mbx-reservations.ts). A run that serves
696
+ // none said why in its own output; the leaf does not run beside the pool.
697
+ const reserved = leaf?.kind === 'reserve' ? { leaf, endpoint: process.env[RESERVATIONS_ENV] ?? '' } : null;
698
+ if (reserved !== null && reserved.endpoint === '') {
699
+ return refuse(
700
+ `cowshed: a reserved leaf takes its cores from its Nx run's mbx reservation endpoint, and ${RESERVATIONS_ENV} names none (the run's preTasksExecution serves it); refusing to run the command\n`,
701
+ );
702
+ }
682
703
 
683
704
  // A granted run serves a sub-pool of its own grant, so its descendants draw from that and never from above it.
684
705
  // A pool that cannot be set up refuses the run: no granted command runs beside its grant.
@@ -730,7 +751,8 @@ export async function runBoundedExec(
730
751
  : withMbxLeaf(unadmitted, leaf);
731
752
  // Only leaves known to be scheduled can exclude zero-hold time. Others keep lifecycle hang protection.
732
753
  const queued = mbxIdleGated(leaf, sized.command, { ...process.env, ...env });
733
- // fd 4 is unused without a subtree admission, so the grant takes fd 5 either way.
754
+ // fd 4 is unused without a subtree admission, so mbx's descriptor takes fd 5 either way: a Cargo leaf's admission
755
+ // stream, or a reserved leaf's ask for cores and their count (mbx-pool.ts, `MBX_GRANT_FD`).
734
756
  const grantSlots: IOType[] = leaf !== null ? ['ignore', 'pipe'] : [];
735
757
  const rootGrantSlot: IOType[] = leaf !== null ? ['pipe'] : [];
736
758
  const root = owner?.rootArgv(shellCommand);
@@ -740,7 +762,7 @@ export async function runBoundedExec(
740
762
  resultWriter = results?.ok ? results.writer : null;
741
763
  try {
742
764
  // fd 3 is the runner lifecycle's duplex socket when the command is hooked; the subtree admission takes fd 4, which
743
- // cowshed closes fd 4 before exec; fd 5 carries mbx's admission/release stream, including Cargo descendants.
765
+ // cowshed closes fd 4 before exec; fd 5 is a Cargo leaf's mbx admission stream or a reserved leaf's ask and grant.
744
766
  child =
745
767
  root === undefined
746
768
  ? spawn(shellCommand, [], {
@@ -882,7 +904,7 @@ export async function runBoundedExec(
882
904
  if (state.settled || state.expiry !== null) return;
883
905
  const scheduled = earliestBound(checkingIdle ? { ...bounds, idle: null } : bounds);
884
906
  if (scheduled?.kind !== kind || scheduled.at !== at) return;
885
- if (kind === 'idle' && leaf !== null) {
907
+ if (kind === 'idle' && leaf?.kind === 'cargo') {
886
908
  if (checkingIdle) return;
887
909
  checkingIdle = true;
888
910
  armBoundTimer();
@@ -937,8 +959,6 @@ export async function runBoundedExec(
937
959
  };
938
960
  const refreshQueue = async (fresh: boolean): Promise<void> => {
939
961
  if (fresh && queueSnapshot !== null) await queueSnapshot;
940
- // A leaf that counts its units before reserving waits in no reservation queue while its fixture builds.
941
- if (counted && !fixtureDone) return;
942
962
  if (state.settled || state.expiry !== null || queueAbort.signal.aborted || child.pid === undefined) return;
943
963
  const pid = child.pid;
944
964
  queueSnapshot ??= (async () => {
@@ -1079,13 +1099,14 @@ export async function runBoundedExec(
1079
1099
  clearTimeout(boundTimer);
1080
1100
  // Preserve explicit live membership before cancellation removes the queued process. A failed bounded observation
1081
1101
  // leaves the admission unconfirmed, never a manufactured queue or admission.
1082
- signalCleanup = (leaf === null ? Promise.resolve() : refreshQueue(false))
1102
+ signalCleanup = (leaf?.kind !== 'cargo' ? Promise.resolve() : refreshQueue(false))
1083
1103
  .finally(() => {
1084
1104
  stopQueueObservation();
1085
1105
  return killChildTree();
1086
1106
  })
1087
1107
  .finally(releaseCpu)
1088
1108
  .finally(() => releaseLease(true))
1109
+ .finally(releaseReservation)
1089
1110
  .finally(async () =>
1090
1111
  recordRun(
1091
1112
  signalToExitCode(signal),
@@ -1257,46 +1278,87 @@ export async function runBoundedExec(
1257
1278
  fixtureFirst: counted && lifecycle !== null,
1258
1279
  queued,
1259
1280
  });
1260
- // Read every byte after spawn accounting: a chunk can contain several grants and releases. EOF is neither.
1261
1281
  if (leaf !== null) {
1262
1282
  const channel = child.stdio.at(MBX_GRANT_FD);
1263
- if (!(channel instanceof Readable)) throw new Error('an mbx leaf requires its grant descriptor');
1264
- let said = false;
1283
+ if (!(channel instanceof Duplex)) throw new Error('an mbx leaf requires its mbx descriptor');
1265
1284
  const failGrant = (detail: string): void => {
1266
1285
  if (lifecycleFailure !== null || state.expiry !== null || state.settled) return;
1267
1286
  lifecycleFailure ??= `mbx grant descriptor: ${detail}`;
1268
1287
  appendStderr(`bounded-exec: ${lifecycleFailure}\n`);
1269
1288
  termination = terminate();
1270
1289
  };
1271
- channel.on('data', (chunk: Buffer) => {
1272
- for (const byte of chunk) {
1273
- if (byte === 103) {
1274
- if (!said) {
1275
- mbxAdmitted = true;
1276
- said = true;
1277
- tell(
1278
- leaf.kind === 'reserve'
1279
- ? `cowshed: mbx reserve granted elapsed=${Date.now() - startedAt}ms priority=${leaf.priority}\n`
1280
- : `cowshed: mbx cargo granted elapsed=${Date.now() - startedAt}ms\n`,
1281
- );
1290
+ channel.on('error', (error) => failGrant(error.message));
1291
+ if (reserved !== null) {
1292
+ // The shell asks for its cores in one line and reads the count granted (mbx-pool.ts, `reservation`). A refusal
1293
+ // ends the descriptor unanswered: the shell's read fails, and nothing runs.
1294
+ const { leaf: reservedLeaf, endpoint } = reserved;
1295
+ let asked = '';
1296
+ channel.setEncoding('utf8');
1297
+ channel.on('data', (text: string) => {
1298
+ if (reservation !== null) {
1299
+ failGrant(`the command asked for cores a second time: ${JSON.stringify(text)}`);
1300
+ return;
1301
+ }
1302
+ asked += text;
1303
+ const end = asked.indexOf('\n');
1304
+ if (end === -1) return;
1305
+ const line = asked.slice(0, end);
1306
+ const cpus = askedCores(line, reservedLeaf.demand.want);
1307
+ if (cpus === null) {
1308
+ failGrant(`the command asked for ${JSON.stringify(line)} cores, not 1 to ${reservedLeaf.demand.want}`);
1309
+ channel.end();
1310
+ return;
1311
+ }
1312
+ admissionWait = stepAdmissionWait(admissionWait, { kind: 'snapshot', at: performance.now(), waiting: true });
1313
+ tell(`cowshed: mbx reserve wait start cpus=${cpus} priority=${reservedLeaf.priority}\n`);
1314
+ const held = reserveCores(endpoint, cpus, reservedLeaf.priority);
1315
+ reservation = held;
1316
+ void held.granted.then((granted) => {
1317
+ if (!granted.ok) {
1318
+ failGrant(`mbx did not grant the reservation: ${granted.error}`);
1319
+ channel.end();
1320
+ return;
1282
1321
  }
1322
+ // A run that ended while it waited gives the grant back with its CPU (`releaseCpu`).
1323
+ if (state.settled || state.expiry !== null) return;
1324
+ mbxAdmitted = true;
1325
+ tell(`cowshed: mbx reserve granted elapsed=${Date.now() - startedAt}ms priority=${reservedLeaf.priority}\n`);
1326
+ channel.end(`${granted.cpus}\n`);
1283
1327
  stepTo({ kind: 'granted', at: Date.now(), testing: isTesting(), fixture: !fixtureDone });
1284
- } else if (byte === 114) {
1285
- stepTo({ kind: 'released', at: Date.now() });
1286
- } else {
1287
- failGrant(`unexpected byte ${byte}`);
1288
- return;
1328
+ void granted.lost.then((why) => {
1329
+ if (state.settled || cpuReleased !== null) return;
1330
+ appendStderr(
1331
+ `cowshed: mbx reservation lost while the command ran (${why}); it runs on unheld until it ends\n`,
1332
+ );
1333
+ });
1334
+ });
1335
+ });
1336
+ } else {
1337
+ // Read every byte after spawn accounting: a chunk can contain several grants and releases. EOF is neither.
1338
+ let said = false;
1339
+ channel.on('data', (chunk: Buffer) => {
1340
+ for (const byte of chunk) {
1341
+ if (byte === 103) {
1342
+ if (!said) {
1343
+ mbxAdmitted = true;
1344
+ said = true;
1345
+ tell(`cowshed: mbx cargo granted elapsed=${Date.now() - startedAt}ms\n`);
1346
+ }
1347
+ stepTo({ kind: 'granted', at: Date.now(), testing: isTesting(), fixture: !fixtureDone });
1348
+ } else if (byte === 114) {
1349
+ stepTo({ kind: 'released', at: Date.now() });
1350
+ } else {
1351
+ failGrant(`unexpected byte ${byte}`);
1352
+ return;
1353
+ }
1289
1354
  }
1290
- }
1291
- });
1292
- channel.on('error', (error) => failGrant(error.message));
1293
- }
1294
-
1295
- if (leaf !== null) {
1296
- queueTimer = setInterval(() => {
1355
+ });
1356
+ // Only mbx's activity report says whether a Cargo leaf's compiler or test waits in the pool.
1357
+ queueTimer = setInterval(() => {
1358
+ void refreshQueue(false);
1359
+ }, MBX_QUEUE_POLL_MS);
1297
1360
  void refreshQueue(false);
1298
- }, MBX_QUEUE_POLL_MS);
1299
- void refreshQueue(false);
1361
+ }
1300
1362
  }
1301
1363
 
1302
1364
  // A grant's connection is a ledger entry, not the command's owner: the lending parent's exit ends it, and what runs
@@ -1356,6 +1418,7 @@ export async function runBoundedExec(
1356
1418
  }
1357
1419
  }
1358
1420
  await releaseLease(code !== 0 || state.expiry !== null || state.poolCloseError !== null || lifecycleFailure !== null);
1421
+ releaseReservation();
1359
1422
 
1360
1423
  // Broken resource ownership, a lost native lifecycle, or leftovers that could not be retired fail even if a shell
1361
1424
  // branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
@@ -2,6 +2,7 @@ import { shellTokens } from '../../runtime-input-policy.js';
2
2
  import {
3
3
  CPU_TOKENS_ENV,
4
4
  type Demand,
5
+ GRANTED_CPUS_ENV,
5
6
  grantedSizing,
6
7
  invocationSpan,
7
8
  type ReservedDemand,
@@ -21,10 +22,11 @@ export interface MbxPool {
21
22
 
22
23
  /**
23
24
  * The pool the environment names, or `null` where the project has no mbx and cowshed's own pools apply unchanged, or
24
- * where this run is already inside an mbx reservation: `mbx reserve` hands its command `MBX_SCHED_DISABLE=1`, and nested
25
- * non-Cargo work shares the enclosing grant instead of reserving twice (specs/cowshed/05_gateway.md, "Parallelism
26
- * inside a job"). A second reservation would wait in the pool while its parent holds a grant, so under a full pool the
27
- * nested leaf queued behind the very work it belongs to.
25
+ * where this run is already inside an mbx reservation: whatever holds one hands its work `MBX_SCHED_DISABLE=1` (a
26
+ * reserved leaf's shell, `mbx reserve`, a held cowshed job), and nested non-Cargo work shares the enclosing grant
27
+ * instead of reserving twice (specs/cowshed/05_gateway.md, "Parallelism inside a job"). A second reservation would
28
+ * wait in the pool while its parent holds a grant, so under a full pool the nested leaf queued behind the very work it
29
+ * belongs to.
28
30
  */
29
31
  export function mbxPool(environment: Readonly<NodeJS.ProcessEnv>): MbxPool | null {
30
32
  if (environment.MBX_SCHEDULER !== '1' || !environment.MBX_CACHE_DIR || environment.MBX_SCHED_DISABLE === '1') {
@@ -35,9 +37,10 @@ export function mbxPool(environment: Readonly<NodeJS.ProcessEnv>): MbxPool | nul
35
37
 
36
38
  /**
37
39
  * How a leaf enters the pool once. A Cargo leaf that builds runs through mbx, which admits its compilers and supported
38
- * test work itself, so wrapping it would charge it twice. Any other leaf reserves for its whole run through
39
- * `mbx reserve --cpus <its runner's count>`, a maximum: mbx grants as many of those cores as are free once one is, and
40
- * the runner runs on the count it is granted (`sizeRunnerToGrant`). That includes a nextest run of test binaries
40
+ * test work itself, so wrapping it would charge it twice. Any other leaf reserves for its whole run, asking its Nx
41
+ * run's mbx reservation endpoint (mbx-reservations.ts) for at most its runner's count: mbx grants as many of those
42
+ * cores as are free once one is, and the runner runs on the count it is granted (`sizeRunnerToGrant`). That includes a
43
+ * nextest run of test binaries
41
44
  * already built (the inferred extracted-archive run, or one from `--archive-file`): it compiles nothing, and beneath a
42
45
  * reservation mbx schedules none of its test processes, so they run inside the one grant. Admitted one by one under
43
46
  * `MBX_SCHEDULER_TESTS=1`, each queued for a ticket of its own: in a downstream gate on 2026-10-30, codec tests of at
@@ -156,8 +159,9 @@ export function mbxIdleGated(leaf: MbxLeaf | null, command: string, environment:
156
159
  }
157
160
 
158
161
  /**
159
- * mbx writes one byte per admission (g) and permit drop (r). Cargo descendants inherit this descriptor through
160
- * MBX_GRANT_FD; reserve names it with --grant-fd and closes it on exec of the external command.
162
+ * The descriptor between a pooled leaf's shell and its executor. For a Cargo leaf mbx writes one byte on it per
163
+ * admission (g) and permit drop (r), every Cargo descendant inheriting it through MBX_GRANT_FD. A reserved leaf's shell
164
+ * writes the cores it asks for on it, one line, and reads back the count the pool granted (`reservation`).
161
165
  */
162
166
  export const MBX_GRANT_FD = 5;
163
167
 
@@ -185,11 +189,27 @@ export function withCountedReservation(fixture: string, command: string, leaf: R
185
189
  }
186
190
 
187
191
  /**
188
- * The reservation of `WANT_ENV` cores, set by the caller, that runs `command` once mbx grants it: the reservation sizes
189
- * the runner to what mbx granted (`grantedSizing`) and says so on stderr. `mbx` is the canonical environment's, whose
190
- * realpath cowshed's admission proved to be the host's pinned mbx; it writes the grant on this descriptor itself.
192
+ * The reservation of `WANT_ENV` cores, set by the caller, that runs `command` once mbx grants it. The shell asks its
193
+ * executor on `MBX_GRANT_FD`, one line naming the count; the executor asks its run's mbx reservation endpoint and
194
+ * answers the cores granted, one line, or ends the descriptor unanswered when mbx refused, so the read fails and
195
+ * nothing runs (executor.ts). The shell then closes the descriptor, which the command never inherits, gives the command
196
+ * what `mbx reserve` gives its own (`GRANTED_CPUS_ENV`, `MBX_SCHED_DISABLE=1` and an empty `MBX_SCHED_DIR`, so mbx work
197
+ * inside runs under the grant), sizes the runner to the grant (`grantedSizing`), says so on stderr and runs the command
198
+ * in the same shell: no process starts for the reservation.
191
199
  */
192
200
  function reservation(command: string, leaf: ReservedLeaf): string {
193
201
  const said = `printf 'cowshed: mbx reserve running cpus=%s asked=%s\\n' "$${CPU_TOKENS_ENV}" "$${WANT_ENV}" >&2; `;
194
- return `export ${WANT_ENV} && mbx reserve --grant-fd ${MBX_GRANT_FD} --cpus "$${WANT_ENV}" --priority ${leaf.priority} -- /bin/sh -c ${shellQuote(`${grantedSizing(leaf.demand)}${said}${command}`)}`;
202
+ const ask = `printf '%s\\n' "$${WANT_ENV}" >&${MBX_GRANT_FD} && IFS= read -r ${GRANTED_CPUS_ENV} <&${MBX_GRANT_FD} && exec ${MBX_GRANT_FD}>&-`;
203
+ const held = `export ${GRANTED_CPUS_ENV} MBX_SCHED_DISABLE=1 MBX_SCHED_DIR=`;
204
+ return `export ${WANT_ENV} && ${ask} && ${held} && eval ${shellQuote(`${grantedSizing(leaf.demand)}${said}${command}`)}`;
205
+ }
206
+
207
+ /**
208
+ * The cores a reserved leaf's shell asked for on `MBX_GRANT_FD` (`reservation`): a whole count from one to `most`, the
209
+ * most its runner may ask for; anything else is `null`.
210
+ */
211
+ export function askedCores(line: string, most: number): number | null {
212
+ if (!/^[1-9]\d{0,9}$/.test(line)) return null;
213
+ const cores = Number(line);
214
+ return cores <= most ? cores : null;
195
215
  }
@@ -0,0 +1,300 @@
1
+ import { type ChildProcessByStdio, spawn } from 'node:child_process';
2
+ import { rm } from 'node:fs/promises';
3
+ import { createConnection, type Socket } from 'node:net';
4
+ import { join } from 'node:path';
5
+ import type { Readable, Writable } from 'node:stream';
6
+ import { logger } from 'nx/src/devkit-exports.js';
7
+ import type { PostTasksExecutionContext, PreTasksExecutionContext } from 'nx/src/project-graph/plugins/public-api.js';
8
+ import typia, { type tags } from 'typia';
9
+ import { ownedSocketDirectory } from '../../testing.js';
10
+ import { CONTROL_MS } from './lease-wait.js';
11
+ import { mbxPool } from './mbx-pool.js';
12
+
13
+ /**
14
+ * mbx's reservation endpoint of one Nx run (specs/cowshed/05_gateway.md, "Parallelism inside a job"). A reserved leaf
15
+ * once ran under its own `mbx reserve`, two mbx processes per task whose dynamic loading alone kept the host's vnode
16
+ * table recycling. With mbx's pool, the run's `preTasksExecution` starts `mbx reservations <socket>` once and publishes
17
+ * the socket in `RESERVATIONS_ENV`, which every task inherits; each task's executor asks it for its leaf's cores over
18
+ * one connection per process (`reserveCores`), and the run's `postTasksExecution` closes the endpoint's standard input,
19
+ * which ends it. Admission is `mbx reserve`'s: the same pool, priority and order. A grant lasts until its leaf ends, or
20
+ * until its connection closes, so an executor that dies gives back what it held.
21
+ */
22
+ export const RESERVATIONS_ENV = 'MBX_RESERVATIONS_SOCKET';
23
+
24
+ const ENDPOINT_DIRECTORY = 'mbx-r-';
25
+ const ENDPOINT_SOCKET = 'r.sock';
26
+ /** What is kept of the endpoint's stderr to say why it ended: its last bytes. */
27
+ const SAID_BYTES = 4096;
28
+
29
+ /** A run's endpoint: the `mbx reservations` process, the directory its socket is in, and how it ended once it has. */
30
+ interface Endpoint {
31
+ readonly server: ChildProcessByStdio<Writable, Readable, Readable>;
32
+ readonly directory: string;
33
+ readonly ended: Promise<string>;
34
+ }
35
+
36
+ type Started = { ok: true; endpoint: Endpoint; socket: string } | { ok: false; error: string };
37
+
38
+ /** The endpoints this process serves, by Nx run id, until that run's `postTasksExecution`. */
39
+ const endpoints = new Map<string, Endpoint>();
40
+
41
+ /**
42
+ * Serve `context`'s run an mbx reservation endpoint and publish it in `env`, which every task inherits, when the
43
+ * environment names mbx's pool (`mbxPool`). An endpoint that does not start is said, and each reserved leaf of the run
44
+ * then refuses to run, naming `RESERVATIONS_ENV`; a leaf in no pool, or one mbx admits itself, runs as before.
45
+ */
46
+ export async function serveRunReservations(
47
+ context: Pick<PreTasksExecutionContext, 'id'>,
48
+ env: NodeJS.ProcessEnv = process.env,
49
+ ): Promise<void> {
50
+ if (mbxPool(env) === null) {
51
+ return;
52
+ }
53
+ const started = await startEndpoint(env);
54
+ if (!started.ok) {
55
+ logger.warn(
56
+ `bounded-exec: this run's mbx reservation endpoint did not start (${started.error}); its reserved leaves refuse to run`,
57
+ );
58
+ return;
59
+ }
60
+ endpoints.set(context.id, started.endpoint);
61
+ env[RESERVATIONS_ENV] = started.socket;
62
+ }
63
+
64
+ /** Close the endpoint `context`'s run was served, if it was served one: its standard input's end ends it. */
65
+ export async function closeRunReservations(context: Pick<PostTasksExecutionContext, 'id'>): Promise<void> {
66
+ const endpoint = endpoints.get(context.id);
67
+ if (endpoint === undefined) {
68
+ return;
69
+ }
70
+ endpoints.delete(context.id);
71
+ endpoint.server.stdin.end();
72
+ const ended = await endedWithin(endpoint, CONTROL_MS);
73
+ await rm(endpoint.directory, { recursive: true, force: true });
74
+ if (ended !== 'mbx reservations ended with status 0') {
75
+ logger.warn(`bounded-exec: this run's mbx reservation endpoint did not close cleanly: ${ended}`);
76
+ }
77
+ }
78
+
79
+ /** Start `mbx reservations` by its PATH name in an owned socket directory, and wait until it says it listens. */
80
+ async function startEndpoint(env: NodeJS.ProcessEnv): Promise<Started> {
81
+ const directory = await ownedSocketDirectory(ENDPOINT_DIRECTORY, ENDPOINT_SOCKET);
82
+ const socket = join(directory, ENDPOINT_SOCKET);
83
+ const server = spawn('mbx', ['reservations', socket], { env, stdio: ['pipe', 'pipe', 'pipe'] });
84
+ let said = '';
85
+ server.stderr.setEncoding('utf8');
86
+ server.stderr.on('data', (text: string) => {
87
+ said = (said + text).slice(-SAID_BYTES);
88
+ });
89
+ // eslint-disable-next-line deadline/no-unbounded-wait -- WHY unbounded: a child that runs ends with its run's postTasksExecution, which ends its standard input and bounds its exit (`endedWithin`); one that cannot run says so at once
90
+ const ended = new Promise<string>((resolve) => {
91
+ server.once('error', (error) => resolve(`mbx reservations could not run: ${error.message}`));
92
+ server.once('close', (code, signal) => {
93
+ const last = said.trim().split('\n').at(-1) ?? '';
94
+ resolve(
95
+ `mbx reservations ended ${signal === null ? `with status ${code}` : `on ${signal}`}${last === '' ? '' : `: ${last}`}`,
96
+ );
97
+ });
98
+ });
99
+ const endpoint: Endpoint = { server, directory, ended };
100
+ const listening = await saysListening(server.stdout, ended);
101
+ if (listening !== null) {
102
+ server.kill('SIGKILL');
103
+ await endedWithin(endpoint, CONTROL_MS);
104
+ await rm(directory, { recursive: true, force: true });
105
+ return { ok: false, error: listening };
106
+ }
107
+ return { ok: true, endpoint, socket };
108
+ }
109
+
110
+ /** Null once the endpoint's first line is `listening`; otherwise why not, within the control-plane bound. */
111
+ function saysListening(stdout: Readable, ended: Promise<string>): Promise<string | null> {
112
+ return new Promise<string | null>((resolve) => {
113
+ let text = '';
114
+ const timer = setTimeout(
115
+ () => resolve(`mbx reservations did not say it was listening within ${CONTROL_MS}ms`),
116
+ CONTROL_MS,
117
+ );
118
+ stdout.setEncoding('utf8');
119
+ stdout.on('data', (chunk: string) => {
120
+ text += chunk;
121
+ const end = text.indexOf('\n');
122
+ if (end === -1) return;
123
+ clearTimeout(timer);
124
+ const line = text.slice(0, end);
125
+ resolve(line === 'listening' ? null : `mbx reservations said ${JSON.stringify(line)}, not that it was listening`);
126
+ });
127
+ void ended.then((why) => {
128
+ clearTimeout(timer);
129
+ resolve(why);
130
+ });
131
+ });
132
+ }
133
+
134
+ /** How the endpoint ended, killing it once `ms` pass first. */
135
+ async function endedWithin(endpoint: Endpoint, ms: number): Promise<string> {
136
+ let timer: NodeJS.Timeout | undefined;
137
+ const late = new Promise<null>((resolve) => (timer = setTimeout(() => resolve(null), ms)));
138
+ const ended = await Promise.race([endpoint.ended, late]);
139
+ clearTimeout(timer);
140
+ if (ended !== null) {
141
+ return ended;
142
+ }
143
+ endpoint.server.kill('SIGKILL');
144
+ return `${await endpoint.ended}, killed after it outlived its standard input by ${ms}ms`;
145
+ }
146
+
147
+ /** The pool's answer to one reservation: the cores it holds and a promise of why it was lost, or why there are none. */
148
+ export type Granted =
149
+ | { readonly ok: true; readonly cpus: number; readonly lost: Promise<string> }
150
+ | { readonly ok: false; readonly error: string };
151
+
152
+ /** One request for cores on a run's endpoint. */
153
+ export interface Reservation {
154
+ /** Settles once the pool grants or refuses it, or its connection ends first. */
155
+ readonly granted: Promise<Granted>;
156
+ /** Gives the grant back, or withdraws the request while it waits. Later calls do nothing. */
157
+ release(): void;
158
+ }
159
+
160
+ type Uint = number & tags.Type<'uint32'>;
161
+
162
+ /** The lines a client writes (mbx's `docs/scheduling.md`, "Serve reservations to many commands"). */
163
+ type ReservationRequest =
164
+ | { type: 'reserve'; id: Uint; cpus: Uint & tags.Minimum<1>; priority: 'normal' | 'low' }
165
+ | { type: 'release'; id: Uint };
166
+
167
+ /** The lines the endpoint answers; an added field is mbx's to add. */
168
+ type ReservationReply =
169
+ | { type: 'granted'; id: Uint; cpus: Uint & tags.Minimum<1> }
170
+ | { type: 'refused'; id: Uint; error: string }
171
+ | { type: 'error'; error: string };
172
+
173
+ const stringifyRequest = typia.json.createStringify<ReservationRequest>();
174
+ const parseReply = typia.json.createValidateParse<ReservationReply>();
175
+
176
+ /** A request's place on its connection: waiting for its answer, or held until released or lost. */
177
+ type Entry =
178
+ | { readonly kind: 'waiting'; readonly answer: (granted: Granted) => void }
179
+ | { readonly kind: 'held'; readonly lose: (why: string) => void };
180
+
181
+ /**
182
+ * One connection to an endpoint, shared by every reservation this process makes there. It keeps the process alive only
183
+ * while one of them waits or holds.
184
+ */
185
+ class Connection {
186
+ readonly #socket: Socket;
187
+ readonly #entries = new Map<number, Entry>();
188
+ #nextId = 1;
189
+ #unread = '';
190
+ #closed: string | null = null;
191
+
192
+ constructor(
193
+ private readonly endpoint: string,
194
+ private readonly forget: () => void,
195
+ ) {
196
+ // eslint-disable-next-line deadline/no-unbounded-wait -- WHY unbounded: a Unix-socket connect completes or fails at once, and a reservation's answer waits as long as the pool's admission holds it, as `mbx reserve`'s did; the connection's end answers every request still waiting
197
+ this.#socket = createConnection(endpoint);
198
+ this.#socket.setEncoding('utf8');
199
+ this.#socket.unref();
200
+ this.#socket.on('data', (text: string) => this.#read(text));
201
+ this.#socket.on('error', (error) => this.#close(`mbx reservation endpoint ${this.endpoint}: ${error.message}`));
202
+ this.#socket.on('close', () => this.#close(`mbx reservation endpoint ${this.endpoint} closed the connection`));
203
+ }
204
+
205
+ reserve(cpus: number, priority: 'normal' | 'low'): Reservation {
206
+ if (this.#closed !== null) {
207
+ return { granted: Promise.resolve({ ok: false, error: this.#closed }), release() {} };
208
+ }
209
+ const id = this.#nextId++;
210
+ let answer!: (granted: Granted) => void;
211
+ const granted = new Promise<Granted>((resolve) => (answer = resolve));
212
+ this.#entries.set(id, { kind: 'waiting', answer });
213
+ this.#socket.ref();
214
+ this.#socket.write(`${stringifyRequest({ type: 'reserve', id, cpus, priority })}\n`);
215
+ let released = false;
216
+ return {
217
+ granted,
218
+ release: () => {
219
+ if (released) return;
220
+ released = true;
221
+ const entry = this.#entries.get(id);
222
+ if (entry === undefined) return;
223
+ this.#entries.delete(id);
224
+ if (entry.kind === 'waiting') entry.answer({ ok: false, error: 'released while it waited' });
225
+ this.#socket.write(`${stringifyRequest({ type: 'release', id })}\n`);
226
+ if (this.#entries.size === 0) this.#socket.unref();
227
+ },
228
+ };
229
+ }
230
+
231
+ #read(text: string): void {
232
+ this.#unread += text;
233
+ for (let end = this.#unread.indexOf('\n'); end !== -1; end = this.#unread.indexOf('\n')) {
234
+ const line = this.#unread.slice(0, end);
235
+ this.#unread = this.#unread.slice(end + 1);
236
+ const parsed = parseReply(line);
237
+ if (!parsed.success) {
238
+ this.#close(`mbx reservation endpoint ${this.endpoint} answered ${JSON.stringify(line)}, which is no reply`);
239
+ return;
240
+ }
241
+ this.#answer(parsed.data);
242
+ if (this.#closed !== null) return;
243
+ }
244
+ }
245
+
246
+ #answer(reply: ReservationReply): void {
247
+ if (reply.type === 'error') {
248
+ this.#close(`mbx reservation endpoint ${this.endpoint} closed this connection: ${reply.error}`);
249
+ return;
250
+ }
251
+ const entry = this.#entries.get(reply.id);
252
+ // Released before its answer arrived: the endpoint drops what it won.
253
+ if (entry === undefined) return;
254
+ if (entry.kind === 'held') {
255
+ this.#close(`mbx reservation endpoint ${this.endpoint} answered reservation ${reply.id} twice`);
256
+ return;
257
+ }
258
+ if (reply.type === 'granted') {
259
+ let lose!: (why: string) => void;
260
+ const lost = new Promise<string>((resolve) => (lose = resolve));
261
+ this.#entries.set(reply.id, { kind: 'held', lose });
262
+ entry.answer({ ok: true, cpus: reply.cpus, lost });
263
+ return;
264
+ }
265
+ this.#entries.delete(reply.id);
266
+ entry.answer({ ok: false, error: reply.error });
267
+ if (this.#entries.size === 0) this.#socket.unref();
268
+ }
269
+
270
+ #close(why: string): void {
271
+ if (this.#closed !== null) return;
272
+ this.#closed = why;
273
+ this.forget();
274
+ for (const entry of this.#entries.values()) {
275
+ if (entry.kind === 'waiting') entry.answer({ ok: false, error: why });
276
+ else entry.lose(why);
277
+ }
278
+ this.#entries.clear();
279
+ this.#socket.destroy();
280
+ }
281
+ }
282
+
283
+ /** This process's connection to each endpoint it has asked. */
284
+ const connections = new Map<string, Connection>();
285
+
286
+ /**
287
+ * Ask the endpoint at `socket` for at most `cpus` cores at `priority`, over this process's one connection to it,
288
+ * opened by its first request and kept while another one may come.
289
+ */
290
+ export function reserveCores(socket: string, cpus: number, priority: 'normal' | 'low'): Reservation {
291
+ let connection = connections.get(socket);
292
+ if (connection === undefined) {
293
+ const opened: Connection = new Connection(socket, () => {
294
+ if (connections.get(socket) === opened) connections.delete(socket);
295
+ });
296
+ connections.set(socket, opened);
297
+ connection = opened;
298
+ }
299
+ return connection.reserve(cpus, priority);
300
+ }
@@ -56,8 +56,9 @@ export type ReservedDemand = Extract<Demand, { kind: 'nextest' | 'bun-parallel'
56
56
  export const CPU_TOKENS_ENV = 'BOUNDED_EXEC_CPU_TOKENS';
57
57
 
58
58
  /**
59
- * The cores mbx granted the reservation a leaf runs under: `mbx reserve --cpus N` names at most N to its command, as many
60
- * as were free once one was.
59
+ * The cores mbx granted the reservation a leaf runs under: its run's mbx reservation endpoint grants at most the N the
60
+ * leaf asks for, as many as were free once one was, and the leaf's shell reads that count into this variable
61
+ * (mbx-pool.ts, `reservation`), the one `mbx reserve --cpus N` names to its own command.
61
62
  */
62
63
  export const GRANTED_CPUS_ENV = 'MBX_RESERVE_CPUS';
63
64
 
@@ -369,13 +370,12 @@ export function sizeRunner(
369
370
  /**
370
371
  * The shell statements that run first in a reserved leaf's reservation and size its runner to the grant, which only
371
372
  * the command learns as it starts: `CPU_TOKENS_ENV` becomes the cores mbx granted (`GRANTED_CPUS_ENV`), never more
372
- * than the leaf asked for (`WANT_ENV`, set before it reserved), which also stands in under an mbx that grants a
373
- * reservation whole and names none. The test-thread variables a sizing sets follow from it, under their inherited
374
- * ceilings, outranking the target's own as a numeric grant's do (`sizeRunner`).
373
+ * than the leaf asked for (`WANT_ENV`, set before it reserved). The test-thread variables a sizing sets follow from it,
374
+ * under their inherited ceilings, outranking the target's own as a numeric grant's do (`sizeRunner`).
375
375
  */
376
376
  export function grantedSizing(demand: ReservedDemand): string {
377
377
  const asked = `$${WANT_ENV}`;
378
- const named = `\${${GRANTED_CPUS_ENV}:-${asked}}`;
378
+ const named = `$${GRANTED_CPUS_ENV}`;
379
379
  const tokens = `$${CPU_TOKENS_ENV}`;
380
380
  const atMost = (ceiling: number | null): string =>
381
381
  ceiling === null ? `"${tokens}"` : `"$(( ${tokens} < ${ceiling} ? ${tokens} : ${ceiling} ))"`;