@smoothbricks/nx-plugin 0.7.4 → 0.7.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/README.md +59 -30
  2. package/dist/.build-inputs.json +15 -8
  3. package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +3 -2
  4. package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
  5. package/dist/executors/bounded-exec/cpu-pool-thread.js +1 -1
  6. package/dist/executors/bounded-exec/cpu-pool.d.ts +24 -10
  7. package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
  8. package/dist/executors/bounded-exec/cpu-pool.js +96 -52
  9. package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
  10. package/dist/executors/bounded-exec/executor.js +192 -82
  11. package/dist/executors/bounded-exec/host-memory.d.ts +47 -0
  12. package/dist/executors/bounded-exec/host-memory.d.ts.map +1 -0
  13. package/dist/executors/bounded-exec/host-memory.darwin.d.ts +3 -0
  14. package/dist/executors/bounded-exec/host-memory.darwin.d.ts.map +1 -0
  15. package/dist/executors/bounded-exec/host-memory.darwin.js +40 -0
  16. package/dist/executors/bounded-exec/host-memory.js +29 -0
  17. package/dist/executors/bounded-exec/host-memory.linux.d.ts +14 -0
  18. package/dist/executors/bounded-exec/host-memory.linux.d.ts.map +1 -0
  19. package/dist/executors/bounded-exec/host-memory.linux.js +104 -0
  20. package/dist/executors/bounded-exec/leaf-peak.d.ts +55 -0
  21. package/dist/executors/bounded-exec/leaf-peak.d.ts.map +1 -0
  22. package/dist/executors/bounded-exec/leaf-peak.js +146 -0
  23. package/dist/executors/bounded-exec/mbx-pool.d.ts +17 -9
  24. package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -1
  25. package/dist/executors/bounded-exec/mbx-pool.js +29 -11
  26. package/dist/executors/bounded-exec/mbx-reservations.d.ts +41 -0
  27. package/dist/executors/bounded-exec/mbx-reservations.d.ts.map +1 -0
  28. package/dist/executors/bounded-exec/mbx-reservations.js +372 -0
  29. package/dist/executors/bounded-exec/peak-waiter.d.ts +2 -0
  30. package/dist/executors/bounded-exec/peak-waiter.d.ts.map +1 -0
  31. package/dist/executors/bounded-exec/peak-waiter.js +53 -0
  32. package/dist/executors/bounded-exec/pool-admission.d.ts +38 -0
  33. package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -0
  34. package/dist/executors/bounded-exec/pool-admission.js +31 -0
  35. package/dist/executors/bounded-exec/run-cpu-pool.d.ts +6 -4
  36. package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
  37. package/dist/executors/bounded-exec/run-cpu-pool.js +20 -7
  38. package/dist/executors/bounded-exec/runner-sizing.d.ts +5 -5
  39. package/dist/executors/bounded-exec/runner-sizing.d.ts.map +1 -1
  40. package/dist/executors/bounded-exec/runner-sizing.js +6 -6
  41. package/dist/executors/bounded-exec/verdict.d.ts +21 -2
  42. package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
  43. package/dist/executors/bounded-exec/verdict.js +27 -4
  44. package/dist/index.d.ts +1 -1
  45. package/dist/index.d.ts.map +1 -1
  46. package/dist/index.js +4 -1
  47. package/package.json +1 -1
  48. package/src/executors/bounded-exec/cpu-pool-thread.ts +4 -3
  49. package/src/executors/bounded-exec/cpu-pool.ts +111 -49
  50. package/src/executors/bounded-exec/executor.ts +202 -77
  51. package/src/executors/bounded-exec/host-memory.darwin.ts +44 -0
  52. package/src/executors/bounded-exec/host-memory.linux.ts +115 -0
  53. package/src/executors/bounded-exec/host-memory.ts +70 -0
  54. package/src/executors/bounded-exec/leaf-peak.ts +151 -0
  55. package/src/executors/bounded-exec/mbx-pool.ts +33 -13
  56. package/src/executors/bounded-exec/mbx-reservations.ts +300 -0
  57. package/src/executors/bounded-exec/peak-waiter.ts +82 -0
  58. package/src/executors/bounded-exec/pool-admission.ts +64 -0
  59. package/src/executors/bounded-exec/run-cpu-pool.ts +20 -6
  60. package/src/executors/bounded-exec/runner-sizing.ts +6 -6
  61. package/src/executors/bounded-exec/verdict.ts +40 -6
  62. package/src/index.ts +4 -1
  63. package/test-timings.json +134 -131
@@ -36,10 +36,20 @@ import {
36
36
  import { GATEWAY_SOCKET } from './gateway-lease.js';
37
37
  import type { HangDiagnostics } from './hang-diagnostics.js';
38
38
  import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
39
+ import { type LeafPeak, PEAK_FD, peakWaiter, readWaiterReport, underWaiter } from './leaf-peak.js';
39
40
  import type { CpuWaitRow } from './lease-wait.js';
40
41
  import type { LockWait } from './lock-wait.js';
41
- import { MBX_GRANT_FD, mbxIdleGated, mbxLeaf, mbxPool, withCountedReservation, withMbxLeaf } from './mbx-pool.js';
42
+ import {
43
+ askedCores,
44
+ MBX_GRANT_FD,
45
+ mbxIdleGated,
46
+ mbxLeaf,
47
+ mbxPool,
48
+ withCountedReservation,
49
+ withMbxLeaf,
50
+ } from './mbx-pool.js';
42
51
  import { MBX_QUEUE_POLL_MS, type MbxQueuedRequest, snapshotMbxQueue } from './mbx-queue.js';
52
+ import { RESERVATIONS_ENV, type Reservation, reserveCores } from './mbx-reservations.js';
43
53
  import { createNoSpaceWatch } from './no-space.js';
44
54
  import {
45
55
  COWSHED_EXECUTABLE_ENV,
@@ -83,6 +93,7 @@ import {
83
93
  judgeRun,
84
94
  type NoSpaceSighting,
85
95
  readJunitReport,
96
+ readLastPeak,
86
97
  taskDirectory,
87
98
  withJunitReport,
88
99
  writeRecord,
@@ -310,6 +321,8 @@ export async function runBoundedExec(
310
321
  ): Promise<BoundedExecResult> {
311
322
  const cwd = resolveCwd(options.cwd, context.root);
312
323
  const directory = context.record?.directory ?? null;
324
+ // The task's last recorded peak is its charge in its parent's pool, read before the clear below removes it.
325
+ const lastPeak = directory !== null && inherited !== null ? await readLastPeak(directory) : null;
313
326
  // A previous run's verdict and report go first, so neither can be read as this run's.
314
327
  if (directory !== null) {
315
328
  await clearTaskDirectory(directory);
@@ -357,6 +370,20 @@ export async function runBoundedExec(
357
370
  // Nobody is left to tell.
358
371
  }
359
372
  };
373
+ // A run refused before its command exists says why and records an operational failure, and no peak: nothing ran.
374
+ const refuseUnstarted = async (message: string): Promise<BoundedExecResult> => {
375
+ appendStderr(message);
376
+ if (context.record !== undefined) {
377
+ const { task, directory: recordDirectory } = context.record;
378
+ await writeRecord(recordDirectory, {
379
+ task: task.id,
380
+ hash: task.hash,
381
+ verdict: judgeOperationalFailure(1, null),
382
+ peakBytes: null,
383
+ });
384
+ }
385
+ return { success: false, terminalOutput: outputChunks.join('') };
386
+ };
360
387
 
361
388
  // The fixture pair names one segment: a fixture with no build budget, or a budget with no fixture, is a
362
389
  // target that does not say what it runs, so it does not run. Units are counted at the end of that segment.
@@ -365,14 +392,9 @@ export async function runBoundedExec(
365
392
  (options.unitsCommand !== undefined && options.fixtureCommand === undefined) ||
366
393
  (fixture !== null && !(fixture.timeoutMs > 0))
367
394
  ) {
368
- appendStderr(
395
+ return refuseUnstarted(
369
396
  'bounded-exec: fixtureCommand and a positive fixtureTimeoutMs must be named together, and unitsCommand only with them; refusing to run the command\n',
370
397
  );
371
- if (context.record !== undefined) {
372
- const { task, directory: recordDirectory } = context.record;
373
- await writeRecord(recordDirectory, { task: task.id, hash: task.hash, verdict: judgeOperationalFailure(1, null) });
374
- }
375
- return { success: false, terminalOutput: outputChunks.join('') };
376
398
  }
377
399
 
378
400
  // A leg that runs bun runs the Bun its workspace declares, or it does not run. This comes before the RAM lease and the
@@ -385,13 +407,15 @@ export async function runBoundedExec(
385
407
  pathSource: options.env?.PATH === undefined ? 'the inherited environment' : "the target's env",
386
408
  });
387
409
  if (runtime !== null) {
388
- appendStderr(describeRuntimePinRefusal(runtime, context.record?.task.id));
389
- if (context.record !== undefined) {
390
- const { task, directory: recordDirectory } = context.record;
391
- await writeRecord(recordDirectory, { task: task.id, hash: task.hash, verdict: judgeOperationalFailure(1, null) });
392
- }
393
- return { success: false, terminalOutput: outputChunks.join('') };
410
+ return refuseUnstarted(describeRuntimePinRefusal(runtime, context.record?.task.id));
411
+ }
412
+ if (lastPeak !== null && !lastPeak.ok) {
413
+ return refuseUnstarted(
414
+ `bounded-exec: ${lastPeak.error}; it is this task's memory charge, so the command does not run\n`,
415
+ );
394
416
  }
417
+ // A task with no recorded peak is charged nothing (null) until it has one.
418
+ const charge = lastPeak?.ok === true ? lastPeak.peakBytes : null;
395
419
 
396
420
  let lease: RamTempLease | null = null;
397
421
  let ordinaryTemp: string | null = null;
@@ -539,7 +563,7 @@ export async function runBoundedExec(
539
563
  let parentRefusal: string | null = null;
540
564
  let parentWait: CpuWaitRow | undefined;
541
565
  if (inherited !== null) {
542
- const fromParent = await takeParentCpuTokens(inherited, demand.want);
566
+ const fromParent = await takeParentCpuTokens(inherited, demand.want, charge);
543
567
  if (fromParent.granted) {
544
568
  grant = fromParent;
545
569
  } else if (fromParent.cause === 'absent') {
@@ -560,16 +584,24 @@ export async function runBoundedExec(
560
584
  sized = sizeRunner(demand, requested, grant.tokens);
561
585
  cpu = { tokens: grant.tokens, want: demand.want, waitedMs };
562
586
  appendStderr(
563
- `cowshed: cpu-pool wait done elapsed=${waitedMs}ms status=ok tokens=${grant.tokens}/${demand.want} runner=${demand.kind}\n`,
587
+ `cowshed: cpu-pool wait done elapsed=${waitedMs}ms status=ok tokens=${grant.tokens}/${demand.want} runner=${demand.kind} memory=${charge ?? 'none'}\n`,
564
588
  );
589
+ if (grant.oversized !== null) {
590
+ appendStderr(
591
+ `cowshed: cpu-pool granted while nothing else holds memory: this task's last peak of ${charge} bytes alone exceeds the run's ${grant.oversized.memory} bytes\n`,
592
+ );
593
+ }
565
594
  } else if (leaf?.kind === 'reserve') {
566
595
  sized = { command: sizeRunnerToGrant(leaf.demand, requested), env: {} };
567
596
  } else if (leaf !== null) {
568
597
  sized = sizeRunner(demand, requested, demand.want);
569
598
  }
570
599
  // The run's sub-pool and grant are given back exactly once, however the run ends. The sub-pool closes first, so no
571
- // descendant still draws on tokens the grant has returned.
600
+ // descendant still draws on tokens the grant has returned. A reserved leaf's mbx grant covers the leaf's conclusion,
601
+ // its subtree's retirement included (11_shell.md), so it goes back after `releaseLease`.
572
602
  let pool: CpuPool | null = null;
603
+ let reservation: Reservation | null = null;
604
+ const releaseReservation = (): void => reservation?.release();
573
605
  let cpuReleased: Promise<void> | null = null;
574
606
  const releaseCpu = (): Promise<void> => {
575
607
  cpuReleased ??= (async () => {
@@ -596,8 +628,9 @@ export async function runBoundedExec(
596
628
  let unconfirmedAdmission: string | null = null;
597
629
  /** When the last successful snapshot completed (`Date.now()`): credit runs to it and no further. */
598
630
  let confirmedAdmissionAt: number | null = null;
631
+ // A reserved leaf's own request on its run's endpoint is its admission wait; a Cargo leaf's comes from mbx's report.
599
632
  const queueWait = (): AdmissionQueueWait | null =>
600
- mbxAdmitted || lastQueueRequests.length === 0
633
+ mbxAdmitted || (lastQueueRequests.length === 0 && reservation === null)
601
634
  ? null
602
635
  : {
603
636
  waitedMs: admissionWaitMs(admissionWait, performance.now()),
@@ -620,6 +653,7 @@ export async function runBoundedExec(
620
653
  operational: boolean,
621
654
  noSpace: NoSpaceSighting | null,
622
655
  lockWaits: readonly LockWait[],
656
+ peakBytes: number | null,
623
657
  ): Promise<void> => {
624
658
  if (context.record === undefined) {
625
659
  return;
@@ -665,6 +699,7 @@ export async function runBoundedExec(
665
699
  verdict,
666
700
  ...(cpu ? { cpu } : {}),
667
701
  ...(parentWait ? { cpuWait: parentWait } : {}),
702
+ peakBytes,
668
703
  });
669
704
  };
670
705
  // A run refused before its command starts says why, gives back its grant and RAM, and records the failure.
@@ -673,17 +708,25 @@ export async function runBoundedExec(
673
708
  await releaseCpu();
674
709
  // The command never started, so its lease holds nothing of its own to keep.
675
710
  await releaseLease(false);
676
- await recordRun(1, null, Date.now() - asked, false, null, []);
711
+ await recordRun(1, null, Date.now() - asked, false, null, [], null);
677
712
  return { success: false, terminalOutput: outputChunks.join('') };
678
713
  };
679
714
  if (parentRefusal !== null) {
680
715
  return refuse(`cowshed: cpu-pool parent pool refused: ${parentRefusal}; refusing to run the command\n`);
681
716
  }
717
+ // A reserved leaf takes its cores from its Nx run's mbx reservation endpoint (mbx-reservations.ts). A run that serves
718
+ // none said why in its own output; the leaf does not run beside the pool.
719
+ const reserved = leaf?.kind === 'reserve' ? { leaf, endpoint: process.env[RESERVATIONS_ENV] ?? '' } : null;
720
+ if (reserved !== null && reserved.endpoint === '') {
721
+ return refuse(
722
+ `cowshed: a reserved leaf takes its cores from its Nx run's mbx reservation endpoint, and ${RESERVATIONS_ENV} names none (the run's preTasksExecution serves it); refusing to run the command\n`,
723
+ );
724
+ }
682
725
 
683
726
  // A granted run serves a sub-pool of its own grant, so its descendants draw from that and never from above it.
684
727
  // A pool that cannot be set up refuses the run: no granted command runs beside its grant.
685
728
  if (grant !== null) {
686
- const created = await createCpuPool(grant.tokens, grant.closed);
729
+ const created = await createCpuPool(grant.tokens, grant.closed, null);
687
730
  if (!created.ok) {
688
731
  return refuse(`cowshed: cpu pool unavailable: ${created.error}; refusing to run the command\n`);
689
732
  }
@@ -730,42 +773,72 @@ export async function runBoundedExec(
730
773
  : withMbxLeaf(unadmitted, leaf);
731
774
  // Only leaves known to be scheduled can exclude zero-hold time. Others keep lifecycle hang protection.
732
775
  const queued = mbxIdleGated(leaf, sized.command, { ...process.env, ...env });
733
- // fd 4 is unused without a subtree admission, so the grant takes fd 5 either way.
776
+ // fd 4 is unused without a subtree admission, so mbx's descriptor takes fd 5 either way: a Cargo leaf's admission
777
+ // stream, or a reserved leaf's ask for cores and their count (mbx-pool.ts, `MBX_GRANT_FD`).
734
778
  const grantSlots: IOType[] = leaf !== null ? ['ignore', 'pipe'] : [];
735
779
  const rootGrantSlot: IOType[] = leaf !== null ? ['pipe'] : [];
736
780
  const root = owner?.rootArgv(shellCommand);
781
+ // The command's own descriptors past stderr, from fd 3: the lifecycle channel, the subtree admission, mbx's grant.
782
+ const descriptors: IOType[] =
783
+ root === undefined
784
+ ? hooked === null && leaf === null
785
+ ? []
786
+ : [hooked === null ? 'ignore' : 'pipe', ...grantSlots]
787
+ : [hooked === null ? 'ignore' : 'pipe', 'pipe', ...rootGrantSlot];
788
+ // A run that records its verdict records its command's peak too: the command runs under a waiter that reports it.
789
+ const waiter = context.record === undefined ? null : peakWaiter();
790
+ const measured =
791
+ waiter?.kind === 'waiter'
792
+ ? underWaiter(waiter.runtime, root?.file ?? '/bin/sh', root?.args ?? ['-c', shellCommand], descriptors)
793
+ : null;
737
794
  let child: ChildProcess;
738
795
  const results = directory === null ? null : await createTestResultWriter(directory, { cwd, root: context.root });
739
796
  if (results !== null && !results.ok) return refuse(`bounded-exec: ${results.error}\n`);
740
797
  resultWriter = results?.ok ? results.writer : null;
741
798
  try {
742
799
  // fd 3 is the runner lifecycle's duplex socket when the command is hooked; the subtree admission takes fd 4, which
743
- // cowshed closes fd 4 before exec; fd 5 carries mbx's admission/release stream, including Cargo descendants.
800
+ // cowshed closes fd 4 before exec; fd 5 is a Cargo leaf's mbx admission stream or a reserved leaf's ask and grant;
801
+ // the waiter, when there is one, passes them on and reports the command's peak on fd 6.
744
802
  child =
745
- root === undefined
746
- ? spawn(shellCommand, [], {
803
+ measured !== null
804
+ ? spawn(measured.file, measured.args, {
747
805
  cwd,
748
806
  env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
749
- shell: true,
750
- stdio:
751
- hooked === null && leaf === null
752
- ? 'pipe'
753
- : ['pipe', 'pipe', 'pipe', hooked === null ? 'ignore' : 'pipe', ...grantSlots],
807
+ stdio: measured.stdio,
754
808
  detached: process.platform !== 'win32',
755
809
  windowsHide: true,
756
810
  })
757
- : spawn(root.file, root.args, {
758
- cwd,
759
- env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
760
- stdio: ['pipe', 'pipe', 'pipe', hooked === null ? 'ignore' : 'pipe', 'pipe', ...rootGrantSlot],
761
- detached: process.platform !== 'win32',
762
- windowsHide: true,
763
- });
811
+ : root === undefined
812
+ ? spawn(shellCommand, [], {
813
+ cwd,
814
+ env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
815
+ shell: true,
816
+ stdio: descriptors.length === 0 ? 'pipe' : ['pipe', 'pipe', 'pipe', ...descriptors],
817
+ detached: process.platform !== 'win32',
818
+ windowsHide: true,
819
+ })
820
+ : spawn(root.file, root.args, {
821
+ cwd,
822
+ env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
823
+ stdio: ['pipe', 'pipe', 'pipe', ...descriptors],
824
+ detached: process.platform !== 'win32',
825
+ windowsHide: true,
826
+ });
764
827
  } catch (error) {
765
828
  // Nothing started, so nothing runs under the grant: it goes back here, with the RAM lease.
766
829
  return refuse(`cowshed: could not start the command: ${error instanceof Error ? error.message : String(error)}\n`);
767
830
  }
768
831
  subtreeRetired = false;
832
+ const reportChannel = child.stdio.at(PEAK_FD);
833
+ const waiterReport =
834
+ measured === null ? null : readWaiterReport(reportChannel instanceof Readable ? reportChannel : null);
835
+ // The leaf's runner is its command: under a waiter, which leads the group, the child the waiter names; else the
836
+ // group's leader itself. A hang snapshot names and samples it (specs/cowshed/08_testing.md).
837
+ const runnerPid: Promise<number | null> = waiterReport?.started ?? Promise.resolve(child.pid ?? null);
838
+ let runner: number | null = null;
839
+ void runnerPid.then((pid) => {
840
+ runner = pid;
841
+ });
769
842
  if (root !== undefined) {
770
843
  const channel = child.stdio[HANDLE_FD];
771
844
  const leased = lease;
@@ -825,22 +898,24 @@ export async function runBoundedExec(
825
898
  captureTimer = setTimeout(
826
899
  () => {
827
900
  if (state.settled || state.expiry !== null || idleCapture !== capture) return;
828
- const pid = child.pid;
829
- if (pid === undefined) {
901
+ const group = child.pid;
902
+ if (group === undefined) {
830
903
  idleCapture.result = { kind: 'failed', detail: 'runner PID unavailable; no holder conclusion' };
831
904
  return;
832
905
  }
833
906
  idleCapture.result = { kind: 'pending', startedAtMs: Date.now() };
834
- void Promise.resolve()
835
- .then(() =>
836
- captureDiagnostics({
907
+ void runnerPid
908
+ .then((pid) => {
909
+ if (pid === null)
910
+ throw new Error('runner PID unavailable: its waiter named no command; no holder conclusion');
911
+ return captureDiagnostics({
837
912
  pid,
838
- pgid: process.platform === 'win32' ? null : pid,
913
+ pgid: process.platform === 'win32' ? null : group,
839
914
  expiresAtMs: capture.expiresAtMs,
840
915
  deadlineMs: Math.max(0, Math.min(HANG_CAPTURE_MS, capture.expiresAtMs - Date.now())),
841
916
  signal: capture.controller.signal,
842
- }),
843
- )
917
+ });
918
+ })
844
919
  .then(
845
920
  (snapshot) => {
846
921
  if (idleCapture === capture) idleCapture.result = { kind: 'captured', snapshot };
@@ -882,7 +957,7 @@ export async function runBoundedExec(
882
957
  if (state.settled || state.expiry !== null) return;
883
958
  const scheduled = earliestBound(checkingIdle ? { ...bounds, idle: null } : bounds);
884
959
  if (scheduled?.kind !== kind || scheduled.at !== at) return;
885
- if (kind === 'idle' && leaf !== null) {
960
+ if (kind === 'idle' && leaf?.kind === 'cargo') {
886
961
  if (checkingIdle) return;
887
962
  checkingIdle = true;
888
963
  armBoundTimer();
@@ -937,8 +1012,6 @@ export async function runBoundedExec(
937
1012
  };
938
1013
  const refreshQueue = async (fresh: boolean): Promise<void> => {
939
1014
  if (fresh && queueSnapshot !== null) await queueSnapshot;
940
- // A leaf that counts its units before reserving waits in no reservation queue while its fixture builds.
941
- if (counted && !fixtureDone) return;
942
1015
  if (state.settled || state.expiry !== null || queueAbort.signal.aborted || child.pid === undefined) return;
943
1016
  const pid = child.pid;
944
1017
  queueSnapshot ??= (async () => {
@@ -1079,13 +1152,14 @@ export async function runBoundedExec(
1079
1152
  clearTimeout(boundTimer);
1080
1153
  // Preserve explicit live membership before cancellation removes the queued process. A failed bounded observation
1081
1154
  // leaves the admission unconfirmed, never a manufactured queue or admission.
1082
- signalCleanup = (leaf === null ? Promise.resolve() : refreshQueue(false))
1155
+ signalCleanup = (leaf?.kind !== 'cargo' ? Promise.resolve() : refreshQueue(false))
1083
1156
  .finally(() => {
1084
1157
  stopQueueObservation();
1085
1158
  return killChildTree();
1086
1159
  })
1087
1160
  .finally(releaseCpu)
1088
1161
  .finally(() => releaseLease(true))
1162
+ .finally(releaseReservation)
1089
1163
  .finally(async () =>
1090
1164
  recordRun(
1091
1165
  signalToExitCode(signal),
@@ -1094,6 +1168,7 @@ export async function runBoundedExec(
1094
1168
  state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null,
1095
1169
  state.noSpace === null ? null : await state.noSpace,
1096
1170
  state.lockWaits,
1171
+ null,
1097
1172
  ),
1098
1173
  )
1099
1174
  .finally(() => process.kill(process.pid, signal));
@@ -1162,7 +1237,7 @@ export async function runBoundedExec(
1162
1237
  const observed = idleCapture?.result;
1163
1238
  state.lockWaits = observed?.kind === 'captured' ? observed.snapshot.lockWaits : [];
1164
1239
  appendStderr(
1165
- `Hang diagnostics: runner pid=${child.pid ?? 'unavailable'} pgid=${
1240
+ `Hang diagnostics: runner pid=${runner ?? 'unavailable'} pgid=${
1166
1241
  process.platform === 'win32' ? 'unavailable' : (child.pid ?? 'unavailable')
1167
1242
  } expiryAtMs=${Date.now()} scheduledExpiryAtMs=${idleCapture?.expiresAtMs ?? 'unavailable'}\n`,
1168
1243
  );
@@ -1257,46 +1332,87 @@ export async function runBoundedExec(
1257
1332
  fixtureFirst: counted && lifecycle !== null,
1258
1333
  queued,
1259
1334
  });
1260
- // Read every byte after spawn accounting: a chunk can contain several grants and releases. EOF is neither.
1261
1335
  if (leaf !== null) {
1262
1336
  const channel = child.stdio.at(MBX_GRANT_FD);
1263
- if (!(channel instanceof Readable)) throw new Error('an mbx leaf requires its grant descriptor');
1264
- let said = false;
1337
+ if (!(channel instanceof Duplex)) throw new Error('an mbx leaf requires its mbx descriptor');
1265
1338
  const failGrant = (detail: string): void => {
1266
1339
  if (lifecycleFailure !== null || state.expiry !== null || state.settled) return;
1267
1340
  lifecycleFailure ??= `mbx grant descriptor: ${detail}`;
1268
1341
  appendStderr(`bounded-exec: ${lifecycleFailure}\n`);
1269
1342
  termination = terminate();
1270
1343
  };
1271
- channel.on('data', (chunk: Buffer) => {
1272
- for (const byte of chunk) {
1273
- if (byte === 103) {
1274
- if (!said) {
1275
- mbxAdmitted = true;
1276
- said = true;
1277
- tell(
1278
- leaf.kind === 'reserve'
1279
- ? `cowshed: mbx reserve granted elapsed=${Date.now() - startedAt}ms priority=${leaf.priority}\n`
1280
- : `cowshed: mbx cargo granted elapsed=${Date.now() - startedAt}ms\n`,
1281
- );
1344
+ channel.on('error', (error) => failGrant(error.message));
1345
+ if (reserved !== null) {
1346
+ // The shell asks for its cores in one line and reads the count granted (mbx-pool.ts, `reservation`). A refusal
1347
+ // ends the descriptor unanswered: the shell's read fails, and nothing runs.
1348
+ const { leaf: reservedLeaf, endpoint } = reserved;
1349
+ let asked = '';
1350
+ channel.setEncoding('utf8');
1351
+ channel.on('data', (text: string) => {
1352
+ if (reservation !== null) {
1353
+ failGrant(`the command asked for cores a second time: ${JSON.stringify(text)}`);
1354
+ return;
1355
+ }
1356
+ asked += text;
1357
+ const end = asked.indexOf('\n');
1358
+ if (end === -1) return;
1359
+ const line = asked.slice(0, end);
1360
+ const cpus = askedCores(line, reservedLeaf.demand.want);
1361
+ if (cpus === null) {
1362
+ failGrant(`the command asked for ${JSON.stringify(line)} cores, not 1 to ${reservedLeaf.demand.want}`);
1363
+ channel.end();
1364
+ return;
1365
+ }
1366
+ admissionWait = stepAdmissionWait(admissionWait, { kind: 'snapshot', at: performance.now(), waiting: true });
1367
+ tell(`cowshed: mbx reserve wait start cpus=${cpus} priority=${reservedLeaf.priority}\n`);
1368
+ const held = reserveCores(endpoint, cpus, reservedLeaf.priority);
1369
+ reservation = held;
1370
+ void held.granted.then((granted) => {
1371
+ if (!granted.ok) {
1372
+ failGrant(`mbx did not grant the reservation: ${granted.error}`);
1373
+ channel.end();
1374
+ return;
1282
1375
  }
1376
+ // A run that ended while it waited gives the grant back with its CPU (`releaseCpu`).
1377
+ if (state.settled || state.expiry !== null) return;
1378
+ mbxAdmitted = true;
1379
+ tell(`cowshed: mbx reserve granted elapsed=${Date.now() - startedAt}ms priority=${reservedLeaf.priority}\n`);
1380
+ channel.end(`${granted.cpus}\n`);
1283
1381
  stepTo({ kind: 'granted', at: Date.now(), testing: isTesting(), fixture: !fixtureDone });
1284
- } else if (byte === 114) {
1285
- stepTo({ kind: 'released', at: Date.now() });
1286
- } else {
1287
- failGrant(`unexpected byte ${byte}`);
1288
- return;
1382
+ void granted.lost.then((why) => {
1383
+ if (state.settled || cpuReleased !== null) return;
1384
+ appendStderr(
1385
+ `cowshed: mbx reservation lost while the command ran (${why}); it runs on unheld until it ends\n`,
1386
+ );
1387
+ });
1388
+ });
1389
+ });
1390
+ } else {
1391
+ // Read every byte after spawn accounting: a chunk can contain several grants and releases. EOF is neither.
1392
+ let said = false;
1393
+ channel.on('data', (chunk: Buffer) => {
1394
+ for (const byte of chunk) {
1395
+ if (byte === 103) {
1396
+ if (!said) {
1397
+ mbxAdmitted = true;
1398
+ said = true;
1399
+ tell(`cowshed: mbx cargo granted elapsed=${Date.now() - startedAt}ms\n`);
1400
+ }
1401
+ stepTo({ kind: 'granted', at: Date.now(), testing: isTesting(), fixture: !fixtureDone });
1402
+ } else if (byte === 114) {
1403
+ stepTo({ kind: 'released', at: Date.now() });
1404
+ } else {
1405
+ failGrant(`unexpected byte ${byte}`);
1406
+ return;
1407
+ }
1289
1408
  }
1290
- }
1291
- });
1292
- channel.on('error', (error) => failGrant(error.message));
1293
- }
1294
-
1295
- if (leaf !== null) {
1296
- queueTimer = setInterval(() => {
1409
+ });
1410
+ // Only mbx's activity report says whether a Cargo leaf's compiler or test waits in the pool.
1411
+ queueTimer = setInterval(() => {
1412
+ void refreshQueue(false);
1413
+ }, MBX_QUEUE_POLL_MS);
1297
1414
  void refreshQueue(false);
1298
- }, MBX_QUEUE_POLL_MS);
1299
- void refreshQueue(false);
1415
+ }
1300
1416
  }
1301
1417
 
1302
1418
  // A grant's connection is a ledger entry, not the command's owner: the lending parent's exit ends it, and what runs
@@ -1356,11 +1472,20 @@ export async function runBoundedExec(
1356
1472
  }
1357
1473
  }
1358
1474
  await releaseLease(code !== 0 || state.expiry !== null || state.poolCloseError !== null || lifecycleFailure !== null);
1475
+ releaseReservation();
1476
+
1477
+ // The command's peak, as its waiter reported it once both ended; a peak that was not measured is said, never zero.
1478
+ const reported: LeafPeak | null =
1479
+ waiterReport === null ? (waiter?.kind === 'unmeasured' ? waiter : null) : await waiterReport.peak;
1480
+ if (reported?.kind === 'unmeasured') {
1481
+ appendStderr(`bounded-exec: this run's memory peak was not measured (${reported.reason}); its record names none\n`);
1482
+ }
1483
+ const peakBytes = reported?.kind === 'measured' ? reported.bytes : null;
1359
1484
 
1360
1485
  // Broken resource ownership, a lost native lifecycle, or leftovers that could not be retired fail even if a shell
1361
1486
  // branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
1362
1487
  const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
1363
- await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits);
1488
+ await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits, peakBytes);
1364
1489
  return {
1365
1490
  success: !operationalFailure && resultFailure === null && state.expiry === null && code === 0,
1366
1491
  terminalOutput: outputChunks.join(''),
@@ -0,0 +1,44 @@
1
+ import { execFile } from 'node:child_process';
2
+ import { promisify } from 'node:util';
3
+ import type { HostMemory, RunMemoryReading } from './host-memory.js';
4
+
5
+ /**
6
+ * macOS's memory mechanisms. A run's memory is the host's, `hw.memsize`: macOS confines no job to a memory limit of
7
+ * its own. Bun reports a leaf's `ru_maxrss` in bytes, as macOS does.
8
+ */
9
+
10
+ const run = promisify(execFile);
11
+ const HW_MEMSIZE = 'hw.memsize';
12
+
13
+ async function readRunMemory(): Promise<RunMemoryReading> {
14
+ let stdout: string;
15
+ try {
16
+ ({ stdout } = await run('sysctl', ['-n', HW_MEMSIZE]));
17
+ } catch (error) {
18
+ return {
19
+ ok: false,
20
+ error: {
21
+ operation: 'read-run-memory',
22
+ code: 'unreadable',
23
+ subject: HW_MEMSIZE,
24
+ cause: error instanceof Error ? error.message : String(error),
25
+ },
26
+ };
27
+ }
28
+ const value = stdout.trim();
29
+ const bytes = Number(value);
30
+ if (!/^\d+$/.test(value) || !Number.isSafeInteger(bytes) || bytes === 0) {
31
+ return {
32
+ ok: false,
33
+ error: {
34
+ operation: 'read-run-memory',
35
+ code: 'malformed',
36
+ subject: HW_MEMSIZE,
37
+ cause: `sysctl printed ${JSON.stringify(value)}, not a byte count`,
38
+ },
39
+ };
40
+ }
41
+ return { ok: true, memory: { bytes, source: HW_MEMSIZE } };
42
+ }
43
+
44
+ export const darwinMemory: HostMemory = { readRunMemory, peakBytes: (maxRss) => maxRss };
@@ -0,0 +1,115 @@
1
+ import { readFile } from 'node:fs/promises';
2
+ import { posix } from 'node:path';
3
+ import type { HostMemory, RunMemory, RunMemoryError, RunMemoryReading } from './host-memory.js';
4
+
5
+ /**
6
+ * Linux's memory mechanisms. The run's memory is the least of the host's `MemTotal` and every finite `memory.max` of
7
+ * the cgroup v2 the run's process is in and of its ancestors, up to the root of the mounted hierarchy: the kernel holds
8
+ * a cgroup to its ancestors' limits too, so a job in cowshed's `background` class, whose own `memory.max` is `max`, has
9
+ * the class's three quarters of the host. A host with no cgroup v2 hierarchy, or a hierarchy that sets no limit, gives
10
+ * the run `MemTotal`.
11
+ */
12
+
13
+ const MEMINFO = '/proc/meminfo';
14
+ const CGROUP = '/proc/self/cgroup';
15
+ const MOUNTINFO = '/proc/self/mountinfo';
16
+
17
+ /** A file's text, or the errno code and message its read failed with. */
18
+ export type FileRead =
19
+ | { readonly ok: true; readonly text: string }
20
+ | { readonly ok: false; readonly code: string; readonly message: string };
21
+
22
+ const failed = (code: RunMemoryError['code'], subject: string, cause: string): RunMemoryReading => ({
23
+ ok: false,
24
+ error: { operation: 'read-run-memory', code, subject, cause },
25
+ });
26
+ const unreadable = (subject: string, read: Extract<FileRead, { ok: false }>): RunMemoryReading =>
27
+ failed('unreadable', subject, `${read.code}: ${read.message}`);
28
+
29
+ /** A mount point or root as mountinfo writes it, its space, tab, newline and backslash escaped in octal. */
30
+ const decodeOctal = (field: string): string =>
31
+ field.replace(/\\([0-7]{3})/g, (_, octal: string) => String.fromCharCode(Number.parseInt(octal, 8)));
32
+
33
+ /** The cgroup v2 mounts mountinfo lists: the cgroup each shows (`root`) and where (`point`). */
34
+ function cgroup2Mounts(mountinfo: string): { root: string; point: string }[] {
35
+ const mounts: { root: string; point: string }[] = [];
36
+ for (const line of mountinfo.split('\n')) {
37
+ const fields = line.split(' ');
38
+ const separator = fields.indexOf('-');
39
+ const [root, point] = [fields[3], fields[4]];
40
+ if (separator !== -1 && fields[separator + 1] === 'cgroup2' && root !== undefined && point !== undefined) {
41
+ mounts.push({ root: decodeOctal(root), point: decodeOctal(point) });
42
+ }
43
+ }
44
+ return mounts;
45
+ }
46
+
47
+ /** The run's memory, reading every file through `read`. */
48
+ export async function linuxRunMemory(read: (path: string) => Promise<FileRead>): Promise<RunMemoryReading> {
49
+ const meminfo = await read(MEMINFO);
50
+ if (!meminfo.ok) return unreadable(MEMINFO, meminfo);
51
+ const total = /^MemTotal:\s+(\d+) kB$/m.exec(meminfo.text)?.[1];
52
+ if (total === undefined) return failed('malformed', MEMINFO, 'it names no MemTotal in kB');
53
+ let least: RunMemory = { bytes: Number(total) * 1024, source: `${MEMINFO} MemTotal` };
54
+ const cgroups = await read(CGROUP);
55
+ if (!cgroups.ok) return unreadable(CGROUP, cgroups);
56
+ const path = /^0::(\/.*)$/m.exec(cgroups.text)?.[1];
57
+ if (path === undefined) {
58
+ // No cgroup v2 hierarchy holds the run: the host's memory is its memory.
59
+ return { ok: true, memory: least };
60
+ }
61
+ const mounts = await read(MOUNTINFO);
62
+ if (!mounts.ok) return unreadable(MOUNTINFO, mounts);
63
+ const mount = cgroup2Mounts(mounts.text).find(
64
+ ({ root }) => root === '/' || path === root || path.startsWith(`${root}/`),
65
+ );
66
+ if (mount === undefined) {
67
+ return failed('unmounted', path, `${MOUNTINFO} lists no cgroup2 mount that shows the run's cgroup`);
68
+ }
69
+ const top = posix.resolve(mount.point);
70
+ const own = posix.resolve(top, `.${mount.root === '/' ? path : path.slice(mount.root.length)}`);
71
+ if (own !== top && !own.startsWith(`${top}/`)) {
72
+ return failed('unmounted', path, `the run's cgroup lies outside the cgroup2 mount at ${top}`);
73
+ }
74
+ for (let directory = own; ; ) {
75
+ const file = posix.join(directory, 'memory.max');
76
+ const limit = await read(file);
77
+ if (limit.ok) {
78
+ const value = limit.text.trim();
79
+ if (value !== 'max') {
80
+ if (!/^\d+$/.test(value)) {
81
+ return failed('malformed', file, `it holds ${JSON.stringify(value)}, not max or bytes`);
82
+ }
83
+ // A limit above the host's memory (a near-2^63 one included) is no constraint, and is never made a number.
84
+ if (BigInt(value) < BigInt(least.bytes)) least = { bytes: Number(value), source: file };
85
+ }
86
+ } else if (limit.code !== 'ENOENT') {
87
+ // ENOENT: the hierarchy's root, or a cgroup whose parent enables no memory controller, has no limit of its own.
88
+ return unreadable(file, limit);
89
+ }
90
+ if (directory === top) break;
91
+ directory = posix.dirname(directory);
92
+ }
93
+ return { ok: true, memory: least };
94
+ }
95
+
96
+ async function readText(path: string): Promise<FileRead> {
97
+ try {
98
+ return { ok: true, text: await readFile(path, 'utf8') };
99
+ } catch (error) {
100
+ const code = error instanceof Error && 'code' in error && typeof error.code === 'string' ? error.code : 'EIO';
101
+ return { ok: false, code, message: error instanceof Error ? error.message : String(error) };
102
+ }
103
+ }
104
+
105
+ /** Bun before 1.4.0 reports Linux's `ru_maxrss` as the kernel does, in KiB; from 1.4.0 it reports bytes. */
106
+ function peakBytes(maxRss: number, bunVersion: string): number {
107
+ const version = /^(\d+)\.(\d+)\./.exec(bunVersion);
108
+ if (version === null) {
109
+ throw new Error(`Bun names its version ${JSON.stringify(bunVersion)}, which is no semver`);
110
+ }
111
+ const [major, minor] = [Number(version[1]), Number(version[2])];
112
+ return major < 1 || (major === 1 && minor < 4) ? maxRss * 1024 : maxRss;
113
+ }
114
+
115
+ export const linuxMemory: HostMemory = { readRunMemory: () => linuxRunMemory(readText), peakBytes };